Skip to contentSkip to conversation

fs: fat: dynamically allocate memory for temporary buffer

PATCH Superseded tien.fong.chee at intel.com 3 participants 9 messages Jan 31, 2019 – Feb 5, 2019 VFAT/FAT/msdos filesystem

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 fs: fat: dynamically allocate memory for temporary buffer

From: Tien Fong Chee <[email protected]>

Drop the statically allocated get_contents_vfatname_block and dynamically allocate a buffer only if required. This saves 64KiB of memory.

---

changes for v2
- Removed the change for debug message.
- Set allocation based on actual required size instead of default max cluster size

---
 fs/fat/fat.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)
fs/fat/fat.c +12 -5
index ecfa255..347787e 100644
@@ -306,9 +306,6 @@ get_cluster(fsdata *mydata, __u32 clustnum, __u8 *buffer, unsigned long size)
  * into 'buffer'.
  * Update the number of bytes read in *gotsize or return -1 on fatal errors.
  */
-__u8 get_contents_vfatname_block[MAX_CLUSTSIZE]
-	__aligned(ARCH_DMA_MINALIGN);
-
 static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos,
 			__u8 *buffer, loff_t maxsize, loff_t *gotsize)
 {
@@ -351,15 +348,25 @@ static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos,
 
 	/* align to beginning of next cluster if any */
 	if (pos) {
+		__u8 *tmp_buffer;
+
 		actsize = min(filesize, (loff_t)bytesperclust);
-		if (get_cluster(mydata, curclust, get_contents_vfatname_block,
+		tmp_buffer = malloc_cache_aligned(actsize);
+		if (!tmp_buffer) {
+			debug("Error: allocating buffer\n");
+			return -ENOMEM;
+		}
+
+		if (get_cluster(mydata, curclust, tmp_buffer,
 				(int)actsize) != 0) {
 			printf("Error reading cluster\n");
+			free(tmp_buffer);
 			return -1;
 		}
 		filesize -= actsize;
 		actsize -= pos;
-		memcpy(buffer, get_contents_vfatname_block + pos, actsize);
+		memcpy(buffer, tmp_buffer + pos, actsize);
+		free(tmp_buffer);
 		*gotsize += actsize;
 		if (!filesize)
 			return 0;
Signature · 1 line
2.2.0
#1 lore raw

PATCH v2 2/2 fs: fat: Reduce default max clustersize 64KiB from malloc pool

From: Tien Fong Chee <[email protected]>

Release cluster block immediately when no longer use would help to reduce 64KiB memory allocated to the memory pool.

---

changes for v2
- Assigned NULL to itr after free.
- Added NULL checking to itr, avoid freeing twice.

---
 fs/fat/fat.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
fs/fat/fat.c +11 -2
index 347787e..fa846ff 100644
@@ -1152,12 +1152,21 @@ int file_fat_read_at(const char *filename, loff_t pos, void *buffer,
 		goto out_free_both;
 
 	debug("reading %s at pos %llu\n", filename, pos);
-	ret = get_contents(&fsdata, itr->dent, pos, buffer, maxsize, actread);
+
+	/* For saving default max clustersize memory allocated to malloc pool */
+	dir_entry *dentptr = itr->dent;
+
+	free(itr);
+
+	itr = NULL;
+
+	ret = get_contents(&fsdata, dentptr, pos, buffer, maxsize, actread);
 
 out_free_both:
 	free(fsdata.fatbuf);
 out_free_itr:
-	free(itr);
+	if (itr)
+		free(itr);
 	return ret;
 }
 
Signature · 1 line
2.2.0
#2 lore raw

PATCH v2 2/2 fs: fat: Reduce default max clustersize 64KiB from malloc pool

Quoted 31 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Release cluster block immediately when no longer use would help to reduce 64KiB memory allocated to the memory pool.

---

changes for v2
- Assigned NULL to itr after free.
- Added NULL checking to itr, avoid freeing twice.

---
 fs/fat/fat.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
fs/fat/fat.c +6 -1
index 347787e..fa846ff 100644
@@ -1152,12 +1152,21 @@ int file_fat_read_at(const char *filename, loff_t pos, void *buffer,
 		goto out_free_both;
 
 	debug("reading %s at pos %llu\n", filename, pos);
-	ret = get_contents(&fsdata, itr->dent, pos, buffer, maxsize, actread);
+
+	/* For saving default max clustersize memory allocated to malloc pool */
+	dir_entry *dentptr = itr->dent;
+
+	free(itr);
+

fs/fat/fat.c int file_fat_read_at(const char *filename, loff_t pos, void *buffer, · line 1169

Quoted 10 lines
fs/fat/fat.c +5 -1
+	itr = NULL;
+
+	ret = get_contents(&fsdata, dentptr, pos, buffer, maxsize, actread);
 
 out_free_both:
 	free(fsdata.fatbuf);
 out_free_itr:
-	free(itr);
+	if (itr)
+		free(itr);

free(NULL) is valid, so you can drop the if conditional.

Quoted 4 lines
fs/fat/fat.c +0 -0
 	return ret;
 }
 
Signature · 2 lines
Best regards,
Marek Vasut
#3 lore raw

PATCH v2 2/2 fs: fat: Reduce default max clustersize 64KiB from malloc pool

Quoted 41 lines · Marek Vasut
Quoted 40 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Release cluster block immediately when no longer use would help to reduce
64KiB memory allocated to the memory pool.

---

changes for v2
- Assigned NULL to itr after free.
- Added NULL checking to itr, avoid freeing twice.
---
 fs/fat/fat.c | 13 +++++++++++--

 1 file changed, 11 insertions(+), 2 deletions(-)
fs/fat/fat.c +0 -0
index 347787e..fa846ff 100644
@@ -1152,12 +1152,21 @@ int file_fat_read_at(const char *filename,
 loff_t pos, void *buffer,
  		goto out_free_both;


  debug("reading %s at pos %llu\n", filename, pos);

fs/fat/fat.c +0 -1
-	ret = get_contents(&fsdata, itr->dent, pos, buffer,

maxsize, actread);

fs/fat/fat.c +2 -0
+
+	/* For saving default max clustersize memory allocated to

malloc pool */

fs/fat/fat.c +7 -0
+	dir_entry *dentptr = itr->dent;
+
+	free(itr);
+
+	itr = NULL;
+
+	ret = get_contents(&fsdata, dentptr, pos, buffer, maxsize,

actread);

Quoted 8 lines

fs/fat/fat.c int file_fat_read_at(const char *filename, loff_t pos, void *buffer, · line 1169

Quoted 7 lines


 out_free_both:
  free(fsdata.fatbuf);
 out_free_itr:

fs/fat/fat.c +2 -1
-	free(itr);
+	if (itr)
+		free(itr);

free(NULL) is valid, so you can drop the if conditional.

Noted.

Quoted 7 lines
Quoted 6 lines

  return ret;
 }

#4 lore raw

PATCH v2 1/2 fs: fat: dynamically allocate memory for temporary buffer

In reply to tien.fong.chee at intel.com
Quoted 39 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Drop the statically allocated get_contents_vfatname_block and dynamically allocate a buffer only if required. This saves 64KiB of memory.

---

changes for v2
- Removed the change for debug message.
- Set allocation based on actual required size instead of default max cluster size

---
 fs/fat/fat.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)
fs/fat/fat.c +2 -3
index ecfa255..347787e 100644
@@ -306,9 +306,6 @@ get_cluster(fsdata *mydata, __u32 clustnum, __u8 *buffer, unsigned long size)
  * into 'buffer'.
  * Update the number of bytes read in *gotsize or return -1 on fatal errors.
  */
-__u8 get_contents_vfatname_block[MAX_CLUSTSIZE]
-	__aligned(ARCH_DMA_MINALIGN);
-
 static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos,
 			__u8 *buffer, loff_t maxsize, loff_t *gotsize)
 {
@@ -351,15 +348,25 @@ static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos,
 
 	/* align to beginning of next cluster if any */
 	if (pos) {
+		__u8 *tmp_buffer;
+

fs/fat/fat.c static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos, · line 361

Quoted 10 lines
fs/fat/fat.c +7 -1
 		actsize = min(filesize, (loff_t)bytesperclust);
-		if (get_cluster(mydata, curclust, get_contents_vfatname_block,
+		tmp_buffer = malloc_cache_aligned(actsize);
+		if (!tmp_buffer) {
+			debug("Error: allocating buffer\n");
+			return -ENOMEM;
+		}
+
+		if (get_cluster(mydata, curclust, tmp_buffer,
 				(int)actsize) != 0) {

Is the cast of actsize needed ?

fs/fat/fat.c static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos, · line 368

Quoted 8 lines
fs/fat/fat.c +2 -1
 			printf("Error reading cluster\n");
+			free(tmp_buffer);
 			return -1;
 		}
 		filesize -= actsize;
 		actsize -= pos;
-		memcpy(buffer, get_contents_vfatname_block + pos, actsize);
+		memcpy(buffer, tmp_buffer + pos, actsize);

Hmmm, tmp_buffer is actsize big , but you're memcpy-ing actsize here, so this would mean you memcpy data past the end of tmp_buffer if pos > 0, no?

Quoted 5 lines
fs/fat/fat.c +1 -0
+		free(tmp_buffer);
 		*gotsize += actsize;
 		if (!filesize)
 			return 0;
Signature · 2 lines
Best regards,
Marek Vasut
#5 lore raw

PATCH v2 1/2 fs: fat: dynamically allocate memory for temporary buffer

Quoted 51 lines · Marek Vasut
Quoted 50 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Drop the statically allocated get_contents_vfatname_block and dynamically allocate a buffer only if required. This saves 64KiB of memory.

---

changes for v2
- Removed the change for debug message.
- Set allocation based on actual required size instead of default max
  cluster size
---
 fs/fat/fat.c | 17 ++++++++++++-----

 1 file changed, 12 insertions(+), 5 deletions(-)
fs/fat/fat.c +0 -0
index ecfa255..347787e 100644
@@ -306,9 +306,6 @@ get_cluster(fsdata *mydata, __u32 clustnum,
 __u8 *buffer, unsigned long size)
   * into 'buffer'.
   * Update the number of bytes read in *gotsize or return -1 on

fatal errors.
  */

fs/fat/fat.c +0 -3
-__u8 get_contents_vfatname_block[MAX_CLUSTSIZE]
-	__aligned(ARCH_DMA_MINALIGN);
-

 static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos,
  __u8 *buffer, loff_t maxsize, loff_t *gotsize)
 {

fs/fat/fat.c +0 -0
@@ -351,15 +348,25 @@ static int get_contents(fsdata *mydata,
 dir_entry *dentptr, loff_t pos,


  /* align to beginning of next cluster if any */ if (pos) {

fs/fat/fat.c +2 -0
+		__u8 *tmp_buffer;
+

  actsize = min(filesize, (loff_t)bytesperclust);

fs/fat/fat.c +0 -1
-		if (get_cluster(mydata, curclust,

get_contents_vfatname_block,

fs/fat/fat.c +1 -0
+		tmp_buffer = malloc_cache_aligned(actsize);
Quoted 8 lines
Quoted 7 lines
fs/fat/fat.c +6 -0
+		if (!tmp_buffer) {
+			debug("Error: allocating buffer\n");
+			return -ENOMEM;
+		}
+
+		if (get_cluster(mydata, curclust, tmp_buffer,

  (int)actsize) != 0) {

Is the cast of actsize needed ?

Okay, i would remove it.

Quoted 15 lines

fs/fat/fat.c static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos, · line 368

Quoted 11 lines

  printf("Error reading cluster\n");

fs/fat/fat.c +1 -0
+			free(tmp_buffer);

  return -1;
  }
  filesize -= actsize;
  actsize -= pos;

fs/fat/fat.c +0 -1
-		memcpy(buffer, get_contents_vfatname_block + pos,

actsize);

fs/fat/fat.c +1 -0
+		memcpy(buffer, tmp_buffer + pos, actsize);

Hmmm, tmp_buffer is actsize big , but you're memcpy-ing actsize here, so
this would mean you memcpy data past the end of tmp_buffer if pos > 0, no?

This wouldn't happen because the pos and actsize are reset based on beginning of current cluster instead of beginning of a file. So, the memcpy would start at pos based on beginning of current cluster until the end of current cluster, that means the size it copies is still within a cluster size.

Quoted 8 lines
Quoted 7 lines
fs/fat/fat.c +1 -0
+		free(tmp_buffer);

  *gotsize += actsize;
  if (!filesize)
  return 0;

#6 lore raw

PATCH v2 1/2 fs: fat: dynamically allocate memory for temporary buffer

Quoted 76 lines · Tien Fong
Quoted 51 lines · Marek Vasut
Quoted 50 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Drop the statically allocated get_contents_vfatname_block and dynamically allocate a buffer only if required. This saves 64KiB of memory.

---

changes for v2
- Removed the change for debug message.
- Set allocation based on actual required size instead of default max
  cluster size
---
 fs/fat/fat.c | 17 ++++++++++++-----

 1 file changed, 12 insertions(+), 5 deletions(-)
fs/fat/fat.c +0 -0
index ecfa255..347787e 100644
@@ -306,9 +306,6 @@ get_cluster(fsdata *mydata, __u32 clustnum,
 __u8 *buffer, unsigned long size)
   * into 'buffer'.
   * Update the number of bytes read in *gotsize or return -1 on

fatal errors.
  */

fs/fat/fat.c +0 -3
-__u8 get_contents_vfatname_block[MAX_CLUSTSIZE]
-	__aligned(ARCH_DMA_MINALIGN);
-

 static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos,
  __u8 *buffer, loff_t maxsize, loff_t *gotsize)
 {

fs/fat/fat.c +0 -0
@@ -351,15 +348,25 @@ static int get_contents(fsdata *mydata,
 dir_entry *dentptr, loff_t pos,


  /* align to beginning of next cluster if any */ if (pos) {

fs/fat/fat.c +2 -0
+		__u8 *tmp_buffer;
+

  actsize = min(filesize, (loff_t)bytesperclust);

fs/fat/fat.c +0 -1
-		if (get_cluster(mydata, curclust,

get_contents_vfatname_block,

fs/fat/fat.c +1 -0
+		tmp_buffer = malloc_cache_aligned(actsize);
Quoted 8 lines
Quoted 7 lines
fs/fat/fat.c +6 -0
+		if (!tmp_buffer) {
+			debug("Error: allocating buffer\n");
+			return -ENOMEM;
+		}
+
+		if (get_cluster(mydata, curclust, tmp_buffer,

  (int)actsize) != 0) {

Is the cast of actsize needed ?

Okay, i would remove it.

Quoted 15 lines

fs/fat/fat.c static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos, · line 368

Quoted 11 lines

  printf("Error reading cluster\n");

fs/fat/fat.c +1 -0
+			free(tmp_buffer);

  return -1;
  }
  filesize -= actsize;
  actsize -= pos;

fs/fat/fat.c +0 -1
-		memcpy(buffer, get_contents_vfatname_block + pos,

actsize);

fs/fat/fat.c +1 -0
+		memcpy(buffer, tmp_buffer + pos, actsize);

Hmmm, tmp_buffer is actsize big , but you're memcpy-ing actsize here, so
this would mean you memcpy data past the end of tmp_buffer if pos > 0, no?

Quoted 5 lines

This wouldn't happen because the pos and actsize are reset based on beginning of current cluster instead of beginning of a file. So, the memcpy would start at pos based on beginning of current cluster until the end of current cluster, that means the size it copies is still within a cluster size.

So pos is always 0 ?

Signature · 2 lines
Best regards,
Marek Vasut
#7 lore raw

PATCH v2 1/2 fs: fat: dynamically allocate memory for temporary buffer

Quoted 88 lines · Marek Vasut
Quoted 87 lines · Tien Fong
Quoted 56 lines · Marek Vasut
Quoted 54 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Drop the statically allocated get_contents_vfatname_block and dynamically allocate a buffer only if required. This saves 64KiB of memory.

---

changes for v2
- Removed the change for debug message.
- Set allocation based on actual required size instead of default
max
  cluster size
---
 fs/fat/fat.c | 17 ++++++++++++-----

 1 file changed, 12 insertions(+), 5 deletions(-)
fs/fat/fat.c +0 -0
index ecfa255..347787e 100644
@@ -306,9 +306,6 @@ get_cluster(fsdata *mydata, __u32 clustnum,
 __u8 *buffer, unsigned long size)
   * into 'buffer'.
   * Update the number of bytes read in *gotsize or return -1 on

fatal errors.
  */

fs/fat/fat.c +0 -3
-__u8 get_contents_vfatname_block[MAX_CLUSTSIZE]
-	__aligned(ARCH_DMA_MINALIGN);
-

 static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t
pos,
  __u8 *buffer, loff_t maxsize, loff_t *gotsize)
 {

fs/fat/fat.c +0 -0
@@ -351,15 +348,25 @@ static int get_contents(fsdata *mydata,
 dir_entry *dentptr, loff_t pos,


  /* align to beginning of next cluster if any */ if (pos) {

fs/fat/fat.c +2 -0
+		__u8 *tmp_buffer;
+

  actsize = min(filesize,
(loff_t)bytesperclust);

fs/fat/fat.c +0 -1
-		if (get_cluster(mydata, curclust,

get_contents_vfatname_block,

fs/fat/fat.c +1 -0
+		tmp_buffer = malloc_cache_aligned(actsize);
Quoted 8 lines
Quoted 7 lines
fs/fat/fat.c +6 -0
+		if (!tmp_buffer) {
+			debug("Error: allocating buffer\n");
+			return -ENOMEM;
+		}
+
+		if (get_cluster(mydata, curclust, tmp_buffer,

  (int)actsize) != 0) {

Is the cast of actsize needed ?

Okay, i would remove it.

Quoted 20 lines
Quoted 13 lines

  printf("Error reading cluster\n");

fs/fat/fat.c +1 -0
+			free(tmp_buffer);

  return -1;
  }
  filesize -= actsize;
  actsize -= pos;

fs/fat/fat.c +0 -1
-		memcpy(buffer, get_contents_vfatname_block +

pos,
actsize);

fs/fat/fat.c static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos, · line 368

Quoted 1 line
fs/fat/fat.c +1 -0
+		memcpy(buffer, tmp_buffer + pos, actsize);

Hmmm, tmp_buffer is actsize big , but you're memcpy-ing actsize here,
so
this would mean you memcpy data past the end of tmp_buffer if pos

Quoted 1 line

0, no?

Quoted 8 lines
Quoted 7 lines

This wouldn't happen because the pos and actsize are reset based on beginning of current cluster instead of beginning of a file. So, the
memcpy would start at pos based on beginning of current cluster until
the end of current cluster, that means the size it copies is still within a cluster size.

So pos is always 0 ?

Nop, reset here means the pos offset would be adjusted accordingly when the base change from beginning of the file to beginning of the cluster where the pos resides in.

You know the driver getting the content based on cluster by cluster.

Quoted 1 line
#8 lore raw

PATCH v2 1/2 fs: fat: dynamically allocate memory for temporary buffer

Quoted 96 lines · Tien Fong
Quoted 95 lines · Marek Vasut
Quoted 94 lines · Tien Fong
Quoted 56 lines · Marek Vasut
Quoted 54 lines · tien.fong.chee at intel.com

From: Tien Fong Chee <[email protected]>

Drop the statically allocated get_contents_vfatname_block and dynamically allocate a buffer only if required. This saves 64KiB of memory.

---

changes for v2
- Removed the change for debug message.
- Set allocation based on actual required size instead of default
max
  cluster size
---
 fs/fat/fat.c | 17 ++++++++++++-----

 1 file changed, 12 insertions(+), 5 deletions(-)
fs/fat/fat.c +0 -0
index ecfa255..347787e 100644
@@ -306,9 +306,6 @@ get_cluster(fsdata *mydata, __u32 clustnum,
 __u8 *buffer, unsigned long size)
   * into 'buffer'.
   * Update the number of bytes read in *gotsize or return -1 on

fatal errors.
  */

fs/fat/fat.c +0 -3
-__u8 get_contents_vfatname_block[MAX_CLUSTSIZE]
-	__aligned(ARCH_DMA_MINALIGN);
-

 static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t
pos,
  __u8 *buffer, loff_t maxsize, loff_t *gotsize)
 {

fs/fat/fat.c +0 -0
@@ -351,15 +348,25 @@ static int get_contents(fsdata *mydata,
 dir_entry *dentptr, loff_t pos,


  /* align to beginning of next cluster if any */ if (pos) {

fs/fat/fat.c +2 -0
+		__u8 *tmp_buffer;
+

  actsize = min(filesize,
(loff_t)bytesperclust);

fs/fat/fat.c +0 -1
-		if (get_cluster(mydata, curclust,

get_contents_vfatname_block,

fs/fat/fat.c +1 -0
+		tmp_buffer = malloc_cache_aligned(actsize);
Quoted 8 lines
Quoted 7 lines
fs/fat/fat.c +6 -0
+		if (!tmp_buffer) {
+			debug("Error: allocating buffer\n");
+			return -ENOMEM;
+		}
+
+		if (get_cluster(mydata, curclust, tmp_buffer,

  (int)actsize) != 0) {

Is the cast of actsize needed ?

Okay, i would remove it.

Quoted 19 lines
Quoted 13 lines

  printf("Error reading cluster\n");

fs/fat/fat.c +1 -0
+			free(tmp_buffer);

  return -1;
  }
  filesize -= actsize;
  actsize -= pos;

fs/fat/fat.c +0 -1
-		memcpy(buffer, get_contents_vfatname_block +

pos,
actsize);

fs/fat/fat.c static int get_contents(fsdata *mydata, dir_entry *dentptr, loff_t pos, · line 368

Quoted 1 line
fs/fat/fat.c +1 -0
+		memcpy(buffer, tmp_buffer + pos, actsize);

Hmmm, tmp_buffer is actsize big , but you're memcpy-ing actsize here,
so
this would mean you memcpy data past the end of tmp_buffer if pos

Quoted 1 line
Quoted 1 line

0, no?

This wouldn't happen because the pos and actsize are reset based on beginning of current cluster instead of beginning of a file. So, the
memcpy would start at pos based on beginning of current cluster until
the end of current cluster, that means the size it copies is still within a cluster size.

Quoted 6 lines
Quoted 1 line

So pos is always 0 ?

Nop, reset here means the pos offset would be adjusted accordingly when the base change from beginning of the file to beginning of the cluster where the pos resides in.

You know the driver getting the content based on cluster by cluster.

OK, reading the code again, after simplification we get:

tmp_buffer = malloc_cache_aligned(actsize);
...
actsize -= pos;
memcpy(buffer, tmp_buffer + pos, actsize);

so there shouldn't be any read past the end of buffer.

Signature · 2 lines
Best regards,
Marek Vasut