xfsrestore: fix inventory unpacking
v4 cover xfsrestore: fix inventory unpacking
When xfsrestore reads its inventory from tape it fails to convert the media record on bigendian systems, if the online inventory is unavailable this results in invalid data being writen to the online inventory and failure to restore non-directory files.
The series fixes the converstion and related issues.
---
v2
- Seperate out cleanup and content.c changes, fix whitespace.
- Show a full reproducer in the first patch.
V3
- Fix whitespace and Signed-off-by.
- Make for loop formatting consistent.
- Rename patches to make their intent clearer.
- Add xfsdump: fix on-media inventory stream packing.
- Add descriptions that reproduce the stream problems.
V4
- Rerun testcase in "xfsdump: fix on-media inventory stream packing" description to avoid confusion.
Donald Douwsma (4):
xfsrestore: fix on-media inventory media unpacking
xfsrestore: fix on-media inventory stream unpacking
xfsdump: fix on-media inventory stream packing
xfsrestore: untangle inventory unpacking logic
inventory/inv_stobj.c | 42 +++++++++++++++--------------------------- restore/content.c | 13 +++++-------- 2 files changed, 20 insertions(+), 35 deletions(-)
Signature · 1 line
2.31.1
PATCH v4 1/4 xfsrestore: fix on-media inventory media unpacking
When xfsrestore reads the inventory from tape media it fails to convert media file records from bigendian. If the xfsdump inventory is not available xfsrestore will write this invalid record to the on-line inventory.
[root@rhel8 ~]# xfsdump -L Test1 -M "" -f /dev/st0 /boot > /dev/null
[root@rhel8 ~]# xfsdump -L Test2 -M "" -f /dev/st0 /boot > /dev/null
[root@rhel8 ~]# rm -rf /var/lib/xfsdump/inventory/
[root@rhel8 ~]# mt -f /dev/nst0 asf 2
[root@rhel8 ~]# xfsrestore -f /dev/nst0 -L Test2 /tmp/test2
xfsrestore: using scsi tape (drive_scsitape) strategy
xfsrestore: version 3.1.8 (dump format 3.0) - type ^C for status and control
xfsrestore: searching media for dump
xfsrestore: preparing drive
xfsrestore: examining media file 3
xfsrestore: found dump matching specified label:
xfsrestore: hostname: rhel8
xfsrestore: mount point: /boot
xfsrestore: volume: /dev/sda1
xfsrestore: session time: Tue Sep 27 16:05:28 2022
xfsrestore: level: 0
xfsrestore: session label: "Test2"
xfsrestore: media label: ""
xfsrestore: file system id: 26dd5aa0-b901-4cf5-9b68-0c5753cb3ab8
xfsrestore: session id: 62402423-7ae0-49ed-8ecb-9e5bc7642b91
xfsrestore: media id: 47ba45ca-3417-4006-ab10-3dc6419b83e2
xfsrestore: incorporating on-media session inventory into online inventory
xfsrestore: /var/lib/xfsdump/inventory created
xfsrestore: using on-media session inventory
xfsrestore: searching media for directory dump
xfsrestore: rewinding
xfsrestore: examining media file 0
xfsrestore: inventory session uuid (62402423-7ae0-49ed-8ecb-9e5bc7642b91) does not match the media header's session uuid (1771d9e8-a1ba-4e87-a61e-f6be97e41b45)
xfsrestore: examining media file 1
xfsrestore: inventory session uuid (62402423-7ae0-49ed-8ecb-9e5bc7642b91) does not match the media header's session uuid (1771d9e8-a1ba-4e87-a61e-f6be97e41b45)
xfsrestore: examining media file 2
xfsrestore: reading directories
xfsrestore: 9 directories and 320 entries processed
xfsrestore: directory post-processing
xfsrestore: restore complete: 0 seconds elapsed
xfsrestore: Restore Summary:
xfsrestore: stream 0 /dev/nst0 OK (success)
xfsrestore: Restore Status: SUCCESS
[root@rhel8 ~]# xfsdump -I
file system 0:
fs id: 26dd5aa0-b901-4cf5-9b68-0c5753cb3ab8
session 0:
mount point: rhel8:/boot
device: rhel8:/dev/sda1
time: Tue Sep 27 16:05:28 2022
session label: "Test2"
session id: 62402423-7ae0-49ed-8ecb-9e5bc7642b91
level: 0
resumed: NO
subtree: NO
streams: 1
stream 0:
pathname: /dev/st0
start: ino 133 offset 0
end: ino 1572997 offset 0
interrupted: YES
media files: 1
media file 0:
mfile index: 33554432
mfile type: data
mfile size: 211187836911616
mfile start: ino 9583660007044415488 offset 0
mfile end: ino 9583686395323482112 offset 0
media label: ""
media id: 47ba45ca-3417-4006-ab10-3dc6419b83e2
xfsdump: Dump Status: SUCCESS
[root@rhel8 ~]#
[root@rhel8 ~]# ls /tmp/test2
efi grub2 loader
The invalid start and end inode information cause xfsrestore to consider that non-directory files do not reside in the current media and will fail to restore them.
The behaviour of an initial restore may succeed if the position of the tape is such that the data file is encountered before the inventory file, or if there is only one dump session on tape, xfsrestore is somewhat inconsistent in this regard. Subsequent restores will use the invalid on-line inventory and fail to restore files.
Fix this by correctly unpacking the inventory data.
--- inventory/inv_stobj.c | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-)
inventory/inv_stobj.c +7 -20
@@ -1008,7 +1008,7 @@ stobj_unpack_sessinfo(size_t bufsz, invt_sessinfo_t *s) { - uint i; + uint i, j;char *tmpbuf;char *p = (char *)bufp;
@@ -1087,26 +1087,13 @@ stobj_unpack_sessinfo(/* all the media files */ s->mfiles = (invt_mediafile_t *)p; - -#ifdef INVT_DELETION - { -int tmpfd =open ("moids" ,O_RDWR |O_CREAT ,S_IRUSR |S_IWUSR ); -uint j; -invt_mediafile_t *mmf = s->mfiles; -for (i=0 ; i< s->ses->s_cur_nstreams; i++) { -for (j=0 ; j< s->strms[i].st_nmediafiles; - j++, mmf++) -xlate_invt_mediafile ((invt_mediafile_t *)mmf, (invt_mediafile_t *)tmpbuf,1 ); -bcopy (tmpbuf, mmf,sizeof (invt_mediafile_t)); -put_invtrecord (tmpfd, &mmf->mf_moid, -sizeof (uuid_t),0 ,SEEK_END ,0 ); - } -close (tmpfd); - } -#endif for (i =0 ; i < s->ses->s_cur_nstreams; i++) { - p += (size_t ) (s->strms[i].st_nmediafiles) - *sizeof (invt_mediafile_t); +for (j =0 ; j < s->strms[i].st_nmediafiles; j++) { +xlate_invt_mediafile ((invt_mediafile_t *)p, + (invt_mediafile_t *)tmpbuf,1 ); +bcopy (tmpbuf, p,sizeof (invt_mediafile_t)); + p +=sizeof (invt_mediafile_t); + } }/* sanity check the size of the buffer given to us vs. the size it
Signature · 1 line
2.31.1
PATCH v4 2/4 xfsrestore: fix on-media inventory stream unpacking
In reply to Donald Douwsmaxfsdump can create multiple streams, when restoring the online inventory with multiple streams we fail to process these and assert when the inventory buffer is not fully decoded.
[root@rhel8 ~]# xfsdump -L "Test1" -f /dev/nst0 -M tape1 -f /dev/nst1 -M tape2 /boot
xfsdump: using scsi tape (drive_scsitape) strategy xfsdump: using scsi tape (drive_scsitape) strategy xfsdump: version 3.1.8 (dump format 3.0) - type ^C for status and control xfsdump: level 0 dump of rhel8:/boot xfsdump: dump date: Thu Oct 6 13:50:45 2022 xfsdump: session id: aa25fa48-4493-45c7-9027-61e53e486445 xfsdump: session label: "Test1" xfsdump: ino map phase 1: constructing initial dump list xfsdump: ino map phase 2: skipping (no pruning necessary) xfsdump: ino map phase 3: identifying stream starting points xfsdump: stream 0: ino 133 offset 0 to ino 28839 offset 0 xfsdump: stream 1: ino 28839 offset 0 to end xfsdump: ino map construction complete xfsdump: estimated dump size: 328720704 bytes xfsdump: estimated dump size per stream: 164375728 bytes xfsdump: /var/lib/xfsdump/inventory created xfsdump: drive 0: preparing drive xfsdump: drive 1: preparing drive xfsdump: drive 1: creating dump session media file 0 (media 0, file 0) xfsdump: drive 1: dumping ino map xfsdump: drive 1: dumping non-directory files xfsdump: drive 0: creating dump session media file 0 (media 0, file 0) xfsdump: drive 0: dumping ino map xfsdump: drive 0: dumping directories xfsdump: drive 0: dumping non-directory files xfsdump: drive 1: ending media file xfsdump: drive 1: media file size 166723584 bytes xfsdump: drive 1: waiting for synchronized session inventory dump xfsdump: drive 0: ending media file xfsdump: drive 0: media file size 165675008 bytes xfsdump: drive 0: waiting for synchronized session inventory dump xfsdump: drive 0: dumping session inventory xfsdump: drive 0: beginning inventory media file xfsdump: drive 0: media file 1 (media 0, file 1) xfsdump: drive 0: ending inventory media file xfsdump: drive 0: inventory media file size 2097152 bytes xfsdump: drive 0: writing stream terminator xfsdump: drive 0: beginning media stream terminator xfsdump: drive 0: media file 2 (media 0, file 2) xfsdump: drive 0: ending media stream terminator xfsdump: drive 0: media stream terminator size 1048576 bytes xfsdump: drive 1: dumping session inventory xfsdump: drive 1: beginning inventory media file xfsdump: drive 1: media file 1 (media 0, file 1) xfsdump: drive 1: ending inventory media file xfsdump: drive 1: inventory media file size 2097152 bytes xfsdump: drive 1: writing stream terminator xfsdump: drive 1: beginning media stream terminator xfsdump: drive 1: media file 2 (media 0, file 2) xfsdump: drive 1: ending media stream terminator xfsdump: drive 1: media stream terminator size 1048576 bytes xfsdump: dump size (non-dir files) : 328189016 bytes xfsdump: dump complete: 4 seconds elapsed xfsdump: Dump Summary: xfsdump: stream 0 /dev/nst0 OK (success) xfsdump: stream 1 /dev/nst1 OK (success) xfsdump: Dump Status: SUCCESS [root@rhel8 ~]# xfsdump -I file system 0: fs id: 26dd5aa0-b901-4cf5-9b68-0c5753cb3ab8 session 0: mount point: rhel8:/boot device: rhel8:/dev/sda1 time: Thu Oct 6 13:50:45 2022 session label: "Test1" session id: aa25fa48-4493-45c7-9027-61e53e486445 level: 0 resumed: NO subtree: NO streams: 2 stream 0: pathname: /dev/nst0 start: ino 133 offset 0 end: ino 28839 offset 0 interrupted: NO media files: 2 media file 0: mfile index: 0 mfile type: data mfile size: 165675008 mfile start: ino 133 offset 0 mfile end: ino 28839 offset 0 media label: "tape1" media id: adb31f2a-f026-4597-a20a-326f28ecbaf1 media file 1: mfile index: 1 mfile type: inventory mfile size: 2097152 media label: "tape1" media id: adb31f2a-f026-4597-a20a-326f28ecbaf1 stream 1: pathname: /dev/nst1 start: ino 28839 offset 0 end: ino 1572997 offset 0 interrupted: NO media files: 2 media file 0: mfile index: 0 mfile type: data mfile size: 166723584 mfile start: ino 28839 offset 0 mfile end: ino 1572997 offset 0 media label: "tape2" media id: 22224f02-b6c7-47d5-ad61-a61ba071c8a8 media file 1: mfile index: 1 mfile type: inventory mfile size: 2097152 media label: "tape2" media id: 22224f02-b6c7-47d5-ad61-a61ba071c8a8
xfsdump: Dump Status: SUCCESS
[root@rhel8 ~]# mv /var/lib/xfsdump/inventory /var/lib/xfsdump/inventory_two_sessions [root@rhel8 ~]# xfsdump -I
xfsdump: Dump Status: SUCCESS
[root@rhel8 ~]# xfsrestore -L Test1 -f /dev/nst0 /tmp/test1/
xfsrestore: using scsi tape (drive_scsitape) strategy
xfsrestore: version 3.1.8 (dump format 3.0) - type ^C for status and control
xfsrestore: searching media for dump
xfsrestore: preparing drive
xfsrestore: examining media file 2
xfsrestore: found dump matching specified label:
xfsrestore: hostname: rhel8
xfsrestore: mount point: /boot
xfsrestore: volume: /dev/sda1
xfsrestore: session time: Thu Oct 6 13:50:45 2022
xfsrestore: level: 0
xfsrestore: session label: "Test1"
xfsrestore: media label: "tape1"
xfsrestore: file system id: 26dd5aa0-b901-4cf5-9b68-0c5753cb3ab8
xfsrestore: session id: aa25fa48-4493-45c7-9027-61e53e486445
xfsrestore: media id: adb31f2a-f026-4597-a20a-326f28ecbaf1
xfsrestore: searching media for directory dump
xfsrestore: rewinding
xfsrestore: examining media file 0
xfsrestore: reading directories
xfsrestore: 9 directories and 320 entries processed
xfsrestore: directory post-processing
xfsrestore: restoring non-directory files
xfsrestore: examining media file 1
xfsrestore: inv_stobj.c:1119: stobj_unpack_sessinfo: Assertion `(size_t) ( p - (char *) bufp ) == bufsz' failed. Aborted (core dumped)
Make sure we unpack multiple streams when restoring the online inventory from media.
--- inventory/inv_stobj.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-)
inventory/inv_stobj.c +7 -6
@@ -1065,25 +1065,26 @@ stobj_unpack_sessinfo(return BOOL_FALSE ; } +/* get the seshdr and then, the remainder of the session */ xlate_invt_seshdr ((invt_seshdr_t *)p, (invt_seshdr_t *)tmpbuf,1 );bcopy (tmpbuf, p,sizeof (invt_seshdr_t)); - -/* get the seshdr and then, the remainder of the session */ s->seshdr = (invt_seshdr_t *)p; s->seshdr->sh_sess_off =-1 ; p +=sizeof (invt_seshdr_t); -xlate_invt_session ((invt_session_t *)p, (invt_session_t *)tmpbuf,1 );bcopy (tmpbuf, p,sizeof (invt_session_t)); s->ses = (invt_session_t *)p; p +=sizeof (invt_session_t);/* the array of all the streams belonging to this session */ -xlate_invt_stream ((invt_stream_t *)p, (invt_stream_t *)tmpbuf,1 ); -bcopy (tmpbuf, p,sizeof (invt_stream_t)); s->strms = (invt_stream_t *)p; - p += s->ses->s_cur_nstreams *sizeof (invt_stream_t); +for (i =0 ; i < s->ses->s_cur_nstreams; i++) { +xlate_invt_stream ((invt_stream_t *)p, + (invt_stream_t *)tmpbuf,1 ); +bcopy (tmpbuf, p,sizeof (invt_stream_t)); + p +=sizeof (invt_stream_t); + }/* all the media files */ s->mfiles = (invt_mediafile_t *)p;
Signature · 1 line
2.31.1
PATCH v4 3/4 xfsdump: fix on-media inventory stream packing
In reply to Donald DouwsmaWith the on-media inventory now being restored for multiple streams we can see that the restored streams both claim to be for /dev/nst0.
[root@rhel8 xfsdump-dev]# xfsdump -L "Test" -f /dev/nst0 -M tape1 -f /dev/nst1 -M tape2 /boot
...
[root@rhel8 ~]# rm -rf /var/lib/xfsdump/inventory
[root@rhel8 xfsdump-dev]# restore/xfsrestore -L Test -f /dev/nst0 -f /dev/nst1 /tmp/test restore/xfsrestore: using scsi tape (drive_scsitape) strategy restore/xfsrestore: using scsi tape (drive_scsitape) strategy restore/xfsrestore: version 3.1.10 (dump format 3.0) - type ^C for status and control
...
restore/xfsrestore: Restore Summary: restore/xfsrestore: stream 0 /dev/nst0 OK (success) restore/xfsrestore: stream 1 /dev/nst1 ALREADY_DONE (another stream completed the operation) restore/xfsrestore: Restore Status: SUCCESS [root@rhel8 xfsdump-dev]# xfsdump -I file system 0: fs id: 26dd5aa0-b901-4cf5-9b68-0c5753cb3ab8 session 0: mount point: rhel8:/boot device: rhel8:/dev/sda1 time: Fri Oct 14 18:31:40 2022 session label: "Test" session id: 96538a3d-2af8-4a79-8865-afec6e3e55f4 level: 0 resumed: NO subtree: NO streams: 2 stream 0: pathname: /dev/nst0 start: ino 133 offset 0 end: ino 28839 offset 0 interrupted: YES media files: 1 media file 0: mfile index: 0 mfile type: data mfile size: 165675008 mfile start: ino 133 offset 0 mfile end: ino 28839 offset 0 media label: "tape1" media id: 8a9d0ced-61f6-4332-a0c1-f1e38641c4e6 stream 1: pathname: /dev/nst0 start: ino 133 offset 0 end: ino 28839 offset 0 interrupted: YES media files: 1 media file 0: mfile index: 0 mfile type: data mfile size: 166723584 mfile start: ino 28839 offset 0 mfile end: ino 1572997 offset 0 media label: "tape2" media id: 7d569377-6bfb-4c02-b299-4dbe753bb048
xfsdump: Dump Status: SUCCESS
[root@rhel8 xfsdump-dev]#
Fix this by indexing the stream being packed for the on-media inventory.
--- inventory/inv_stobj.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
inventory/inv_stobj.c +1 -1
@@ -798,7 +798,7 @@ stobj_pack_sessinfo(int fd, invt_session_t *ses, invt_seshdr_t *hdr, sesbuf +=sizeof (invt_session_t);for (i =0 ; i < ses->s_cur_nstreams; i++) { -xlate_invt_stream (strms, (invt_stream_t *)sesbuf,1 ); +xlate_invt_stream (&strms[i], (invt_stream_t *)sesbuf,1 ); sesbuf +=sizeof (invt_stream_t); }
Signature · 1 line
2.31.1
PATCH v4 4/4 xfsrestore: untangle inventory unpacking logic
In reply to Donald Douwsmastobj_unpack_sessinfo returns bool_t, fix logic in pi_addfile so errors can be properly reported.
--- restore/content.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-)
restore/content.c +5 -8
@@ -5463,17 +5463,14 @@ pi_addfile(Media_t *Mediap,* desc. */ sessp =0 ; -if (!buflen) { - ok =BOOL_FALSE ; - }else { -/* extract the session information from the buffer */ -if (stobj_unpack_sessinfo (bufp, buflen, &sessinfo)<0 ) { - ok =BOOL_FALSE ; - }else { + ok =BOOL_FALSE ; +/* extract the session information from the buffer */ +if (buflen && +stobj_unpack_sessinfo (bufp, buflen, &sessinfo)) {stobj_convert_sessinfo (&sessp, &sessinfo); ok =BOOL_TRUE ; - } } +if (!ok || !sessp) { mlog(MLOG_DEBUG | MLOG_WARNING | MLOG_MEDIA, _( "on-media session "
Signature · 1 line
2.31.1