booting from BTRFS works only with one device in the pool

Hello,

I am running CentOS from a btrfs root.
This worked fine until I added a device to that pool:
btrfs device add /dev/sda3 /
reboot

This now causes the errors:
BTRFS: failed to read chunk tree on sdb3
BTRFS: open_ctree failed

Here I am stuck in a recovery prompt.

btrfs fi show displays the file system correctly with 2.1GiB used for sdb3 and 0.00GiB used on sda3

btrfs-tools version reports
btrfs-progs v4.3.1

Now, I read that in case of this issue, should add the second device of the pool to the commandline argument of the kernel/the boot options/grub.cfg

But I am not sure how to do this.
I can mount /boot/ and the /boot/grub2/grub.cfg contains:
insmod ext2 (but not btrfs!)
set root='hd0,msdos1'
search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1' 4a470ac6-f013-4e7b-a4f3-3a58cc4debc3

after removing sda3 from the pool again, the system boots normally.

blkid gives:
/dev/sdb3: LABEL="rockstor_rockstor" UUID="f9de7c11-012e-4e5d-8b53-0e6d6c2916a3" UUID_SUB="24bdf07b-dbd3-44dc-9195-4b0bfedf974f" TYPE="btrfs" PARTLABEL="Linux filesystem" PARTUUID="c438bd3c-df9a-4e49-8607-47cd9b45e212" (note /dev/sda3 is not shown here)

btrfs fi show
Label: 'rockstor_rockstor' uuid: f9de7c11-012e-4e5d-8b53-0e6d6c2916a3
Total devices 1 FS bytes used 1.38GiB
devid 1 size 6.87GiB used 2.10GiB path /dev/sdb3

It's a pitty that the only NAS Distribution built around btrfs does not support the full feature-set of btrfs on its root partition. Could you please help me fixing this?

Below you find the complete grub.cfg.

Regards,
Hendrik

cat /boot/grub2/grub.cfg
#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub2-mkconfig using templates

# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
set pager=1

if [ -s $prefix/grubenv ]; then
load_env
fi
if [ "${next_entry}" ] ; then
set default="${next_entry}"
set next_entry=
save_env next_entry
set boot_once=true
else
set default="${saved_entry}"
fi

if [ x"${feature_menuentry_id}" = xy ]; then
menuentry_id_option="--id"
else
menuentry_id_option=""
fi

export menuentry_id_option

if [ "${prev_saved_entry}" ]; then
set saved_entry="${prev_saved_entry}"
save_env saved_entry
set prev_saved_entry=
save_env prev_saved_entry
set boot_once=true
fi

function savedefault {
if [ -z "${boot_once}" ]; then

    saved_entry="${chosen}"
    save_env saved_entry

fi
}

function load_video {
if [ x$feature_all_video_module = xy ]; then
insmod all_video
else

    insmod efi_gop
    insmod efi_uga
    insmod ieee1275_fb
    insmod vbe
    insmod vga
    insmod video_bochs
    insmod video_cirrus

fi
}

terminal_output console
if [ x$feature_timeout_style = xy ] ; then
set timeout_style=menu
set timeout=5
# Fallback normal timeout code in case the timeout_style feature is

# unavailable.
else
  set timeout=5
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/00_tuned ###
set tuned_params=""
### END /etc/grub.d/00_tuned ###

### BEGIN /etc/grub.d/01_users ###
if [ -f ${prefix}/user.cfg ]; then
source ${prefix}/user.cfg
if [ -n "${GRUB2_PASSWORD}" ]; then

    set superusers="root"
    export superusers
    password_pbkdf2 root ${GRUB2_PASSWORD}

fi
fi
### END /etc/grub.d/01_users ###

### BEGIN /etc/grub.d/10_linux ###

menuentry 'Rockstor (4.3.3-1.el7.elrepo.x86_64) 3 (Core)' --class rhel fedora --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-4.3.3-1.el7.elrepo.x86_64-advanced-f9de7c11-012e-4e5d-8b53-0e6d6c2916a3' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod ext2
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root 
--hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1' 4a470ac6-f013-4e7b-a4f3-3a58cc4debc3
        else
          search --no-floppy --fs-uuid --set=root 
4a470ac6-f013-4e7b-a4f3-3a58cc4debc3
        fi
        linux16 /vmlinuz-4.3.3-1.el7.elrepo.x86_64 
root=UUID=f9de7c11-012e-4e5d-8b53-0e6d6c2916a3 ro rootflags=subvol=root crashkernel=auto rhgb quiet LANG=en_US.UTF-8
        initrd16 /initramfs-4.3.3-1.el7.elrepo.x86_64.img
}
menuentry 'Rockstor (0-rescue-f5f625480f394bdc90d6d3c06be7fb88) 3 (Core)' --class rhel fedora --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-f5f625480f394bdc90d6d3c06be7fb88-advanced-f9de7c11-012e-4e5d-8b53-0e6d6c2916a3' {
        load_video
        insmod gzio
        insmod part_msdos
        insmod ext2
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root 
--hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1' 4a470ac6-f013-4e7b-a4f3-3a58cc4debc3
        else
          search --no-floppy --fs-uuid --set=root 
4a470ac6-f013-4e7b-a4f3-3a58cc4debc3
        fi
        linux16 /vmlinuz-0-rescue-f5f625480f394bdc90d6d3c06be7fb88 
root=UUID=f9de7c11-012e-4e5d-8b53-0e6d6c2916a3 ro rootflags=subvol=root crashkernel=auto rhgb quiet
        initrd16 /initramfs-0-rescue-f5f625480f394bdc90d6d3c06be7fb88.img
}

### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###

### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/20_ppc_terminfo ###
### END /etc/grub.d/20_ppc_terminfo ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries. Simply type the

# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f ${config_directory}/custom.cfg ]; then
source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f $prefix/custom.cfg ]; then
source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###

---
Diese E-Mail wurde von Avast Antivirus-Software auf Viren geprüft.
https://www.avast.com/antivirus

By far the simplest and most reliable method of doing this is to
use an initramfs with the command "btrfs dev scan" in it somewhere before mounting. Most of the major distributions already have an initramfs set up (as does yours, I see), and will install the correct commands in the initramfs if you install the btrfs-progs package (btrfs-tools in Debian derivatives).

The other approach is to add "device=/dev/sda3,device=/dev/sdb3" to
the "rootflags=" option on the grub command line, but that's going to break if your kernel or hardware decides to reorder your devices.

Do the sensible thing and just use the initramfs solution.

Hugo.

Hugo Mills             | I must be musical: I've got *loads* of CDs
hugo@... carfax.org.uk |
http://carfax.org.uk/  |
PGP: E2AB1DE4          |                                     Fran, Black Books

Hugo Mills posted on Mon, 01 Feb 2016 22:11:20 +0000 as excerpted:

As a btrfs user with a two-device btrfs root filesystem who has had to deal with this same problem myself...

Unless the problem has been fixed in very recent kernels, device= doesn't work in rootflags= for btrfs on the kernel commandline in any case. I don't know why, but I suspect it might be the fact that with rootflags=device=, there's at least two =, and the kernel may split it at the wrong = and instead of seeing a rootflags= option it knows what to do with, it sees a rootflags=device= option that it ignores as making no sense, making it a kernel commandline parsing bug. Alternatively, another poster hinted that the kernel may get it correct, but btrfs for some reason can't use device= in the form it gets passed in from rootflags=, while it can use it when passed in from userspace, which would make it a btrfs bug.

Either way, (1) the problem isn't new and has been there for quite some time, since early in the kernel 3.x era at least, but (2) other options such as degraded _can_ be passed successfully by rootflags=, so btrfs can use rootflags= in general, it just has problems with device=, for whatever reason.

Which means...

Exactly.

For over a decade I used reiserfs on / and was able to avoid an initr* entirely. And single-device btrfs / works without an initr*. But multi- device... not so much. While I definitely wasn't thrilled at the prospect of having to complicate my boot process with an initr* that _shouldn't_ be necessary, and had to spend some time learning about initr* (I had initially dumped initr* early in my Linux experience and hadn't had to learn about them until I tried multi-device btrfs /) so as to be able to satisfy myself that I could work with it in both normal operation and disaster recovery scenarios...

At this point there's really only two choices if you want multi-device btrfs /, either use an initr* and have your multi-device btrfs /, or give up on multi-device btrfs / and be satisfied with a more traditional single-device /, btrfs or otherwise.

Well, unless you're a kernel-level dev sufficiently motivated to look into the problem, devise a patch to solve the problem, and get that patch thru the approval process and into kernel mainline. In that case there's three options, and I dearly wish someone with the necessary kernel level coder qualification would take that third option, making initr*less multi- device btrfs / a viable option for the rest of us who in the mean time are stuck with only the first two options!

Duncan - List replies preferred.   No HTML msgs.
"Every nonfree program has a lord, a master --
and if you use the program, he is your master."  Richard Stallman

The problem with that approach is that you're going to have to put
policy into the kernel (which devices do you bother to scan? -- we've hit this already with long scan times caused by timeouts on /dev/fd0 and /dev/cdrom0). That on its own is, I think, enough to doom any attempt at putting device scan into the kernel.

An initramfs has been a requirement for so many configurations for
so long, and has largely been a solved problem for the major distributions, I'm really bemused about people's adamant resistance to having one. It's *less* trouble than a kernel-based solution, because at least it's easier to debug. With a decent initramfs builder, you can get a recovery shell to poke around and fix things (or at least determine what's broken).

Hugo, wearing the Hat of Limited Sympathy.

Hugo Mills             | What's a Nazgûl like you doing in a place like this?
hugo@... carfax.org.uk |
http://carfax.org.uk/  |
PGP: E2AB1DE4          |                                                Illiad

Hello Hugo,

I would like to go the sensible way :-)
But can you hint me how and where to add the btrfs device scan option to the initramfs?

Regards,
Hendrik

---
Diese E-Mail wurde von Avast Antivirus-Software auf Viren geprüft.
https://www.avast.com/antivirus

If btrfs-progs 4.3.1 is installed already, dracut -f will rebuild the initramfs and should just drag in current tools which will include 'btrfs device scan'.

Chris Murphy

Hello,

Yes, 4.3.1 is installed and even in the recovery-mode (i.e. if boot fails) btrfs reports version 4.3.1.
Also, in this recovery mode, btrfs dev scan is working. Furthermore, I can mount the drive in this recovery mode (which I *suspect* is the initramfs).

My understanding is/was, that it is not sufficient that btrfs dev scan is available, but also its execution must be triggered. Is that right?

I have run dracut -f. It does not solve the issue.

Yes, it is Rockstor (CentOS 7.2 based) installed in the way you mention.

Regards,
Hendrik

---
Diese E-Mail wurde von Avast Antivirus-Software auf Viren geprüft.
https://www.avast.com/antivirus

Like I said above, just install the distribution's own
btrfs-progs/btrfs-tools package, and it should do the right thing. You may have to tell the distribution to rebuild the initramfs -- I find I don't have to on Debian, and Chris just gave the recipe for distributions using dracut.

Hugo.

Hugo Mills             | I believe that it's closely correlated with the
hugo@... carfax.org.uk | aeroswine coefficient
http://carfax.org.uk/  |
PGP: E2AB1DE4          |                                       Adrian Bridgett

That's a bit weird. This is BIOS or UEFI system? On UEFI, the prebaked grubx64.efi includes btrfs, so insmod isn't strictly needed. But on BIOS it would be. I wonder if possibly this is malformed due to some confusion on GRUBs part when it finds a multiple device btrfs volume? But anyway, it might not be related since it sounds like the bootloader found the kernel and the initramfs, and what's happening is it's breaking in the initramfs.

It might be as simple as manually mounting:

btrfs dev scan
btrfs fi show ## hopefully both devices are now associated with the volume mount /dev/sdXY /sysroot
exit

If it mounts, you can exit to continue the startup process. And then:

dracut -f

That'll rebuild the initramfs.

That's unexpected. Both devices should now refer to each other, so either device missing should fail, it's effectively raid0 except on a chunk level.

Chris Murphy

Hello Chris,

it is a Virtual-Box-VM. It is a BIOS system

## hopefully both devices are now associated with the volume

The mount works.
After entering "exit" I get the feedback "logout" and the system hangs.

And dracut then somehow understands that btrfs dev scan is needed?

That's way beyond my understanding. I am not sure how this entry is generated. But it is somehow a default behavior, as it seems (I have not done this)

Greetings,
Hendrik

---
Diese E-Mail wurde von Avast Antivirus-Software auf Viren geprüft.
https://www.avast.com/antivirus

This is CentOS 7.2 installed to a single VDI file, and then you use 'btrfs dev add' to add another VDI file? I'd like to know how to reproduce the conditions so I can figure out what's wrong because it ought to work, seeing as it worked for me with Fedora 19 and CentOS 7.x is in the vicinity of Fedora 19/20.

What do you get for
rpm -q grub2

Chris Murphy

Sorry, I missed this:

grub2-2.02-0.34.el7.centos.x86_64

Greetings,
Hendrik

---
Diese E-Mail wurde von Avast Antivirus-Software auf Viren geprüft.
https://www.avast.com/antivirus

Weird. I was not able to reproduce the problem with CentOS 7.0 doing the following in virt-manager with two qcow2s backing /dev/vda and /dev/vdb.

1. Install only to /dev/vda using custom partitioning, partition scheme is btrfs.
2. Reboot.
3. Install btrfs-progs-4.4-1.fc23.x86_64.rpm
4. Install kernel-ml-4.4.1-1.el7.elrepo.x86_64.rpm
## I did those installs separately to make sure the newer progs goes in the initramfs that's created when the newer kernel is installed; it's just a time savings is all.
5. Reboot
6. btrfs dev add /dev/vdb /
7. Reboot
8. yum upgrade
## at this point both devices have chunks
9. Reboot
10. btrfs balance start /
## now only vdb has chunks (due to how single balance allocator works, and vdb is bigger than vda2)
11. Reboot.
## still works

Of course with a VM it's better to just grow the backing device, and then something like 'btrfs fi resize max /' rather than have two devices. But on a real system, you'd have two devices (with the caveat that this is more fragile a setup than a single device because of course if either device dies, the whole Btrfs volume implodes).

I can try it without any updates and see if it works....

Chris Murphy

1. Install CentOS 7.0 to vda
2. reboot
3. btrfs dev add /dev/vdb /
4. reboot
## works
5. btrfs balance start /
6. reboot
## works

Same thing when starting with CentOS 7.2 media.

This is a NAS product using CentOS 7.2? My only guess is they've changed something broke this.

Chris Murphy

Hello Chris,

thanks, I appreciate your help

-----------------
1. Install CentOS 7.0 to vda
2. reboot
3. btrfs dev add /dev/vdb /
4. reboot
## works
5. btrfs balance start /
6. reboot
## works

Same thing when starting with CentOS 7.2 media.

This is a NAS product using CentOS 7.2? My only guess is they've
changed something broke this.
-----------------
I confirm that it runs also for me on CentOS 7 Minimal. Thus we can rule out CentOS and myself as the Source of the problem. Rockstor is Based on Centos 7.2.

So, I compared the grub.cfg of Centos 7.2 -which works for me to Rockstor.

I see very few differences apart from UUIDs:
1) menuentry 'Rockstor' --class rhel fedora
menuentry 'Centos ' --class centos
2) insmod ext2
insmod zfs

I have attached the two files.

Can you point me at other settings/files I could compare, please?

Regards,
Hendrik

---
Diese E-Mail wurde von Avast Antivirus-Software auf Viren geprüft.
https://www.avast.com/antivirus

#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub2-mkconfig using templates

# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
set pager=1

if [ -s $prefix/grubenv ]; then
load_env
fi
if [ "${next_entry}" ] ; then
set default="${next_entry}"
set next_entry=
save_env next_entry
set boot_once=true
else
set default="${saved_entry}"
fi

if [ x"${feature_menuentry_id}" = xy ]; then
menuentry_id_option="--id"
else
menuentry_id_option=""
fi

export menuentry_id_option

if [ "${prev_saved_entry}" ]; then
set saved_entry="${prev_saved_entry}"
save_env saved_entry
set prev_saved_entry=
save_env prev_saved_entry
set boot_once=true
fi

function savedefault {
if [ -z "${boot_once}" ]; then

    saved_entry="${chosen}"
    save_env saved_entry

fi
}

function load_video {
if [ x$feature_all_video_module = xy ]; then
insmod all_video
else

    insmod efi_gop
    insmod efi_uga
    insmod ieee1275_fb
    insmod vbe
    insmod vga
    insmod video_bochs
    insmod video_cirrus

fi
}

terminal_output console
if [ x$feature_timeout_style = xy ] ; then
set timeout_style=menu
set timeout=5
# Fallback normal timeout code in case the timeout_style feature is

# unavailable.
else
  set timeout=5
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/00_tuned ###
set tuned_params=""
### END /etc/grub.d/00_tuned ###

### BEGIN /etc/grub.d/01_users ###
if [ -f ${prefix}/user.cfg ]; then
source ${prefix}/user.cfg
if [ -n "${GRUB2_PASSWORD}" ]; then

    set superusers="root"
    export superusers
    password_pbkdf2 root ${GRUB2_PASSWORD}

fi
fi
### END /etc/grub.d/01_users ###

### BEGIN /etc/grub.d/10_linux ###

menuentry 'CentOS Linux (3.10.0-327.4.5.el7.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-327.4.5.el7.x86_64-advanced-e37695d8-f6df-490a-9a0a-b13cdfbea99d' {
	load_video
	set gfxpayload=keep
	insmod gzio
	insmod part_msdos
	insmod xfs
	set root='hd0,msdos1'
	if [ x$feature_platform_search_hint = xy ]; then
	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  e7d4883f-c825-49bc-907a-8ec9bd22e774
	else
	  search --no-floppy --fs-uuid --set=root e7d4883f-c825-49bc-907a-8ec9bd22e774
	fi
	linux16 /vmlinuz-3.10.0-327.4.5.el7.x86_64 root=UUID=e37695d8-f6df-490a-9a0a-b13cdfbea99d ro rootflags=subvol=root crashkernel=auto rhgb quiet LANG=en_US.UTF-8
	initrd16 /initramfs-3.10.0-327.4.5.el7.x86_64.img
}
menuentry 'CentOS Linux (0-rescue-e8569f127454499cb00018a87844b14d) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-e8569f127454499cb00018a87844b14d-advanced-e37695d8-f6df-490a-9a0a-b13cdfbea99d' {
	load_video
	insmod gzio
	insmod part_msdos
	insmod xfs
	set root='hd0,msdos1'
	if [ x$feature_platform_search_hint = xy ]; then
	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  e7d4883f-c825-49bc-907a-8ec9bd22e774
	else
	  search --no-floppy --fs-uuid --set=root e7d4883f-c825-49bc-907a-8ec9bd22e774
	fi
	linux16 /vmlinuz-0-rescue-e8569f127454499cb00018a87844b14d root=UUID=e37695d8-f6df-490a-9a0a-b13cdfbea99d ro rootflags=subvol=root crashkernel=auto rhgb quiet
	initrd16 /initramfs-0-rescue-e8569f127454499cb00018a87844b14d.img
}

### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###
### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/20_ppc_terminfo ###
### END /etc/grub.d/20_ppc_terminfo ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries. Simply type the

# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f ${config_directory}/custom.cfg ]; then
source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f $prefix/custom.cfg ]; then
source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###

#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub2-mkconfig using templates

# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
set pager=1

if [ -s $prefix/grubenv ]; then
load_env
fi
if [ "${next_entry}" ] ; then
set default="${next_entry}"
set next_entry=
save_env next_entry
set boot_once=true
else
set default="${saved_entry}"
fi

if [ x"${feature_menuentry_id}" = xy ]; then
menuentry_id_option="--id"
else
menuentry_id_option=""
fi

export menuentry_id_option

if [ "${prev_saved_entry}" ]; then
set saved_entry="${prev_saved_entry}"
save_env saved_entry
set prev_saved_entry=
save_env prev_saved_entry
set boot_once=true
fi

function savedefault {
if [ -z "${boot_once}" ]; then

    saved_entry="${chosen}"
    save_env saved_entry

fi
}

function load_video {
if [ x$feature_all_video_module = xy ]; then
insmod all_video
else

    insmod efi_gop
    insmod efi_uga
    insmod ieee1275_fb
    insmod vbe
    insmod vga
    insmod video_bochs
    insmod video_cirrus

fi
}

terminal_output console
if [ x$feature_timeout_style = xy ] ; then
set timeout_style=menu
set timeout=5
# Fallback normal timeout code in case the timeout_style feature is

# unavailable.
else
  set timeout=5
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/00_tuned ###
set tuned_params=""
### END /etc/grub.d/00_tuned ###

### BEGIN /etc/grub.d/01_users ###
if [ -f ${prefix}/user.cfg ]; then
source ${prefix}/user.cfg
if [ -n "${GRUB2_PASSWORD}" ]; then

    set superusers="root"
    export superusers
    password_pbkdf2 root ${GRUB2_PASSWORD}

fi
fi
### END /etc/grub.d/01_users ###

### BEGIN /etc/grub.d/10_linux ###

menuentry 'Rockstor (4.3.3-1.el7.elrepo.x86_64) 3 (Core)' --class rhel fedora --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-4.3.3-1.el7.elrepo.x86_64-advanced-43b85d9b-a042-4f4b-869f-9fefc37e45e6' {
	load_video
	set gfxpayload=keep
	insmod gzio
	insmod part_msdos
	insmod ext2
	set root='hd0,msdos1'
	if [ x$feature_platform_search_hint = xy ]; then
	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  53f8a17d-3820-4a4c-9899-6ef54b660e16
	else
	  search --no-floppy --fs-uuid --set=root 53f8a17d-3820-4a4c-9899-6ef54b660e16
	fi
	linux16 /vmlinuz-4.3.3-1.el7.elrepo.x86_64 root=UUID=43b85d9b-a042-4f4b-869f-9fefc37e45e6 ro rootflags=subvol=root crashkernel=auto rhgb quiet LANG=en_US.UTF-8
	initrd16 /initramfs-4.3.3-1.el7.elrepo.x86_64.img
}
menuentry 'Rockstor (0-rescue-49bed99205f941db9ea41e79cc8fbe84) 3 (Core)' --class rhel fedora --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-49bed99205f941db9ea41e79cc8fbe84-advanced-43b85d9b-a042-4f4b-869f-9fefc37e45e6' {
	load_video
	insmod gzio
	insmod part_msdos
	insmod ext2
	set root='hd0,msdos1'
	if [ x$feature_platform_search_hint = xy ]; then
	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  53f8a17d-3820-4a4c-9899-6ef54b660e16
	else
	  search --no-floppy --fs-uuid --set=root 53f8a17d-3820-4a4c-9899-6ef54b660e16
	fi
	linux16 /vmlinuz-0-rescue-49bed99205f941db9ea41e79cc8fbe84 root=UUID=43b85d9b-a042-4f4b-869f-9fefc37e45e6 ro rootflags=subvol=root crashkernel=auto rhgb quiet
	initrd16 /initramfs-0-rescue-49bed99205f941db9ea41e79cc8fbe84.img
}

### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###

### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/20_ppc_terminfo ###
### END /etc/grub.d/20_ppc_terminfo ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries. Simply type the

# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f ${config_directory}/custom.cfg ]; then
source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f $prefix/custom.cfg ]; then
source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###

Rockstor uses an ext234 file system for /boot, where CentOS 7x by default uses XFS. This is a non-factor. I see no meaningful difference in the grub.cfg.

I expect the problem is in the initramfs or udev rules. You'll probably need to boot with some combination of 'systemd.log_level=debug rd.debug rd.udev.debug' using all three options at the same time will make boot incredibly slow, and 'journalctl -b' will be extremely verbose. So you just try systemd debug first and see if that helps sort out whether this is a dracut or udev problem.

Chris Murphy