[arch-releng] [PATCH] Rewrite of finddisks and findblockdevices. Rename findblockdevices to find_usable_devices
find_usable_devs returns a list of partitioned and non-partitioned block devices. These devices include SCSI/Sata, CCISS, IDA, LVM, SOFT RAID devices. Devices in LVM and SOFT RAID configurations are excluded from this list. --- src/core/libs/lib-blockdevices-filesystems.sh | 165 +++++++++++++------------ src/core/libs/lib-ui-interactive.sh | 4 +- 2 files changed, 88 insertions(+), 81 deletions(-) diff --git a/src/core/libs/lib-blockdevices-filesystems.sh b/src/core/libs/lib-blockdevices-filesystems.sh index ac749a3..c74b938 100644 --- a/src/core/libs/lib-blockdevices-filesystems.sh +++ b/src/core/libs/lib-blockdevices-filesystems.sh @@ -206,99 +206,106 @@ getlabel() { # find partitionable blockdevices # $1 extra things to echo for each device (optional) (backslash escapes will get interpreted) finddisks() { - workdir="$PWD" - if cd /sys/block 2>/dev/null - then - # ide devices - for dev in $(ls | egrep '^hd') - do - if [ "$(cat $dev/device/media)" = "disk" ] - then - echo -ne "/dev/$dev $1" - fi - done - #scsi/sata devices, and virtio blockdevices (/dev/vd*) - for dev in $(ls | egrep '^[sv]d') - do - # TODO: what is the significance of 5? ASKDEV - if [ "$(cat $dev/device/type)" != "5" ] - then - echo -ne "/dev/$dev $1" + shopt -s nullglob + + # Block Devices + for dev in /sys/block/*; do + if [[ -f $dev/device/type ]]; then + local type + read type < /sys/block/${dev##*/}/device/type + # Block Devices with size =< 0 may be an empty card reader + read size < /sys/block/${dev##*/}/size + # Type 5 is a ROM Device - Optical Drives + if [[ $type != 5 ]] && (( $size > 0 )); then + source "$dev/uevent" + echo -ne "/dev/$DEVNAME $1" + unset DEVNAME fi - done - fi + fi + done + # cciss controllers - if cd /dev/cciss 2>/dev/null - then - for dev in $(ls | egrep -v 'p') - do - echo -ne "/dev/cciss/$dev $1" - done - fi - # Smart 2 controllers - if cd /dev/ida 2>/dev/null - then - for dev in $(ls | egrep -v 'p') - do - echo -ne "/dev/ida/$dev $1" - done - fi - cd "$workdir" + for dev in /dev/cciss/*; do + if [[ $dev != *[[:digit:]]p[[:digit:]]* ]]; then + echo "$dev $1" + fi + done + + # Smart 2 Controller + for dev in /dev/ida/*; do + if [[ $dev != *[[:digit:]]p[[:digit:]]* ]]; then + echo "$dev $1" + fi + done + + shopt -u nullglob } -# find block devices, both partionable or not (i.e. partitions themselves) + +# find usable devices, both partionable or not (i.e. partitions themselves) # $1 extra things to echo for each partition (optional) (backslash escapes will get interpreted) -findblockdevices() { - workdir="$PWD" - for devpath in $(finddisks) - do - disk=$(basename $devpath) - echo -ne "/dev/$disk $1" - cd /sys/block/$disk - shopt -s nullglob - for part in $disk* - do - # check if not already assembled to a raid device. TODO: what is the significance of the 5? ASKDEV - if [ -n "$part" ] && ! grep -q $part /proc/mdstat 2>/dev/null && ! fstype 2>/dev/null </dev/$part | grep -q lvm2 && ! sfdisk -c /dev/$disk $(echo $part | sed -e "s#$disk##g") 2>/dev/null | grep -q '5' - then - if [ -d $part ] - then - echo -ne "/dev/$part $1" - fi +find_usable_blockdevices() { + shopt -s nullglob + + local parts + + # Cycle through all root block devices (sda, sdb, etc...) + # Look for partitions and include them only if they are not part of a + # RAID or LVM configuration. Also, exclude extended partitions + for devpath in $(finddisks); do + hidebldev= + unset parts + + # Glob allows for following matches: + # /dev/sda -> /dev/sda1 + # /dev/cciss/c0d1 -> /dev/cciss/c0d1p1 + for dev in ${devpath}*[[:digit:]]*; do + local disk="${dev%%[[:digit:]]*}" + local partnum="${dev##*[[:alpha:]]}" + + # Don't display extened partition (not logical parts) + [ "$(sfdisk -c "$disk" "$partnum" 2>/dev/null)" = 5 ] && continue; + + # Don't list parts that are part of RAID or LVM volumes + # Although we strongly encourage users to setup LVM in AIF, we want to make + # life a bit easier for those who have setup LVM/softraid before running AIF + if grep -qsw "${dev##*/}" /proc/mdstat || { pvscan -s 2>/dev/null | grep -q "$dev"; }; then + hidebldev="True" + else + parts+=("$dev") fi done - shopt -u nullglob + + # If hidebldev is not set and we have no partitions then + # Echo root block device if root block device is not part of a LVM or RAID configuration + # Otherwise echo the partitions + if [[ -z $hidebldev ]] && (( ! ${#parts[@]} )); then + if ! grep -qsw "${devpath##*/}" /proc/mdstat && ! pvscan -s 2>/dev/null | grep -q "$devpath"; then + echo -ne "$devpath $1" + fi + elif (( ${#parts[@]} )); then + for part in ${parts[@]}; do + echo -ne "$part $1" + done + fi done + # mapped devices - for devpath in $(ls /dev/mapper 2>/dev/null | grep -v control) - do - echo -ne "/dev/mapper/$devpath $1" + for devpath in /dev/mapper/*; do + # Exclude /control directory and other non-block files + if [[ -b $devpath ]]; then + echo -ne "$devpath $1" + fi done + # raid md devices - for devpath in $(ls -d /dev/md* | grep '[0-9]' 2>/dev/null) - do - if grep -qw $(echo $devpath /proc/mdstat | sed -e 's|/dev/||g') - then + for devpath in /dev/md[0-9]*; do + if grep -qw "${devpath//\/dev\//}" /proc/mdstat; then echo -ne "$devpath $1" fi done - # cciss controllers - if cd /dev/cciss 2>/dev/null - then - for dev in $(ls | egrep 'p') - do - echo -ne "/dev/cciss/$dev $1" - done - fi - # Smart 2 controllers - if cd /dev/ida 2>/dev/null - then - for dev in $(ls | egrep 'p') - do - echo -ne "/dev/ida/$dev $1" - done - fi - cd "$workdir" + + shopt -u nullglob } diff --git a/src/core/libs/lib-ui-interactive.sh b/src/core/libs/lib-ui-interactive.sh index b244a6e..64be570 100644 --- a/src/core/libs/lib-ui-interactive.sh +++ b/src/core/libs/lib-ui-interactive.sh @@ -557,7 +557,7 @@ interactive_filesystems() { if [ ! -f $TMP_BLOCKDEVICES ] || ! ask_yesno "Previous blockdevice definitions found:\n`cat $TMP_BLOCKDEVICES`\n\ Use these as a starting point? Make sure your disk(s) are partitioned correctly so your definitions can be applied. Pick 'no' when in doubt to start from scratch" no then - findblockdevices 'raw no_label no_fs\n' > $TMP_BLOCKDEVICES + find_usable_blockdevices 'raw no_label no_fs\n' > $TMP_BLOCKDEVICES fi [ -z "$PART_ACCESS" ] && PART_ACCESS=dev @@ -885,7 +885,7 @@ interactive_grub() { # Create and edit the grub menu.lst interactive_grub_menulst - DEVS="$(findblockdevices '_ ')" + DEVS="$(find_usable_blockdevices '_ ')" if [ "$DEVS" = " " ]; then notify "No hard drives were found" return 1 -- 1.7.4.1
Case: reinstall of my mediaserver: * Archlinux on SSD * files on (Adaptec 2610SA) RAID-5 (LVM) Followed the wiki on SSD, so I had to install gdisk and perform the partitioning with gdisk: /dev/sdb1: 1MiB formatted as BIOS /dev/sdb2: 300MiB formatted as ext2 (/boot) /dev/sdb3: rest formatted as ext4 (/) /arch/setup: * ERROR 1: I was trying to find the best way to 'manage the filesystems', and I screwed up, at which point I hit 'cancel'. I received an error 'ask_option $2 must be the title' and the installer exited. * ERROR 2: I tried once to use BTRFS instead of ext4, but it never allowed me to chose a mountpoint (BUT: as I continued trying, and used LVM - infra - I noticed that new entries were added to the list of filesystems to manage, so it could be that I did not notice the same behaviour when trying out BTRFS - if that is the case, one might chose to add a blank line before the new entries and after the /dev/sdxy's) * ERROR 3: Then I used ext4 on my sdb3, checking the option to format the partition with aif, but that failed. exiting the installer and preparing/formatting the disk manually, worked. Using aif only to set the mountpoints worked. * ERROR 4: Next was creating an LVM on the array. It was a simple LVM: 1 primary volume, 1 logical volume and formatted as ext4. I got an error on 'making lvm-lv': 'process_filesystem needs a partition as $1'. It occurred to me later that I might have entered a partition-size that was larger than the available amount, but the installer did not warn me about that and just exited. I also found it hard to use the LVM-portion of aif. At this point, I just wanted to get it installed - LVM was not necessary for me to complete the install. Chose the option 'by UUID' - that went all fine. Installing the base and base-devel went fine. Then I had to install a bootloader, but in my case I was told grub would not cut it: I needed grub2 or burg, so I exited without installing grub. I noticed that the installer report said that 'installing bootloader' had finished succesfully. I exited the installer and installed grub2 (I had to remove aif in order to be able to remove grub-legacy). I mounted my /dev/sdb2 as /boot and ran grub-install /dev/sdb This failed and I gave up (my wife was calling me for a little quality time). I think I can live with most of this, but any hints on how to install grub2 with aif (or after aif) would be much appreciated. The wiki on 'SSD' also told me that I had to make a 1 MiB bios-partition at the beginning of my SSD, but the wiki never says what you should do with that partition... That's it. Hope this info is helpful for you devvers. Keep on rocking! Vincent
Hi Vincent, first off all, please compose new mails rather then replying to unrelated threads. On Mon, 28 Feb 2011 17:15:49 +0100 Vincent Van Houtte <vvh@synergylaw.be> wrote:
Case: reinstall of my mediaserver: * Archlinux on SSD * files on (Adaptec 2610SA) RAID-5 (LVM)
Followed the wiki on SSD, so I had to install gdisk and perform the partitioning with gdisk: /dev/sdb1: 1MiB formatted as BIOS /dev/sdb2: 300MiB formatted as ext2 (/boot) /dev/sdb3: rest formatted as ext4 (/)
/arch/setup:
* ERROR 1: I was trying to find the best way to 'manage the filesystems', and I screwed up, at which point I hit 'cancel'. I received an error 'ask_option $2 must be the title' and the installer exited.
Do you have the aif logfile? Or the output of the report-issues script? Any specific reason you did not run report-issues? (it should inform you about that through the MOTD, is it not visible - enough - ?)
* ERROR 2: I tried once to use BTRFS instead of ext4, but it never allowed me to chose a mountpoint (BUT: as I continued trying, and used LVM - infra - I noticed that new entries were added to the list of filesystems to manage, so it could be that I did not notice the same behaviour when trying out BTRFS - if that is the case, one might chose to add a blank line before the new entries and after the /dev/sdxy's)
https://bugs.archlinux.org/task/22427
* ERROR 3: Then I used ext4 on my sdb3, checking the option to format the partition with aif, but that failed. exiting the installer and preparing/formatting the disk manually, worked. Using aif only to set the mountpoints worked.
I really need to know the specific error to be able to fix that. maybe the device was in use because of something you were doing? (or because aif did something with it in a previous run, in which case you should have rollbacked - unless aif crashed, of course)
* ERROR 4: Next was creating an LVM on the array. It was a simple LVM: 1 primary volume, 1 logical volume and formatted as ext4. I got an error on 'making lvm-lv': 'process_filesystem needs a partition as $1'. It occurred to me later that I might have entered a partition-size that was larger than the available amount, but the installer did not warn me about that and just exited. I also found it hard to use the LVM-portion of aif.
https://bugs.archlinux.org/task/19454
At this point, I just wanted to get it installed - LVM was not necessary for me to complete the install.
Chose the option 'by UUID' - that went all fine.
Installing the base and base-devel went fine.
Then I had to install a bootloader, but in my case I was told grub would not cut it: I needed grub2 or burg, so I exited without installing grub.
I noticed that the installer report said that 'installing bootloader' had finished succesfully.
maybe you selected the entry in the menu and then said "i will install my own", aif counts that as success.
I think I can live with most of this, but any hints on how to install grub2 with aif (or after aif) would be much appreciated.
grub2 is not supported, and I have no idea how it works. we might support it if someone can contribute simple code to make it work. we will probably support syslinux though, work on that is in progress (as you can see on this list)
The wiki on 'SSD' also told me that I had to make a 1 MiB bios-partition at the beginning of my SSD, but the wiki never says what you should do with that partition...
yeah well.. it's up to the community to maintain the wiki. I only take care of the official installation guide (and even there I expect input from the community)
That's it.
Hope this info is helpful for you devvers. Keep on rocking!
thanks, Dieter
Thank you for replying to my message.
first off all, please compose new mails rather then replying to unrelated threads.
My bad - you are right.
* ERROR 1: I was trying to find the best way to 'manage the filesystems', and I screwed up, at which point I hit 'cancel'. I received an error 'ask_option $2 must be the title' and the installer exited.
Do you have the aif logfile? Or the output of the report-issues script? Any specific reason you did not run report-issues? (it should inform you about that through the MOTD, is it not visible - enough - ?)
I think I have seen it, but the fact that I did not think of it a few moments later, I guess I didn't take it in. I'll try to reproduce and then run report-issues.
* ERROR 2: I tried once to use BTRFS instead of ext4, but it never allowed me to chose a mountpoint (BUT: as I continued trying, and used LVM - infra - I noticed that new entries were added to the list of filesystems to manage, so it could be that I did not notice the same behaviour when trying out BTRFS - if that is the case, one might chose to add a blank line before the new entries and after the /dev/sdxy's)
OK - no problem: I plan on using ext4 anyway.
* ERROR 3: Then I used ext4 on my sdb3, checking the option to format the partition with aif, but that failed. exiting the installer and preparing/formatting the disk manually, worked. Using aif only to set the mountpoints worked.
I really need to know the specific error to be able to fix that. maybe the device was in use because of something you were doing? (or because aif did something with it in a previous run, in which case you should have rollbacked - unless aif crashed, of course)
Nope, I forgot to add to my message that rolling back worked just fine. There was no real 'error' and aif did not crash - it just trew an error about being unable to create an ext4 filesystem. I'll try to reproduce it this evening and see if I can get an exact warning/error.
* ERROR 4: Next was creating an LVM on the array. It was a simple LVM: 1 primary volume, 1 logical volume and formatted as ext4. I got an error on 'making lvm-lv': 'process_filesystem needs a partition as $1'. It occurred to me later that I might have entered a partition-size that was larger than the available amount, but the installer did not warn me about that and just exited. I also found it hard to use the LVM-portion of aif.
Aah, my post-factum thought was probably correct. I'll try to do this correct tonight and report back. I noticed that there is no option to use the entire disk for an LV, which might be a handy feature.
I noticed that the installer report said that 'installing bootloader' had finished succesfully.
maybe you selected the entry in the menu and then said "i will install my own", aif counts that as success.
Yes, that's what happened. Maybe I could suggest to change the 'succes'-message to 'manual installation'? It's not a bug per se, only something I noticed.
I think I can live with most of this, but any hints on how to install grub2 with aif (or after aif) would be much appreciated.
grub2 is not supported, and I have no idea how it works. we might support it if someone can contribute simple code to make it work. we will probably support syslinux though, work on that is in progress (as you can see on this list)
Yes, I have seen the patches. In the Ubuntu help docs I found an extra argument to grub-install (<grub2) named --root-directory, which might help installing grub2 for the newly installed system that is mounted under /mnt...
The wiki on 'SSD' also told me that I had to make a 1 MiB bios-partition at the beginning of my SSD, but the wiki never says what you should do with that partition...
yeah well.. it's up to the community to maintain the wiki. I only take care of the official installation guide (and even there I expect input from the community)
I was not blaming you or anyone else - it's just something I don't know, and kinda hoped someone on this list might shed some light upon (after all, there are devs around in this channel writing code around bootloaders etc...)
thanks,
No, thank YOU for your time and efforts. I can only hope that a report like this one is helpful and not just taking more of your time. That's why I will run the same routine this evening and provide 'report-issues' where I can... Vincent
So, back after three new install-runs. I used report-issues this time. The install went fine up until I chose to manage the mount points. I did not chose aif to handle making partitions, because cfdisk cannot handle GPT. My settings were as follows: 1. My SSD needs two partitions: /boot on ext2 and / on ext4. Recreating both these FS's have worked 3/3 - confirmed; 2. My backup disk needed to be mounted without recreating the FS: /media/backup on ext4: worked 3/3 - although not really confirmed: it never threw an error, but maybe aif crashed before getting to this point; 3. My RAID-array needed to be mounted as a new LVM. I want the LV to take up the whole array, so I chose the value that aif showed as its capacity: 915522MiB. This failed: 0/1 Next, I tried with a value '915000MiB': this failed also: 0/2 Last, I tried with the default value of 5000: this also failed: 0/3. aif showed all three times the error 'process_filesystem error': error creating filesystem lvm-pv on /dev/sda Then it proceeded with making the FS's on /dev/sdb Then it gave me 10 iterations of the same error as above, but then with regards to lvm-vg and again lvm-pv. Then aif crashed and I reported the crash: http://sprunge.us/aMUe http://sprunge.us/jMGM (third time I forgot to do this, but the settings - except for the size of the lvm - and errors were identical). Final notice: During my second and third try, I selected 'no' to start over and not go further where the last installation had failed. Nonetheless the dialog box when creating FS's included all the messages from the previous installation-attempt.
* ERROR 3: Then I used ext4 on my sdb3, checking the option to format the partition with aif, but that failed. exiting the installer and preparing/formatting the disk manually, worked. Using aif only to set the mountpoints worked.
I really need to know the specific error to be able to fix that. maybe the device was in use because of something you were doing? (or because aif did something with it in a previous run, in which case you should have rollbacked - unless aif crashed, of course)
I did not see the same error this evening: creating the ext4 FS worked 3/3.
* ERROR 4: Next was creating an LVM on the array. It was a simple LVM: 1 primary volume, 1 logical volume and formatted as ext4. I got an error on 'making lvm-lv': 'process_filesystem needs a partition as $1'. It occurred to me later that I might have entered a partition-size that was larger than the available amount, but the installer did not warn me about that and just exited. I also found it hard to use the LVM-portion of aif.
Apparently, this was not the case for my problem. I know the RAID-5 array is 4x 320GB, of which 3x320GB should be usable. even when I chose 5000MiB, it failed. I hope the report-issues bring you more insight in this. Vincent -- Advocatenkantoor Suy, Van Baeveghem & Van Houtte Brusselsestraat 108 9200 Dendermonde T +32.52.52.06.05 F +32.52.52.06.46 W http://www.synergylaw.be
After failing miserably for several hours, I found out that:
aif showed all three times the error 'process_filesystem error': error creating filesystem lvm-pv on /dev/sda
This is due to not having made a partition on the device before making it an lvm-pv. Vincent -- Advocatenkantoor Suy, Van Baeveghem & Van Houtte Brusselsestraat 108 9200 Dendermonde T +32.52.52.06.05 F +32.52.52.06.46 W http://www.synergylaw.be
On Mon, 28 Feb 2011 23:01:59 +0100 Vincent Van Houtte <vvh@synergylaw.be> wrote:
After failing miserably for several hours, I found out that:
aif showed all three times the error 'process_filesystem error': error creating filesystem lvm-pv on /dev/sda
This is due to not having made a partition on the device before making it an lvm-pv.
Vincent
yeah, if you want to make a partition an lvm-pv, the partition should obviously exist first. Dieter
Hello,
After failing miserably for several hours, I found out that:
aif showed all three times the error 'process_filesystem error': error creating filesystem lvm-pv on /dev/sda
This is due to not having made a partition on the device before making it an lvm-pv.
yeah, if you want to make a partition an lvm-pv, the partition should obviously exist first.
You are obviously right, but I think it should be made more clear, maybe by only showing available partitions instead of devices in that stage of the install process. For me it was my first experience with LVM, and I didn't know LVM works on top of partitions instead of devices (which, I know, is my fault entirely). Vincent
On Wed, 2 Mar 2011 15:08:27 +0100 Vincent Van Houtte <vvh@synergylaw.be> wrote:
Hello,
After failing miserably for several hours, I found out that:
aif showed all three times the error 'process_filesystem error': error creating filesystem lvm-pv on /dev/sda
This is due to not having made a partition on the device before making it an lvm-pv.
yeah, if you want to make a partition an lvm-pv, the partition should obviously exist first.
You are obviously right, but I think it should be made more clear, maybe by only showing available partitions instead of devices in that stage of the install process. For me it was my first experience with LVM, and I didn't know LVM works on top of partitions instead of devices (which, I know, is my fault entirely).
Vincent
you should be able to make any block device an LVM physical volume, it doesn't need to be a partition. Dieter
Op woensdag 02-03-2011 om 15:12 uur [tijdzone +0100], schreef Dieter Plaetinck:
you should be able to make any block device an LVM physical volume, it doesn't need to be a partition.
In that case aif did not handle this well. It crashed three times upon setting a pv directly onto a device (/dev/sda). Vincent -- Advocatenkantoor Suy, Van Baeveghem & Van Houtte Brusselsestraat 108 9200 Dendermonde T +32.52.52.06.05 F +32.52.52.06.46 W http://www.synergylaw.be
How hard is it to just go over everything I asked, fix it and if you don't know what I mean, ask for clarifications? (I'm referring to my questions such as "non-block files such as?", the "extended partition" typo, my response to $hidebldev, and a few other remarks about comments) It would at least be simpler then expecting me to do the same work multiple times. But fuck it.. the code seems like good quality so I merged it and did some minor changes to your patch myself. I want people to be able to test this new code. I also cleaned up the commit message (<80 chars and it had the wrong function name and unclear info) so yeah thanks, this is a nice contribution (and your other work aswell) after all. you just need to work a bit on being more thorough so we can work efficiently. Dieter
participants (3)
-
Dieter Plaetinck
-
pyther@pyther.net
-
Vincent Van Houtte