Various improvements to submodule handling

Some users can't run a bare 'git' command, due to need for a transparent proxying solution such as 'tsocks'. This adds an argument to configure to let users specify such a thing:

  ./configure --with-git="tsocks git"

The submodule script is also updated to give the user a hint about using this flag, if we fail to checkout modules.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 Makefile                 |  4 ++--
 configure                |  5 +++++
 scripts/git-submodule.sh | 30 +++++++++++++++++++++++++-----
 3 files changed, 32 insertions(+), 7 deletions(-)

Makefile +2 -2

index 9372742f86..4c9d0eaef2 100644
@@ -21,14 +21,14 @@ git-submodule-update:
 ifeq (0,$(MAKELEVEL))
   git_module_status := $(shell \
     cd '$(SRC_PATH)' && \
-    ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
     echo $$?; \
   )
 
 ifeq (1,$(git_module_status))
 git-submodule-update:
 	$(call quiet-command, \
-          (cd $(SRC_PATH) && ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
+          (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif

configure +5 -0

index 03547cea6a..65765968f3 100755
@@ -271,6 +271,7 @@ then
 else
     git_submodules=""
 fi
+git="git"
 
 # Don't accept a target_list environment variable.
 unset target_list

@@ -1294,6 +1295,8 @@ for opt do
           error_exit "vhost-user isn't available on win32"
       fi
   ;;
+  --with-git=*) git="$optarg"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"

@@ -5338,6 +5341,7 @@ echo "local state directory   queried at runtime"
 echo "Windows SDK       $win_sdk"
 fi
 echo "Source path       $source_path"
+echo "GIT binary        $git"
 echo "GIT submodules    $git_submodules"
 echo "C compiler        $cc"
 echo "Host C compiler   $host_cc"

@@ -5528,6 +5532,7 @@ echo "extra_cxxflags=$EXTRA_CXXFLAGS" >> $config_host_mak
 echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
+echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak

scripts/git-submodule.sh +13 -5

index 08932a35f0..c66567d409 100755
@@ -3,14 +3,19 @@
 # This code is licensed under the GPL version 2 or later.  See
 # the COPYING file in the top-level directory.
 
-set -e
-
 substat=".git-submodule-status"
 
 command=$1
 shift
 modules="$@"
 
+test -z "$GIT" && GIT=git
+
+error() {
+    printf "$0: %s\n" "$*" >&2
+    exit 1
+}
+
 if test -z "$modules"
 then
     test -e $substat || touch $substat

@@ -27,12 +32,27 @@ case "$command" in
 status)
     test -f "$substat" || exit 1
     trap "rm -f ${substat}.tmp" EXIT
-    git submodule status $modules > "${substat}.tmp"
+    $GIT submodule status $modules > "${substat}.tmp"
+    test $? -ne 0 && error "failed to query git submodule status"
     diff "${substat}" "${substat}.tmp" >/dev/null
     exit $?
     ;;
 update)
-    git submodule update --init $modules 1>/dev/null
-    git submodule status $modules > "${substat}"
+    $GIT submodule update --init $modules 1>/dev/null
+    if test $? -ne 0 ; then
+        echo
+        echo "Unable to automatically checkout GIT submodules '$modules'."

scripts/git-submodule.sh +2 -0

     ;;
 esac
+
+exit 0

Quoted 117 lines · Daniel P. Berrange

Some users can't run a bare 'git' command, due to need for a transparent proxying solution such as 'tsocks'. This adds an argument to configure to let users specify such a thing:

  ./configure --with-git="tsocks git"

The submodule script is also updated to give the user a hint about using this flag, if we fail to checkout modules.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 Makefile                 |  4 ++--
 configure                |  5 +++++
 scripts/git-submodule.sh | 30 +++++++++++++++++++++++++-----
 3 files changed, 32 insertions(+), 7 deletions(-)

Makefile +2 -2

index 9372742f86..4c9d0eaef2 100644
@@ -21,14 +21,14 @@ git-submodule-update:
 ifeq (0,$(MAKELEVEL))
   git_module_status := $(shell \
     cd '$(SRC_PATH)' && \
-    ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
     echo $$?; \
   )
 
 ifeq (1,$(git_module_status))
 git-submodule-update:
 	$(call quiet-command, \
-          (cd $(SRC_PATH) && ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
+          (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif

configure +5 -0

index 03547cea6a..65765968f3 100755
@@ -271,6 +271,7 @@ then
 else
     git_submodules=""
 fi
+git="git"
 
 # Don't accept a target_list environment variable.
 unset target_list

@@ -1294,6 +1295,8 @@ for opt do
           error_exit "vhost-user isn't available on win32"
       fi
   ;;
+  --with-git=*) git="$optarg"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"

@@ -5338,6 +5341,7 @@ echo "local state directory   queried at runtime"
 echo "Windows SDK       $win_sdk"
 fi
 echo "Source path       $source_path"
+echo "GIT binary        $git"
 echo "GIT submodules    $git_submodules"
 echo "C compiler        $cc"
 echo "Host C compiler   $host_cc"

@@ -5528,6 +5532,7 @@ echo "extra_cxxflags=$EXTRA_CXXFLAGS" >> $config_host_mak
 echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
+echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak

scripts/git-submodule.sh +13 -5

index 08932a35f0..c66567d409 100755
@@ -3,14 +3,19 @@
 # This code is licensed under the GPL version 2 or later.  See
 # the COPYING file in the top-level directory.
 
-set -e
-
 substat=".git-submodule-status"
 
 command=$1
 shift
 modules="$@"
 
+test -z "$GIT" && GIT=git
+
+error() {
+    printf "$0: %s\n" "$*" >&2
+    exit 1
+}
+
 if test -z "$modules"
 then
     test -e $substat || touch $substat

@@ -27,12 +32,27 @@ case "$command" in
 status)
     test -f "$substat" || exit 1
     trap "rm -f ${substat}.tmp" EXIT
-    git submodule status $modules > "${substat}.tmp"
+    $GIT submodule status $modules > "${substat}.tmp"
+    test $? -ne 0 && error "failed to query git submodule status"
     diff "${substat}" "${substat}.tmp" >/dev/null
     exit $?
     ;;
 update)
-    git submodule update --init $modules 1>/dev/null
-    git submodule status $modules > "${substat}"
+    $GIT submodule update --init $modules 1>/dev/null
+    if test $? -ne 0 ; then
+        echo
+        echo "Unable to automatically checkout GIT submodules '$modules'."

scripts/git-submodule.sh case "$command" in · line 54

Quoted 10 lines

scripts/git-submodule.sh +10 -0

+        echo "If you require use of an alternative GIT binary (for example to"
+        echo "enable use of a transparent proxy), then please specify it by"
+        echo "running configure by with the '--with-git' argument. e.g."
+        echo
+        echo " $ ./configure --with-git='tsocks git'"
+        echo
+        exit 1
+    fi
+    $GIT submodule status $modules > "${substat}"
+    test $? -ne 0 && error "failed to save git submodule status"

The way I am testing it - I simply delete .git-submodule-status (I used to change it but deleting works as well) and then I get:

./scripts/git-submodule.sh: 74: ./scripts/git-submodule.sh: cannot create .git-submodule-status: Read-only file system

because "git submodule update" returns 0 (as everything is up to date) but updating status fails. Which is fine, I would just like to get a better message as even after few days of reading this script, I do not remember in what order I should pass submodules to scripts/git-submodule.sh. Yeah, I can find it in output but even the name of script to run does not stick to my brain :(

Something like this:

(quoted diff) +1 -1

-    test $? -ne 0 && error "failed to save git submodule status"
+    test $? -ne 0 && error "\"$GIT submodule status $modules\" failed to

save git submodule status"

Quoted 143 lines · Alexey Kardashevskiy

Quoted 117 lines · Daniel P. Berrange

Some users can't run a bare 'git' command, due to need for a transparent proxying solution such as 'tsocks'. This adds an argument to configure to let users specify such a thing:

  ./configure --with-git="tsocks git"

The submodule script is also updated to give the user a hint about using this flag, if we fail to checkout modules.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 Makefile                 |  4 ++--
 configure                |  5 +++++
 scripts/git-submodule.sh | 30 +++++++++++++++++++++++++-----
 3 files changed, 32 insertions(+), 7 deletions(-)

Makefile +2 -2

index 9372742f86..4c9d0eaef2 100644
@@ -21,14 +21,14 @@ git-submodule-update:
 ifeq (0,$(MAKELEVEL))
   git_module_status := $(shell \
     cd '$(SRC_PATH)' && \
-    ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
     echo $$?; \
   )
 
 ifeq (1,$(git_module_status))
 git-submodule-update:
 	$(call quiet-command, \
-          (cd $(SRC_PATH) && ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
+          (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif

configure +5 -0

index 03547cea6a..65765968f3 100755
@@ -271,6 +271,7 @@ then
 else
     git_submodules=""
 fi
+git="git"
 
 # Don't accept a target_list environment variable.
 unset target_list

@@ -1294,6 +1295,8 @@ for opt do
           error_exit "vhost-user isn't available on win32"
       fi
   ;;
+  --with-git=*) git="$optarg"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"

@@ -5338,6 +5341,7 @@ echo "local state directory   queried at runtime"
 echo "Windows SDK       $win_sdk"
 fi
 echo "Source path       $source_path"
+echo "GIT binary        $git"
 echo "GIT submodules    $git_submodules"
 echo "C compiler        $cc"
 echo "Host C compiler   $host_cc"

@@ -5528,6 +5532,7 @@ echo "extra_cxxflags=$EXTRA_CXXFLAGS" >> $config_host_mak
 echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
+echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak

scripts/git-submodule.sh +13 -5

index 08932a35f0..c66567d409 100755
@@ -3,14 +3,19 @@
 # This code is licensed under the GPL version 2 or later.  See
 # the COPYING file in the top-level directory.
 
-set -e
-
 substat=".git-submodule-status"
 
 command=$1
 shift
 modules="$@"
 
+test -z "$GIT" && GIT=git
+
+error() {
+    printf "$0: %s\n" "$*" >&2
+    exit 1
+}
+
 if test -z "$modules"
 then
     test -e $substat || touch $substat

@@ -27,12 +32,27 @@ case "$command" in
 status)
     test -f "$substat" || exit 1
     trap "rm -f ${substat}.tmp" EXIT
-    git submodule status $modules > "${substat}.tmp"
+    $GIT submodule status $modules > "${substat}.tmp"
+    test $? -ne 0 && error "failed to query git submodule status"
     diff "${substat}" "${substat}.tmp" >/dev/null
     exit $?
     ;;
 update)
-    git submodule update --init $modules 1>/dev/null
-    git submodule status $modules > "${substat}"
+    $GIT submodule update --init $modules 1>/dev/null
+    if test $? -ne 0 ; then
+        echo
+        echo "Unable to automatically checkout GIT submodules '$modules'."

scripts/git-submodule.sh case "$command" in · line 54

Quoted 10 lines

scripts/git-submodule.sh +10 -0

+        echo "If you require use of an alternative GIT binary (for example to"
+        echo "enable use of a transparent proxy), then please specify it by"
+        echo "running configure by with the '--with-git' argument. e.g."
+        echo
+        echo " $ ./configure --with-git='tsocks git'"
+        echo
+        exit 1
+    fi
+    $GIT submodule status $modules > "${substat}"
+    test $? -ne 0 && error "failed to save git submodule status"

The way I am testing it - I simply delete .git-submodule-status (I used to change it but deleting works as well) and then I get:

./scripts/git-submodule.sh: 74: ./scripts/git-submodule.sh: cannot create .git-submodule-status: Read-only file system

because "git submodule update" returns 0 (as everything is up to date) but updating status fails. Which is fine, I would just like to get a better message as even after few days of reading this script, I do not remember in what order I should pass submodules to scripts/git-submodule.sh. Yeah, I can find it in output but even the name of script to run does not stick to my brain :(

Quoted 5 lines

Something like this:

(quoted diff) +1 -1

-    test $? -ne 0 && error "failed to save git submodule status"
+    test $? -ne 0 && error "\"$GIT submodule status $modules\" failed to

save git submodule status"

Take a look at the 3rd patch - it now prints out the exact command you would need to run in the writable-source dir.

Quoted 150 lines · Daniel P. Berrange

Quoted 148 lines · Alexey Kardashevskiy

Quoted 117 lines · Daniel P. Berrange

Some users can't run a bare 'git' command, due to need for a transparent proxying solution such as 'tsocks'. This adds an argument to configure to let users specify such a thing:

  ./configure --with-git="tsocks git"

The submodule script is also updated to give the user a hint about using this flag, if we fail to checkout modules.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 Makefile                 |  4 ++--
 configure                |  5 +++++
 scripts/git-submodule.sh | 30 +++++++++++++++++++++++++-----
 3 files changed, 32 insertions(+), 7 deletions(-)

Makefile +2 -2

index 9372742f86..4c9d0eaef2 100644
@@ -21,14 +21,14 @@ git-submodule-update:
 ifeq (0,$(MAKELEVEL))
   git_module_status := $(shell \
     cd '$(SRC_PATH)' && \
-    ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
     echo $$?; \
   )
 
 ifeq (1,$(git_module_status))
 git-submodule-update:
 	$(call quiet-command, \
-          (cd $(SRC_PATH) && ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
+          (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif

configure +5 -0

index 03547cea6a..65765968f3 100755
@@ -271,6 +271,7 @@ then
 else
     git_submodules=""
 fi
+git="git"
 
 # Don't accept a target_list environment variable.
 unset target_list

@@ -1294,6 +1295,8 @@ for opt do
           error_exit "vhost-user isn't available on win32"
       fi
   ;;
+  --with-git=*) git="$optarg"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"

@@ -5338,6 +5341,7 @@ echo "local state directory   queried at runtime"
 echo "Windows SDK       $win_sdk"
 fi
 echo "Source path       $source_path"
+echo "GIT binary        $git"
 echo "GIT submodules    $git_submodules"
 echo "C compiler        $cc"
 echo "Host C compiler   $host_cc"

@@ -5528,6 +5532,7 @@ echo "extra_cxxflags=$EXTRA_CXXFLAGS" >> $config_host_mak
 echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
+echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak

scripts/git-submodule.sh +13 -5

index 08932a35f0..c66567d409 100755
@@ -3,14 +3,19 @@
 # This code is licensed under the GPL version 2 or later.  See
 # the COPYING file in the top-level directory.
 
-set -e
-
 substat=".git-submodule-status"
 
 command=$1
 shift
 modules="$@"
 
+test -z "$GIT" && GIT=git
+
+error() {
+    printf "$0: %s\n" "$*" >&2
+    exit 1
+}
+
 if test -z "$modules"
 then
     test -e $substat || touch $substat

@@ -27,12 +32,27 @@ case "$command" in
 status)
     test -f "$substat" || exit 1
     trap "rm -f ${substat}.tmp" EXIT
-    git submodule status $modules > "${substat}.tmp"
+    $GIT submodule status $modules > "${substat}.tmp"
+    test $? -ne 0 && error "failed to query git submodule status"
     diff "${substat}" "${substat}.tmp" >/dev/null
     exit $?
     ;;
 update)
-    git submodule update --init $modules 1>/dev/null
-    git submodule status $modules > "${substat}"
+    $GIT submodule update --init $modules 1>/dev/null
+    if test $? -ne 0 ; then
+        echo
+        echo "Unable to automatically checkout GIT submodules '$modules'."

scripts/git-submodule.sh case "$command" in · line 54

Quoted 10 lines

scripts/git-submodule.sh +10 -0

+        echo "If you require use of an alternative GIT binary (for example to"
+        echo "enable use of a transparent proxy), then please specify it by"
+        echo "running configure by with the '--with-git' argument. e.g."
+        echo
+        echo " $ ./configure --with-git='tsocks git'"
+        echo
+        exit 1
+    fi
+    $GIT submodule status $modules > "${substat}"
+    test $? -ne 0 && error "failed to save git submodule status"

The way I am testing it - I simply delete .git-submodule-status (I used to change it but deleting works as well) and then I get:

./scripts/git-submodule.sh: 74: ./scripts/git-submodule.sh: cannot create .git-submodule-status: Read-only file system

because "git submodule update" returns 0 (as everything is up to date) but updating status fails. Which is fine, I would just like to get a better message as even after few days of reading this script, I do not remember in what order I should pass submodules to scripts/git-submodule.sh. Yeah, I can find it in output but even the name of script to run does not stick to my brain :(

Something like this:

(quoted diff) +1 -1

-    test $? -ne 0 && error "failed to save git submodule status"
+    test $? -ne 0 && error "\"$GIT submodule status $modules\" failed to

save git submodule status"

Quoted 2 lines

Take a look at the 3rd patch - it now prints out the exact command you would need to run in the writable-source dir.

For the message in 3/4 to show up, I need to 1) know about --disable-git-update and 2) use it. My testcase is lot more common - I did not use --disable-git-update, and a strange message about writing read-only folder appears, exactly like when I started the conversation.

Quoted 154 lines · Alexey Kardashevskiy

Quoted 152 lines · Daniel P. Berrange

Quoted 147 lines · Alexey Kardashevskiy

Quoted 117 lines · Daniel P. Berrange

Some users can't run a bare 'git' command, due to need for a transparent proxying solution such as 'tsocks'. This adds an argument to configure to let users specify such a thing:

  ./configure --with-git="tsocks git"

The submodule script is also updated to give the user a hint about using this flag, if we fail to checkout modules.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 Makefile                 |  4 ++--
 configure                |  5 +++++
 scripts/git-submodule.sh | 30 +++++++++++++++++++++++++-----
 3 files changed, 32 insertions(+), 7 deletions(-)

Makefile +2 -2

index 9372742f86..4c9d0eaef2 100644
@@ -21,14 +21,14 @@ git-submodule-update:
 ifeq (0,$(MAKELEVEL))
   git_module_status := $(shell \
     cd '$(SRC_PATH)' && \
-    ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
     echo $$?; \
   )
 
 ifeq (1,$(git_module_status))
 git-submodule-update:
 	$(call quiet-command, \
-          (cd $(SRC_PATH) && ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
+          (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif

configure +5 -0

index 03547cea6a..65765968f3 100755
@@ -271,6 +271,7 @@ then
 else
     git_submodules=""
 fi
+git="git"
 
 # Don't accept a target_list environment variable.
 unset target_list

@@ -1294,6 +1295,8 @@ for opt do
           error_exit "vhost-user isn't available on win32"
       fi
   ;;
+  --with-git=*) git="$optarg"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"

@@ -5338,6 +5341,7 @@ echo "local state directory   queried at runtime"
 echo "Windows SDK       $win_sdk"
 fi
 echo "Source path       $source_path"
+echo "GIT binary        $git"
 echo "GIT submodules    $git_submodules"
 echo "C compiler        $cc"
 echo "Host C compiler   $host_cc"

@@ -5528,6 +5532,7 @@ echo "extra_cxxflags=$EXTRA_CXXFLAGS" >> $config_host_mak
 echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
+echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak

scripts/git-submodule.sh +13 -5

index 08932a35f0..c66567d409 100755
@@ -3,14 +3,19 @@
 # This code is licensed under the GPL version 2 or later.  See
 # the COPYING file in the top-level directory.
 
-set -e
-
 substat=".git-submodule-status"
 
 command=$1
 shift
 modules="$@"
 
+test -z "$GIT" && GIT=git
+
+error() {
+    printf "$0: %s\n" "$*" >&2
+    exit 1
+}
+
 if test -z "$modules"
 then
     test -e $substat || touch $substat

@@ -27,12 +32,27 @@ case "$command" in
 status)
     test -f "$substat" || exit 1
     trap "rm -f ${substat}.tmp" EXIT
-    git submodule status $modules > "${substat}.tmp"
+    $GIT submodule status $modules > "${substat}.tmp"
+    test $? -ne 0 && error "failed to query git submodule status"
     diff "${substat}" "${substat}.tmp" >/dev/null
     exit $?
     ;;
 update)
-    git submodule update --init $modules 1>/dev/null
-    git submodule status $modules > "${substat}"
+    $GIT submodule update --init $modules 1>/dev/null
+    if test $? -ne 0 ; then
+        echo
+        echo "Unable to automatically checkout GIT submodules '$modules'."

scripts/git-submodule.sh case "$command" in · line 54

Quoted 10 lines

scripts/git-submodule.sh +10 -0

+        echo "If you require use of an alternative GIT binary (for example to"
+        echo "enable use of a transparent proxy), then please specify it by"
+        echo "running configure by with the '--with-git' argument. e.g."
+        echo
+        echo " $ ./configure --with-git='tsocks git'"
+        echo
+        exit 1
+    fi
+    $GIT submodule status $modules > "${substat}"
+    test $? -ne 0 && error "failed to save git submodule status"

The way I am testing it - I simply delete .git-submodule-status (I used to change it but deleting works as well) and then I get:

./scripts/git-submodule.sh: 74: ./scripts/git-submodule.sh: cannot create .git-submodule-status: Read-only file system

because "git submodule update" returns 0 (as everything is up to date) but updating status fails. Which is fine, I would just like to get a better message as even after few days of reading this script, I do not remember in what order I should pass submodules to scripts/git-submodule.sh. Yeah, I can find it in output but even the name of script to run does not stick to my brain :(

Something like this:

(quoted diff) +1 -1

-    test $? -ne 0 && error "failed to save git submodule status"
+    test $? -ne 0 && error "\"$GIT submodule status $modules\" failed to

Quoted 1 line

save git submodule status"

Take a look at the 3rd patch - it now prints out the exact command you would need to run in the writable-source dir.

Quoted 4 lines

For the message in 3/4 to show up, I need to 1) know about --disable-git-update and 2) use it. My testcase is lot more common - I did not use --disable-git-update, and a strange message about writing read-only folder appears, exactly like when I started the conversation.

That isn't the behaviour I get with this patch series applied. I made my source dir readonly, and then tried a VPATH build and got the expected messages

...snip....
GIT ui/keycodemapdb
error: could not lock config file .git/config: Permission denied
error: could not lock config file .git/config: Permission denied
fatal: Failed to register url for submodule path 'ui/keycodemapdb'

Unable to automatically checkout GIT submodules ' ui/keycodemapdb'. If you require use of an alternative GIT binary (for example to enable use of a transparent proxy), then please specify it by running configure by with the '--with-git' argument. e.g.

 $ ./configure --with-git='tsocks git'

Alternatively you may disable automatic GIT submodule checkout with:

 $ ./configure --disable-git-update'

and then manually update submodules prior to running make, with:

 $ scripts/git-sbumodule.sh update  ui/keycodemapdb

make: *** [Makefile:40: git-submodule-update] Error 1

Regards,
Daniel

I run ./scripts/git-submodule.sh on a server (where source directory is writeable), them "rm .git-submodule-status", then run "Makefile" on a build machine (which has the source directory mounted as read-only). I basically recreate the situation where I was when I just discovered this brand new ./scripts/git-submodule.sh.

I know that now, all I am asking is an error message to print exact command to run...

Quoted 6 lines · Alexey Kardashevskiy

I run ./scripts/git-submodule.sh on a server (where source directory is writeable), them "rm .git-submodule-status", then run "Makefile" on a build machine (which has the source directory mounted as read-only). I basically recreate the situation where I was when I just discovered this brand new ./scripts/git-submodule.sh.

Don't rm the .git-submodule-status. That means 'make' thinks the modules are out of date.

Just run 'scripts/git-submodules.sh ...mods..' on the writable source dir, and then run 'make' on the build machine.

Quoted 2 lines

I know that now, all I am asking is an error message to print exact command to run...

If you hadn't deleted the .git-submodule-status, it would have worked fine.

Quoted 11 lines · Daniel P. Berrange

Quoted 6 lines · Alexey Kardashevskiy

I run ./scripts/git-submodule.sh on a server (where source directory is writeable), them "rm .git-submodule-status", then run "Makefile" on a build machine (which has the source directory mounted as read-only). I basically recreate the situation where I was when I just discovered this brand new ./scripts/git-submodule.sh.

Don't rm the .git-submodule-status. That means 'make' thinks the modules are out of date.

Quoted 7 lines

Just run 'scripts/git-submodules.sh ...mods..' on the writable source dir, and then run 'make' on the build machine.

Quoted 2 lines

I know that now, all I am asking is an error message to print exact command to run...

If you hadn't deleted the .git-submodule-status, it would have worked fine.

No.

I do this on a server:

[vpl1 qemu]$ git co v2.10.0
At this point no .git-submodule-status is expected/required/exist.

[vpl1 qemu]$ ssh aikhostos2 make -C /home/aik/pbuild/qemu-aikhostos2-ppc64/ -j24

Configure succeeds, compiles just fine, it has been working like this for years.

Now:

[vpl1 qemu]$ git co git-submodule (this is your stuff)
[vpl1 qemu]$ ssh aikhostos2 make -C /home/aik/pbuild/qemu-aikhostos2-ppc64/ -j24
touch: cannot touch ‘.git-submodule-status’: Read-only file system
make: Entering directory `/home/aik/pbuild/qemu-aikhostos2-ppc64'
GEN ppc64-softmmu/config-devices.mak.tmp
[...]

./scripts/git-submodule.sh: line 74: .git-submodule-status: Read-only file system
./scripts/git-submodule.sh: failed to save git submodule status
make: *** [git-submodule-update] Error 1

Quoted 44 lines · Alexey Kardashevskiy

Quoted 11 lines · Daniel P. Berrange

Quoted 6 lines · Alexey Kardashevskiy

I run ./scripts/git-submodule.sh on a server (where source directory is writeable), them "rm .git-submodule-status", then run "Makefile" on a build machine (which has the source directory mounted as read-only). I basically recreate the situation where I was when I just discovered this brand new ./scripts/git-submodule.sh.

Don't rm the .git-submodule-status. That means 'make' thinks the modules are out of date.

Quoted 7 lines

Just run 'scripts/git-submodules.sh ...mods..' on the writable source dir, and then run 'make' on the build machine.

Quoted 2 lines

I know that now, all I am asking is an error message to print exact command to run...

If you hadn't deleted the .git-submodule-status, it would have worked fine.

No.

I do this on a server:

[vpl1 qemu]$ git co v2.10.0
At this point no .git-submodule-status is expected/required/exist.

[vpl1 qemu]$ ssh aikhostos2 make -C /home/aik/pbuild/qemu-aikhostos2-ppc64/ -j24

Configure succeeds, compiles just fine, it has been working like this for years.

Now:

[vpl1 qemu]$ git co git-submodule (this is your stuff)
[vpl1 qemu]$ ssh aikhostos2 make -C /home/aik/pbuild/qemu-aikhostos2-ppc64/ -j24
touch: cannot touch ‘.git-submodule-status’: Read-only file system
make: Entering directory `/home/aik/pbuild/qemu-aikhostos2-ppc64'
GEN ppc64-softmmu/config-devices.mak.tmp
[...]

Quoted 4 lines

./scripts/git-submodule.sh: line 74: .git-submodule-status: Read-only file system
./scripts/git-submodule.sh: failed to save git submodule status
make: *** [git-submodule-update] Error 1

Oh I see, the git submodules list is empty because you have a previous built source tree, so its shortcircuiting the extra check I added. Tihs is easy enough to address

There are cases where users do VPATH builds with the source directory being on a read-only volume. In such a case they have to manually run the command 'git-submodule.sh ...modules...' manually ahead of time. When checking for

status we should not then write into the source dir.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 scripts/git-submodule.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

scripts/git-submodule.sh +0 -0

index c66567d409..586ff32293 100755

Quoted 3 lines · Daniel P. Berrange

There are cases where users do VPATH builds with the source directory being on a read-only volume. In such a case they have to manually run the command 'git-submodule.sh ...modules...' manually ahead of time. When checking for

One of the two "manually" can be removed (either one, and the sentence still makes sense)

Quoted 7 lines

status we should not then write into the source dir.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 scripts/git-submodule.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

scripts/git-submodule.sh fi · line 34

Quoted 13 lines

scripts/git-submodule.sh +1 -4

index c66567d409..586ff32293 100755
@@ -31,10 +31,10 @@ fi
 case "$command" in
 status)
     test -f "$substat" || exit 1
-    trap "rm -f ${substat}.tmp" EXIT
-    $GIT submodule status $modules > "${substat}.tmp"
-    test $? -ne 0 && error "failed to query git submodule status"
-    diff "${substat}" "${substat}.tmp" >/dev/null
+    substat_tmp=$(mktemp)

Is mktemp portable enough? Hopefully so.

Some people building QEMU use VPATH builds where the source directory is on a read-only volume. In such a case 'scripts/git-submodules.sh update' will always fail and users are required to run it manually themselves on their original writable source directory.

If going back in time in git history, across a commit that introduces a new submodule, the 'git-submodule.sh' script will fail, causing rebuild to fail.

This is because config-host.mak contains a GIT_SUBMODULES variable that lists a submodule that only exists in the later commit. config-host.mak won't get repopulated until config.status is invoked, but make won't get this far due to the submodule error.

This change makes 'git-submodule.sh' check whether each module is known to git and drops any which are not present. A warning message will be printed when any submodule is dropped in this manner.

Signed-off-by Daniel P. Berrange <[email protected]>

---
 scripts/git-submodule.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

scripts/git-submodule.sh +2 -2

index 30fd83db55..60b3b9bdeb 100755
@@ -7,7 +7,7 @@ substat=".git-submodule-status"
 
 command=$1
 shift
-modules="$@"
+maybe_modules="$@"
 
 test -z "$GIT" && GIT=git
 

@@ -16,12 +16,24 @@ error() {
     exit 1
 }
 
-if test -z "$modules"
+if test -z "$maybe_modules"

scripts/git-submodule.sh +7 -0

+    then
+        modules="$modules $m"
+    else
+        echo "warn: ignoring non-existant submodule $m"
+    fi
+done
+
 if ! test -e ".git"
 then
     echo "$0: unexpectedly called with submodules but no git checkout exists"

scripts/git-submodule.sh error() { · line 29

Quoted 7 lines · Daniel P. Berrange

If going back in time in git history, across a commit that introduces a new submodule, the 'git-submodule.sh' script will fail, causing rebuild to fail.

scripts/git-submodule.sh +3 -0

+do
+    $GIT submodule status $m 1> /dev/null 2>&1
+    if test $? == 0

./scripts/git-submodule.sh: 29: test: 0: unexpected operator

I've sent a variety of improvements to submodule handling as separate patches and its getting slightly confusing. So here is a series with all patches added together. I've also included a fix that helps us go backwards in time across the introduction of a submodule.

Daniel P. Berrange (4):
build: allow setting a custom GIT binary for transparent proxying
build: don't create temporary files in source dir
build: allow automatic git submodule updates to be disabled
build: don't fail if given a git submodule which does not exist

 Makefile                 | 15 ++++++++++--
 configure                | 14 +++++++++++-
 scripts/git-submodule.sh | 59 ++++++++++++++++++++++++++++++++++++++++--------
 3 files changed, 76 insertions(+), 12 deletions(-)