New script: cg-clean
Hello, Petr!
Please consider this script for Cogito.
#!/usr/bin/env bash
#
# Clean unknown files from the working tree.
# Copyright (c) Pavel Roskin, 2005
#
# Cleans file and directories that are not under version control. # Only regular files that are not ignored by cg-status are cleaned
# by default. # # OPTIONS # ------- # -i:: # Clean files ignored by cg-status, such as object files. # # -s:: # Clean symlinks, fifos, sockets and other special files. # # -r:: # Clean directories. # # -R:: # Clean directories, try harder. Make directories writeable # recursively before removing. # # -a:: # Clean all the above. # # If any other arguments are specified, they will be the only files # considered for removal. It will also imply the `-a' option.
USAGE="cg-clean [-i] [-s] [-r] [-R] [-a] [FILE]..."
. ${COGITO_LIB}cg-Xlib
cleanexclude=
cleanspecial=
cleandir=
cleandirhard=
while optparse; do
if optparse -i; then
cleanexclude=1
elif optparse -s; then
cleanspecial=1
elif optparse -r; then
cleandir=1
elif optparse -R; then
cleandirhard=1
elif optparse -a; then
cleanexclude=1
cleandirhard=1
else
optfail
fi
done
if [ "$ARGS" ]; then
cleanexclude=1
cleandirhard=1
fi
# Good candidate for cg-Xlib
# Put exclude options for git-ls-files to EXCLUDE
set_exclude() {
stdignores=('*.[ao]' '.*' tags '*~' '#*' ',,merge*')
for ign in "${stdignores[@]}"; do
EXCLUDE="$EXCLUDE --exclude=$ign"
done
EXCLUDEFILE=$_git/exclude
if [ -f "$EXCLUDEFILE" ]; then
EXCLUDE="$EXCLUDE --exclude-from=$EXCLUDEFILE"
fi
}EXCLUDE=
if [ -z "$cleanexclude" ]; then
set_exclude
fi
git-update-cache --refresh > /dev/null
do_clean() {
# FIXME - very suboptimal
if [ "$ARGS" ]; then
local found=
for arg in "${ARGS[@]}"; do
if [ "$arg" = "$file" ]; then
found=1
break
fi
done
[ "$found" ] || return
fi
if [ "$cleandirhard" ]; then
chmod -R 700 "$file"
rm -rf "$file"
return
fi
if [ "$cleandir" ]; then
rm -rf "$file"
return
fi
if [ "$cleanspecial" ]; then
[ -d "$file" ] && return
rm -f "$file"
return
fi
[ -f "$file" ] && rm -f "$file"
}# Need to use temporary file so that changing IFS doesn't affect $EXCLUDE
# expansion. filelist=$(mktemp -t gitlsfiles.XXXXXX) git-ls-files --others $EXCLUDE >"$filelist" IFS=$'\n' for file in $(cat "$filelist"); do do_clean "$file" done
rm -f "$filelist"
Signature · 2 lines
Regards, Pavel Roskin
Quoted 2 lines · In message <1120862084.17812.6.camel@dv> Pavel Roskin wrote:
Please consider this script for Cogito.
...
Quoted 4 lines
# OPTIONS
# -------
# -i::
# Clean files ignored by cg-status, such as object files.
May I suggest to give "-i" the standard "--interactive" meaning (= prompt before removal) like with "rm" etc. ?
Thanks.
Wolfgang Denk
Signature · 3 lines
Software Engineering: Embedded and Realtime Systems, Embedded Linux Phone: (+49)-8142-66989-10 Fax: (+49)-8142-66989-80 Email: [email protected] How come everyone's going so slow if it's called rush hour?
Dear diary, on Sat, Jul 09, 2005 at 12:34:44AM CEST, I got a letter where Pavel Roskin <[email protected]> told me that...
Quoted 1 line
Hello, Petr!
Hello,
Quoted 3 lines
Please consider this script for Cogito.
Signed-off-by Pavel Roskin <[email protected]>
the script is definitively interesting, but I have couple of notes about it first:
(i) -i sounds wrong for anything but being interactive here ;-) What about -A?
(ii) I'm confused - if -a is all of the above, how do I clean _only_ regular files, and only those not ignored by cg-status?
(iii) Makes it any sense to remove only special files?
(iv) -r implies being recursive, but it has nothing to do with that here.
(v) Semantically, I think it's quite close to cg-reset. What about making it part of cg-reset instead of a separate command? I tend to be careful about command inflation. (That's part of being careful about the usability in general.) That's just an idea and possibly a bad one, what do you think?
Thanks,
Signature · 3 lines
Petr "Pasky" Baudis Stuff: http://pasky.or.cz/ <Espy> be careful, some twit might quote you out of context..
Hello, Petr!
Sorry for delay.
Quoted 8 lines · Petr Baudis
Dear diary, on Sat, Jul 09, 2005 at 12:34:44AM CEST, I got a letter where Pavel Roskin <[email protected]> told me that...
Quoted 1 line
Hello, Petr!
Hello,
Quoted 2 lines
Please consider this script for Cogito.
Quoted 7 lines
Quoted 1 line
Signed-off-by Pavel Roskin <[email protected]>the script is definitively interesting, but I have couple of notes about it first:
(i) -i sounds wrong for anything but being interactive here ;-) What about -A?
I agree that -i could be confusing, but -A would seem to imply "All", so let it be -x from "exclude".
Quoted 2 lines
(ii) I'm confused - if -a is all of the above, how do I clean _only_ regular files, and only those not ignored by cg-status?
cg-clean without options. I'm changing the description to avoid confusion.
Quoted 1 line
(iii) Makes it any sense to remove only special files?
I thought it would make sense to have an option to remove them in addition to regular files, but now I think it's not worth the trouble to distinguish between them.
Quoted 2 lines
(iv) -r implies being recursive, but it has nothing to do with that here.
Renamed to -d. Other confusing options have been removed. "-a" is retired because it's not hard to type "-dx". Explicit arguments are not accepted - one can easily use "rm" instead. That should make cg-clean much simpler.
Quoted 5 lines
(v) Semantically, I think it's quite close to cg-reset. What about making it part of cg-reset instead of a separate command? I tend to be careful about command inflation. (That's part of being careful about the usability in general.) That's just an idea and possibly a bad one, what do you think?
I understand your concern, but cg-reset does other things. cg-reset changes the branch. cg-clean allows to start the build from scratch without changing the branch.
It's not uncommon for me to revert patches one-by-one looking for the patch that breaks something. I could make minor changes e.g for debugging or to fix breakage in certain revisions. I would revert such by cg-clean before going to another revision. cg-reset would be an overkill - it would move me to the latest release.
I can imagine that cg-reset would call cg-clean (optionally) to allow fresh start on the main branch. The opposite would be wrong.
Here's the simplified cg-clean script. Note that the "-d" option is not working with the current version of git of a bug in git-ls-files. I can work it around by scanning all directories in bash, but I think it's easier to fix git (remove "continue" before DT_REG in ls-files.c).
Processing of .gitignore was taken from cg-status, and I don't really understand it. But I think it's important to keep that code in sync. It could later go to cg-Xlib.
#!/usr/bin/env bash
#
# Clean unknown files from the working tree.
# Copyright (c) Pavel Roskin, 2005
#
# Cleans file and directories that are not under version control. # When run without arguments, files ignored by cg-status and directories
# are not removed. # # OPTIONS # ------- # -d:: # Also clean directories. # # -x:: # Also clean files ignored by cg-status, such as object files.
USAGE="cg-clean [-d] [-x]"
. ${COGITO_LIB}cg-Xlib
cleanexclude=
cleandir=
while optparse; do
if optparse -d; then
cleandir=1
elif optparse -x; then
cleanexclude=1
else
optfail
fi
done
# Good candidate for cg-Xlib
# Put exclude options for git-ls-files to EXCLUDE
set_exclude() {
EXCLUDE=
stdignores=('*.[ao]' '.*' tags '*~' '#*' ',,merge*')
for ign in "${stdignores[@]}"; do
EXCLUDE="$EXCLUDE --exclude=$ign"
done
EXCLUDEFILE=$_git/exclude
if [ -f "$EXCLUDEFILE" ]; then
EXCLUDE="$EXCLUDE --exclude-from=$EXCLUDEFILE"
fi
{
path="$_git_relpath"
dir=
[ -r .gitignore ] && EXCLUDE="$EXCLUDE --exclude-from=.gitignore"
while [[ "$path" == */* ]]; do
dir="${dir:-.}/${path%%/*}"
path="${path#*/}"
[ -r $dir/.gitignore ] && EXCLUDE="$EXCLUDE --exclude-from=$dir/.gitignore"
done
}
}if [ -z "$cleanexclude" ]; then
set_exclude
else
EXCLUDE=
fi
git-update-cache --refresh > /dev/null
# Need to use temporary file so that changing IFS doesn't affect $EXCLUDE
# expansion. filelist=$(mktemp -t gitlsfiles.XXXXXX) git-ls-files --others $EXCLUDE >"$filelist" save_IFS="$IFS" IFS=$'\n' for file in $(cat "$filelist"); do IFS="$save_IFS" if [ -d "$file" ]; then if [ "$cleandir" ]; then # Try really hard by changing permissions chmod -R 700 "$file" rm -rf "$file" fi return fi rm -f "$file" done
rm -f "$filelist"
Signature · 2 lines
Regards, Pavel Roskin
Dear diary, on Sat, Aug 06, 2005 at 09:14:03AM CEST, I got a letter where Pavel Roskin <[email protected]> told me that...
Quoted 1 line
Hello, Petr!
Hello,
Quoted 1 line
Sorry for delay.
no problem, and sorry for another delay on my side too. :-)
Quoted 11 lines · Petr Baudis
Quoted 5 lines
(v) Semantically, I think it's quite close to cg-reset. What about making it part of cg-reset instead of a separate command? I tend to be careful about command inflation. (That's part of being careful about the usability in general.) That's just an idea and possibly a bad one, what do you think?
I understand your concern, but cg-reset does other things. cg-reset changes the branch. cg-clean allows to start the build from scratch without changing the branch.
Quoted 8 lines
It's not uncommon for me to revert patches one-by-one looking for the patch that breaks something. I could make minor changes e.g for debugging or to fix breakage in certain revisions. I would revert such by cg-clean before going to another revision. cg-reset would be an overkill - it would move me to the latest release.
I can imagine that cg-reset would call cg-clean (optionally) to allow fresh start on the main branch. The opposite would be wrong.
Yes, that makes sense.
Quoted 4 lines
Here's the simplified cg-clean script. Note that the "-d" option is not working with the current version of git of a bug in git-ls-files. I can work it around by scanning all directories in bash, but I think it's easier to fix git (remove "continue" before
DT_REGinls-files.c).
Is that fixed already?
Quoted 3 lines
Processing of .gitignore was taken from cg-status, and I don't really understand it. But I think it's important to keep that code in sync. It could later go to cg-Xlib.
It became much simpler a short while later, thankfully. Judging from your another patch, I suppose you are going to update this script accordingly?
Quoted 9 lines
Signed-off-by Pavel Roskin <[email protected]>#!/usr/bin/env bash
#
# Clean unknown files from the working tree.
# Copyright (c) Pavel Roskin, 2005
#
# Cleans file and directories that are not under version control. # When run without arguments, files ignored by cg-status and directories
Quoted 6 lines
# are not removed. # # OPTIONS # ------- # -d:: # Also clean directories.
Perhaps add "and their contents" to prevent bad surprises.
Quoted 22 lines
#
# -x::
# Also clean files ignored by cg-status, such as object files.USAGE="cg-clean [-d] [-x]"
. ${COGITO_LIB}cg-Xlib
cleanexclude=
cleandir=
while optparse; do
if optparse -d; then
cleandir=1
elif optparse -x; then
cleanexclude=1
else
optfail
fi
done# Good candidate for cg-Xlib
# Put exclude options for git-ls-files to EXCLUDE
..snip..
Quoted 10 lines
git-update-cache --refresh >
/dev/null# Need to use temporary file so that changing IFS doesn't affect $EXCLUDE
# expansion. filelist=$(mktemp -t gitlsfiles.XXXXXX) git-ls-files --others $EXCLUDE >"$filelist" save_IFS="$IFS" IFS=$'\n' for file in $(cat "$filelist"); do
Why can't you use git-ls-files | IFS=$'\n' while ?
Quoted 7 lines
IFS="$save_IFS" if [ -d "$file" ]; then if [ "$cleandir" ]; then # Try really hard by changing permissions chmod -R 700 "$file" rm -rf "$file" fi
An error message would be in order otherwise, I guess.
Quoted 1 line
return
I suppose you mean continue? I'm not really sure what does return do here, if it jumps out of the do block or what, and continue is nicely explicit. :-)
Quoted 5 lines
fi rm -f "$file"done
rm -f "$filelist"
Signature · 4 lines
Petr "Pasky" Baudis Stuff: http://pasky.or.cz/ If you want the holes in your knowledge showing up try teaching someone. -- Alan Cox
Hi, Petr!
Unfortunately, my latest revision of cg-clean has "committed suicide" just when I was about to post it. Anyway, I would prefer to wait until you apply my patch to cg-status to ignore all ignores. That would allow me to reuse cg-status.
Quoted 6 lines · Petr Baudis
Quoted 4 lines
Here's the simplified cg-clean script. Note that the "-d" option is not working with the current version of git of a bug in git-ls-files. I can work it around by scanning all directories in bash, but I think it's easier to fix git (remove "continue" before
DT_REGinls-files.c).Is that fixed already?
It turn out it's quite tricky. git-ls-files doesn't distinguish between known and unknown directories. One way to do it would be to check all cached files and find all directories they are in. Another approach would involve "git-ls-tree -r", but I don't think it would be right because we work with cache and current files, not with committed data (but my knowledge is limited to make a call - I still need to read the documentation about git).
What I did was following:
"git-ls-files --cached" is run, directories are extracted, sorted but "sort -u" and put to a temporary file dirlist1. "find -type d" is run, ".git" is filtered out, the list is merged with dirlist1, sorted by "sort -u", and put to dirlist2. dirlist1 and dirlist2 are compared by diff, the entries in dirlist2 are unknown directories. (That's the kind of task where Perl hashes would be great). A technical detail - both lists are limited to $_git_relpath if it's defined.
Quoted 7 lines
Quoted 3 lines
Processing of .gitignore was taken from cg-status, and I don't really understand it. But I think it's important to keep that code in sync. It could later go to cg-Xlib.
It became much simpler a short while later, thankfully. Judging from your another patch, I suppose you are going to update this script accordingly?
Yes, I'm going to use "cg-status -w" for files.
Quoted 4 lines
Quoted 2 lines
# -d::
# Also clean directories.Perhaps add "and their contents" to prevent bad surprises.
Too late :-)
Quoted 7 lines
Quoted 5 lines
filelist=$(mktemp -t gitlsfiles.XXXXXX)
git-ls-files --others $EXCLUDE >"$filelist"
save_IFS="$IFS"
IFS=$'\n'
for file in $(cat "$filelist"); doWhy can't you use git-ls-files | IFS=$'\n' while ?
Good idea.
Quoted 9 lines
Quoted 7 lines
IFS="$save_IFS" if [ -d "$file" ]; then if [ "$cleandir" ]; then # Try really hard by changing permissions chmod -R 700 "$file" rm -rf "$file" fiAn error message would be in order otherwise, I guess.
You mean, list directories if "-d" is not specified? Good idea. Also, the latest revision included the "-v" option for "verbose" - it would show everything that gets deleted.
Quoted 5 lines
Quoted 1 line
return
I suppose you mean continue? I'm not really sure what does return do here, if it jumps out of the do block or what, and continue is nicely explicit. :-)
My error, it was fixed soon after I posted the script.
Signature · 2 lines
Regards, Pavel Roskin
Dear diary, on Fri, Aug 12, 2005 at 02:54:13AM CEST, I got a letter where Pavel Roskin <[email protected]> told me that...
Quoted 1 line
Hi, Petr!
Hi,
Quoted 4 lines
Unfortunately, my latest revision of cg-clean has "committed suicide" just when I was about to post it. Anyway, I would prefer to wait until you apply my patch to cg-status to ignore all ignores. That would allow me to reuse cg-status.
well, I did quite a while ago. Unless the kernel.org mirroring system broke, it should be already public.
Quoted 10 lines · Petr Baudis
Quoted 6 lines
Quoted 4 lines
Here's the simplified cg-clean script. Note that the "-d" option is not working with the current version of git of a bug in git-ls-files. I can work it around by scanning all directories in bash, but I think it's easier to fix git (remove "continue" before
DT_REGinls-files.c).Is that fixed already?
It turn out it's quite tricky. git-ls-files doesn't distinguish between known and unknown directories.
In the long term, I would prefer to have directory information in the index file - to make this kind of tasks easier, allow juggling with empty directories etc.
Quoted 6 lines
One way to do it would be to check all
cached files and find all directories they are in. Another approach would involve "git-ls-tree -r", but I don't think it would be right because we work with cache and current files, not with committed data (but my knowledge is limited to make a call - I still need to read the documentation about git).
Yes, we should certainly follow the index, otherwise you will e.g. lose the files added by cg-add but not committed yet.
Signature · 4 lines
Petr "Pasky" Baudis Stuff: http://pasky.or.cz/ If you want the holes in your knowledge showing up try teaching someone. -- Alan Cox
Quoted 6 lines · Petr Baudis
Dear diary, on Fri, Aug 12, 2005 at 02:54:13AM CEST, I got a letter where Pavel Roskin <[email protected]> told me that...
Quoted 1 line
Hi, Petr!
Hi,
Quoted 7 lines
Quoted 4 lines
Unfortunately, my latest revision of cg-clean has "committed suicide" just when I was about to post it. Anyway, I would prefer to wait until you apply my patch to cg-status to ignore all ignores. That would allow me to reuse cg-status.
well, I did quite a while ago. Unless the kernel.org mirroring system broke, it should be already public.
Yes, it's there. Thank you.
This is another attempt at cg-clean. Verbose mode is now default (i.e. everything that's removed is reported), "-q" makes the script quiet. I've reinstated the "hard" option, it's now "-D".
#!/usr/bin/env bash
#
# Clean unknown files from the working tree.
# Copyright (c) Pavel Roskin, 2005
#
# Cleans file and directories that are not under version control. # When run without arguments, files ignored by cg-status and directories
# are not removed. # # OPTIONS # ------- # -d:: # Also clean directories and their contents. # # -D:: # Same as -d but try harder (change permissions first). # # -q:: # Quiet - don't report what's being cleaned. # # -x:: # Also clean files ignored by cg-status, such as object files.
USAGE="cg-clean [-d] [-D] [-q] [-x]"
. ${COGITO_LIB}cg-Xlib || exit 1
noexclude=
cleandir=
cleandirhard=
quiet=
while optparse; do
if optparse -d; then
cleandir=1
elif optparse -D; then
cleandir=1
cleandirhard=1
elif optparse -q; then
quiet=1
elif optparse -x; then
noexclude=1
else
optfail
fi
done
[ "$ARGS" ] && usage
clean_dirs()
{
dirlist=$(mktemp -t gitlsfiles.XXXXXX)
git-ls-files --cached | sed -n 's|^'"$_git_relpath"'||p' |
sed -n 's|/[^/]*$||p' | sort -u >"$dirlist"
save_IFS="$IFS"
IFS=$'\n'
fpath=${_git_relpath-./}
find "$fpath" -type d -print | \
sed 's|^'"$fpath"'||;/^$/d;/^\.git$/d;/^\.git\//d' |
cat - "$dirlist" | sort -u | diff - "$dirlist" |
sed -n 's/< //p' |
for file in $(cat); do
path="${_git_relpath}$file"
if [ -z "$cleandir" ]; then
echo "Not removing $file/"
continue
fi
if [ ! -d "$path" ]; then
# Perhaps directory was removed with its parent
continue
fi
[ "$quiet" ] || echo "Removing $file/"
if [ "$cleandirhard" ]; then
chmod -R 700 "$path"
fi
rm -rf "$path"
if [ -e "$path" -o -L "$path" ]; then
echo "Cannot remove $file/"
fi
done
IFS="$save_IFS"
rm -f "$dirlist"
}clean_files()
{
xopt=
if [ "$noexclude" ]; then
xopt="-x"
fi
save_IFS="$IFS"
IFS=$'\n'
cg-status "$xopt" -w | sed -n 's/^? //p' |
for file in $(cat); do
path="${_git_relpath}$file"
if [ -d "$path" ]; then
# Sanity check, shouldn't happen
echo "FATAL: cg-status reports directories" >&2
exit 1
elif [ -e "$path" -o -L "$path" ]; then
[ "$quiet" ] || echo "Removing $file"
rm -f "$path"
if [ -e "$path" -o -L "$path" ]; then
echo "Cannot remove $file"
fi
else
echo "File $file has disappeared!"
fi
done
IFS="$save_IFS"
}# Even if -d or -D is not specified, we want to tell user about
# directories that are not removed if [ -z "$quiet" -o "$cleandir" ]; then clean_dirs fi
clean_files
Signature · 2 lines
Regards, Pavel Roskin