Skip to contentSkip to conversation

git-grep: support --no-external-grep

PATCH In review Nguyễn Thái Ngọc Duy 8 participants 23 messages Jun 15, 2016

lore
Share & tools
lore
#0 lore raw

PATCH 2/12 git-grep: support --no-external-grep

this one is mainly for testing builtin grep

---
 builtin-grep.c |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)
builtin-grep.c +7 -1
index cc9ba6b..ee96d01 100644
@@ -20,6 +20,8 @@
 #endif
 #endif
 
+static int no_external_grep;
+
 /*
  * git grep pathspecs are somewhat different from diff-tree pathspecs;
  * pathname wildcards are allowed.
@@ -386,7 +388,7 @@ static int grep_cache(struct grep_opt *opt, const char **paths, int cached)
 	 * we grep through the checked-out files. It tends to
 	 * be a lot more optimized
 	 */
-	if (!cached) {
+	if (!cached && !no_external_grep) {
 		hit = external_grep(opt, paths, cached);
 		if (hit >= 0)
 			return hit;
@@ -544,6 +546,10 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 			cached = 1;
 			continue;
 		}
+		if (!strcmp("--no-external-grep", arg)) {
+			no_external_grep = 1;
+			continue;
+		}
 		if (!strcmp("-a", arg) ||
 		    !strcmp("--text", arg)) {
 			opt.binary = GREP_BINARY_TEXT;
Signature · 1 line
1.5.5.GIT
#3 lore raw

PATCH 3/12 Introduce sparse prefix

Sparse prefix is actually a list of prefixes separated by colons, that will limit worktree usage within it.

This patch adds core.sparsecheckout and
"git rev-parse --show-sparse-prefix". This also adds manipulation functions that will get used later.

---
 builtin-rev-parse.c |    4 +
 cache.h             |   13 +++
 config.c            |    7 ++
 environment.c       |  221 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 245 insertions(+), 0 deletions(-)
builtin-rev-parse.c +4 -0
index aa71f4a..4200f14 100644
@@ -499,6 +499,10 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix)
 					puts(prefix);
 				continue;
 			}
+			if (!strcmp(arg, "--show-sparse-prefix")) {
+				puts(get_sparse_prefix_str());
+				continue;
+			}
 			if (!strcmp(arg, "--show-cdup")) {
 				const char *pfx = prefix;
 				if (!is_inside_work_tree()) {
cache.h +13 -0
index 38985aa..4687096 100644
@@ -319,6 +319,19 @@ extern const char *get_git_work_tree(void);
 extern const char *read_gitfile_gently(const char *path);
 extern void set_git_work_tree(const char *tree);
 
+extern int sparse_checkout_enabled();
+extern char **get_sparse_prefix(void);
+extern const char *get_sparse_prefix_str(void);
+extern char **save_sparse_prefix();
+extern char **restore_sparse_prefix(char **prefix);
+extern int outside_sparse_prefix(const char *prefix);
+extern int index_outside_sparse_prefix(const char *prefix);
+extern void set_sparse_prefix(const char *prefix);
+extern char **split_prefix(const char *env);
+extern char **combine_prefix_list(char **p1, char **p2);
+extern void free_prefix_list(char **prefix_list);
+extern int outside_prefix_list(char **iprefix, const char *prefix);
+
 #define ALTERNATE_DB_ENVIRONMENT "GIT_ALTERNATE_OBJECT_DIRECTORIES"
 
 extern const char **get_pathspec(const char *prefix, const char **pathspec);
config.c +7 -0
index 1e066c7..e7b457b 100644
@@ -467,6 +467,13 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.sparsecheckout")) {
+		if (!value)
+			return config_error_nonbool(var);
+		set_sparse_prefix(value);
+		return 0;
+	}
+
 	/* Add other config variables here and to Documentation/config.txt. */
 	return 0;
 }
environment.c +221 -0

236 lines not on this page · raw message

Signature · 1 line
1.5.5.GIT
#4 lore raw

PATCH 8/12 git-clone: support --path to do sparse clone

---
 builtin-clone.c |   13 +++++++++++++
 1 files changed, 13 insertions(+), 0 deletions(-)
builtin-clone.c +13 -0
index 3522245..229f2e2 100644
@@ -36,6 +36,7 @@ static const char * const builtin_clone_usage[] = {
 static int option_quiet, option_no_checkout, option_bare;
 static int option_local, option_no_hardlinks, option_shared;
 static char *option_template, *option_reference, *option_depth;
+static char *option_sparse_prefix;
 static char *option_origin = NULL;
 static char *option_upload_pack = "git-upload-pack";
 
@@ -43,6 +44,8 @@ static struct option builtin_clone_options[] = {
 	OPT__QUIET(&option_quiet),
 	OPT_BOOLEAN('n', "no-checkout", &option_no_checkout,
 		    "don't create a checkout"),
+	OPT_STRING(0, "path", &option_sparse_prefix, "prefixes",
+		    "limit checkout to specified paths (sparse checkout)"),
 	OPT_BOOLEAN(0, "bare", &option_bare, "create a bare repository"),
 	OPT_BOOLEAN(0, "naked", &option_bare, "create a bare repository"),
 	OPT_BOOLEAN('l', "local", &option_local,
@@ -364,9 +367,14 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 			die("--bare and --origin %s options are incompatible.",
 			    option_origin);
 		option_no_checkout = 1;
+		if (option_sparse_prefix)
+			die("--bare and --path options are incompatible.");
 		use_separate_remote = 0;
 	}
 
+	if (option_no_checkout && option_sparse_prefix)
+		die("--no-checkout and --path options are incompatible.");
+
 	if (!option_origin)
 		option_origin = "origin";
 
@@ -549,6 +557,11 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 		/* We need to be in the new work tree for the checkout */
 		setup_work_tree();
 
+		if (option_sparse_prefix) {
+			git_config_set("core.sparsecheckout", option_sparse_prefix);
+			set_sparse_prefix(option_sparse_prefix);
+		}
+
 		fd = hold_locked_index(lock_file, 1);
 
 		memset(&opts, 0, sizeof opts);
Signature · 1 line
1.5.5.GIT
#5 lore raw

Re: [PATCH 08/12] git-clone: support --path to do sparse clone

builtin-clone.c int cmd_clone(int argc, const char **argv, const char *prefix) · line 564

Quoted 5 lines · Nguyễn Thái Ngọc Duy
builtin-clone.c +5 -0
+		if (option_sparse_prefix) {
+			git_config_set("core.sparsecheckout", option_sparse_prefix);
+			set_sparse_prefix(option_sparse_prefix);
+		}
+

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

So instead I would suggest that this be mentioned in the documentation for --path, but there doesn't seem to be any.

-Peff

#6 lore raw

sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

Hi,

Quoted 4 lines · Jeff King

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

I think this is not even worth of a series. Sure, it would have benefits for those who want sparse checkouts. But it comes for a high price on everyone else:

- security issues (you'd need to open the git protocol to give you something else than a ref, _including_ refs that were deleted)

- performance issues (the server would have to do a lot more, faking commits, or in the alternative serving a gazillion more sessions if the client does the reconstruction)

... and I am sure there are tons more issues.

Ciao,
Dscho

#7 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

Quoted 8 lines · Johannes Schindelin
Quoted 4 lines

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

I think this is not even worth of a series. Sure, it would have benefits for those who want sparse checkouts. But it comes for a high price on everyone else:

I agree there are a lot of issues. I am just thinking of the person who said they had a >100G repository. But I am also not volunteering to do it, so I will let somebody who really cares about it try to defend the idea.

-Peff

#13 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Johannes Schindelin

Hi,

Quoted 11 lines · Johannes Schindelin
Quoted 4 lines · Jeff King

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

I think this is not even worth of a series. Sure, it would have benefits for those who want sparse checkouts. But it comes for a high price on everyone else:

Quoted 6 lines

- security issues (you'd need to open the git protocol to give you something else than a ref, _including_ refs that were deleted)

- performance issues (the server would have to do a lot more, faking commits, or in the alternative serving a gazillion more sessions if the client does the reconstruction)

I don't follow how these two issues arise, if the server will do the
pruning for you. It will just skip entering some tree objects when doing object traversal; why opening the git protocol or faking commits? This would be a simple extra capability in the protocol.

One question is what to do with delta chains including unwanted
objects, but I think that given the objects' associativity for delta chains, this shouldn't be huge practical issues and it could be affordable in principle to include even unwanted objects.

Quoted 1 line

... and I am sure there are tons more issues.

I do agree on this. :-)

Signature · 3 lines
				Petr "Pasky" Baudis
As in certain cults it is possible to kill a process if you know
its true name.  -- Ken Thompson and Dennis M. Ritchie
#14 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

Quoted 4 lines · Petr Baudis

I don't follow how these two issues arise, if the server will do the
pruning for you. It will just skip entering some tree objects when doing object traversal; why opening the git protocol or faking commits? This would be a simple extra capability in the protocol.

Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a lot of overhead there I reckon.

Quoted 4 lines

One question is what to do with delta chains including unwanted objects, but I think that given the objects' associativity for delta chains, this shouldn't be huge practical issues and it could be affordable in principle to include even unwanted objects.

Just keep them? What we're doing here is trying to optimize in the case that someone has a sparse checkout, nothing bad will happen if they get too many info surely? (Save for them not getting as much improvement as would have been possible would the pack have been created differently.)

Signature · 3 lines
Cheers,

Sverre Rabbelier
#18 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Johannes Schindelin
Quoted 13 lines · Johannes Schindelin

Hi,

Quoted 4 lines · Jeff King

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

I think this is not even worth of a series. Sure, it would have benefits for those who want sparse checkouts. But it comes for a high price on everyone else:

Quoted 8 lines

- security issues (you'd need to open the git protocol to give you something else than a ref, _including_ refs that were deleted)

- performance issues (the server would have to do a lot more, faking commits, or in the alternative serving a gazillion more sessions if the client does the reconstruction)

... and I am sure there are tons more issues.

Widen checkout won't work and probably more.

Signature · 1 line
Duy
#19 lore raw

Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Jeff King
Quoted 9 lines · Jeff King

builtin-clone.c int cmd_clone(int argc, const char **argv, const char *prefix) · line 564

Quoted 5 lines · Nguyen Thai Ngoc Duy
builtin-clone.c +5 -0
+             if (option_sparse_prefix) {
+                     git_config_set("core.sparsecheckout", option_sparse_prefix);
+                     set_sparse_prefix(option_sparse_prefix);
+             }
+
Quoted 7 lines

As a user, I would expect "sparse clone" to also be sparse on the
fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

So instead I would suggest that this be mentioned in the documentation for --path, but there doesn't seem to be any.

Thanks. Will mention it when I write documentation for this.

Signature · 1 line
Duy
#20 lore raw

PATCH 10/12 git-checkout: support --full and --path to manipulate sparse checkout

"git checkout --full" will quit sparse checkout mode while "git checkout --sparse=prefix" will turn on sparse checkout or update sparse prefix.

twoway_merge has been updated to deal with sparse checkout update. Files that are in old sparse prefix only will get removed while files that are in new sparse prefix only will get added.

---
 builtin-checkout.c |   44 +++++++++++++++++++
 cache.h            |    2 +
 unpack-trees.c     |  120 ++++++++++++++++++++++++++++++++++++++++++++++++----
 unpack-trees.h     |    3 +-
 4 files changed, 160 insertions(+), 9 deletions(-)
builtin-checkout.c +44 -0
index eec1bde..410a53a 100644
@@ -160,6 +160,9 @@ struct checkout_opts {
 	char *new_branch;
 	int new_branch_log;
 	enum branch_track track;
+
+	char *sparse_prefix;
+	int no_sparse_checkout;
 };
 
 static int reset_tree(struct tree *tree, struct checkout_opts *o, int worktree)
@@ -255,7 +258,25 @@ static int merge_working_tree(struct checkout_opts *opts,
 		tree = parse_tree_indirect(new->commit->object.sha1);
 		init_tree_desc(&trees[1], tree->buffer, tree->size);
 
+		if (opts->no_sparse_checkout)
+			topts.new_sparse_prefix = 1;
+
+		if (opts->sparse_prefix) {
+			char **new_sparse_prefix = split_prefix(opts->sparse_prefix);
+
+			if (!new_sparse_prefix) {
+				error("new sparse prefix invalid");
+				return 1;
+			}
+			topts.new_sparse_prefix = 1;
+			topts.unpack_data = new_sparse_prefix;
+		}
+
 		ret = unpack_trees(2, trees, &topts);
+
+		if (topts.new_sparse_prefix && topts.unpack_data)
+			free_prefix_list((char **)topts.unpack_data);
+
 		if (ret == -1) {
 			/*
 			 * Unpack couldn't do a trivial merge; either
@@ -299,6 +320,14 @@ static int merge_working_tree(struct checkout_opts *opts,
 	    commit_locked_index(lock_file))
 		die("unable to write new index file");
 
+	/*
+	 * update sparse checkout accordingly
+	 * show_local_changes will need it
+	 */
+	if (opts->no_sparse_checkout)
+		free_prefix_list(save_sparse_prefix());
+	if (opts->sparse_prefix)
+		set_sparse_prefix(opts->sparse_prefix);
 	if (!opts->force)
 		show_local_changes(&new->commit->object);
 
@@ -409,6 +438,13 @@ static int switch_branches(struct checkout_opts *opts, struct branch_info *new)
 
 	update_refs_for_switch(opts, &old, new);
 
+	/* now save new sparse prefix */
+	if (opts->no_sparse_checkout)
+		git_config_set("core.sparsecheckout", NULL);
+
+	if (opts->sparse_prefix)
+		git_config_set("core.sparsecheckout", opts->sparse_prefix);
+
 	ret = post_checkout_hook(old.commit, new->commit, 1);
 	return ret || opts->writeout_error;
 }
@@ -428,6 +464,8 @@ int cmd_checkout(int argc, const char **argv, const char *prefix)
 			BRANCH_TRACK_EXPLICIT),
 		OPT_BOOLEAN('f', NULL, &opts.force, "force"),
 		OPT_BOOLEAN('m', NULL, &opts.merge, "merge"),
+		OPT_STRING(0, "path", &opts.sparse_prefix, "prefixes", "update sparse checkout"),
+		OPT_BOOLEAN(0, "full", &opts.no_sparse_checkout, "quit sparse checkout"),
 		OPT_END(),
 	};
 
@@ -468,6 +506,9 @@ int cmd_checkout(int argc, const char **argv, const char *prefix)
 	if (!opts.new_branch && (opts.track != git_branch_track))
 		die("git checkout: --track and --no-track require -b");
 
+	if (opts.no_sparse_checkout && opts.sparse_prefix)
+		die("git checkout: --path and --full are incompatible");
+
 	if (opts.force && opts.merge)
 		die("git checkout: -f and -m are incompatible");
 
@@ -486,6 +527,9 @@ int cmd_checkout(int argc, const char **argv, const char *prefix)
 			}
 		}
 
+		if (opts.no_sparse_checkout || opts.sparse_prefix)
+			die("git checkout: updating paths is incompatible with setting sparse checkout");
+
 		return checkout_paths(source_tree, pathspec);
 	}
 
cache.h +2 -0
index b9a1d96..9e7f146 100644
@@ -138,6 +138,8 @@ struct cache_entry {
 #define CE_HASHED    (0x100000)
 #define CE_UNHASHED  (0x200000)
 
+#define CE_WD_REMOVE (0x400000)
+
 /*
  * Copy the sha1 and stat state of a cache entry from one to
  * another. But we never change the name, or the hash state!
unpack-trees.c +112 -8

205 lines not on this page · raw message

unpack-trees.h +2 -1
index a1b46f9..5a29fc4 100644
@@ -27,7 +27,8 @@ struct unpack_trees_options {
 		     aggressive:1,
 		     skip_unmerged:1,
 		     gently:1,
-		     check_index_prefix:1;
+		     check_index_prefix:1,
+		     new_sparse_prefix:1; /* unpack_data must contain new prefix */
 	const char *prefix;
 	int pos;
 	struct dir_struct *dir;
Signature · 1 line
1.5.5.GIT
#21 lore raw

Hi,

the patches 2, 3, 8 and 10 of your series introduce config options,
commandline switches and environment variables that are not documented anywhere - can you please include documentation updates in your patch series as well? It might be worth it to find some suitable place in the documentation where to explain the concept itself; it took me quite sometime before I actually realized (I think) what are your patches about. (Which is also why is it useful to write cover letters for patch series.)

Signature · 3 lines
				Petr "Pasky" Baudis
As in certain cults it is possible to kill a process if you know
its true name.  -- Ken Thompson and Dennis M. Ritchie
#22 lore raw

Hi!

Quoted 2 lines · Petr Baudis

(Which is also why is it useful to write cover letters for patch series.)

sorry, I did not notice the cover letter, already separated from the
rest by a sprouting thread. Which is also why it is useful to thread up all the patches. ;-)

Petr "Pasky" Baudis

#1 lore raw
In reply to Nguyễn Thái Ngọc Duy

Nguyễn Thái Ngọc Duy, Wed, Jul 23, 2008 16:55:49 +0200:

builtin-grep.c line 24

Quoted 3 lines
builtin-grep.c +2 -0
+static int no_external_grep;
+

If you have used direct logic instead of negative, your patch would have been simplier and the code would be easier to understand. I.e.:

+static int external_grep = 1;

builtin-grep.c static int grep_cache(struct grep_opt *opt, const char **paths, int cached) · line 391

Quoted 6 lines
builtin-grep.c +1 -1
@@ -386,7 +388,7 @@ static int grep_cache(struct grep_opt *opt, const char **paths, int cached)
 	 * we grep through the checked-out files. It tends to
 	 * be a lot more optimized
 	 */
-	if (!cached) {
+	if (!cached && !no_external_grep) {

+ if (!cached && external_grep) {

builtin-grep.c int cmd_grep(int argc, const char **argv, const char *prefix) · line 552

Quoted 8 lines
builtin-grep.c +4 -0
@@ -544,6 +546,10 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 			cached = 1;
 			continue;
 		}
+		if (!strcmp("--no-external-grep", arg)) {
+			no_external_grep = 1;
+			continue;
+		}

+ if (!strcmp("--no-external-grep", arg)) {
+ external_grep = 0;
+ continue;
+ }

BTW, how about a config option for this? And maybe --external-grep={yes|no}

#2 lore raw
Quoted 9 lines · Alex Riesen

Nguyen Thai Ngoc Duy, Wed, Jul 23, 2008 16:55:49 +0200:

builtin-grep.c line 24

Quoted 3 lines
builtin-grep.c +2 -0
+static int no_external_grep;
+

If you have used direct logic instead of negative, your patch would
have been simplier and the code would be easier to understand. I.e.:

Point taken.

Quoted 6 lines

+ if (!strcmp("--no-external-grep", arg)) {
+ external_grep = 0;
+ continue;
+ }

BTW, how about a config option for this? And maybe --external-grep={yes|no}

I don't need --external-grep=yes now, but that would be good for testing external grep. About config option, is it really needed?

Signature · 1 line
Duy
#8 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Jeff King

Hi,

Quoted 8 lines · Jeff King
Quoted 6 lines · Johannes Schindelin
Quoted 5 lines

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

Quoted 8 lines
Quoted 3 lines

I think this is not even worth of a series. Sure, it would have benefits for those who want sparse checkouts. But it comes for a high price on everyone else:

I agree there are a lot of issues. I am just thinking of the person who said they had a >100G repository. But I am also not volunteering to do it, so I will let somebody who really cares about it try to defend the idea.

I never said that there were no benefits. I argued that there are too many _downsides_ to those who _don't_ benefit.

Ciao,
Dscho

#9 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Jeff King
Quoted 4 lines · Jeff King <peff <at> peff.net> writes:

I agree there are a lot of issues. I am just thinking of the person who said they had a >100G repository. But I am also not volunteering to do it, so I will let somebody who really cares about it try to defend the idea.

If you're referring to me (I mentioned a 144G CVS repo), then let me clarify a couple of things:

1. Probably more than 50% of the 144G is crud that should never have been checked in, but I have some undisciplined coworkers who like to blindly check in everything in their work trees. If/when we moved to git, I would get rid of all that crud. I'm also thinking about throwing out a lot of the history, since those same undisciplined coworkers like to use empty and/or useless log messages, so a lot of the history isn't very valuable anyways.

2. Git of course will store the remaining ~70G much more efficiently than CVS. I think git will be especially better than CVS for this repo, because it contains many instances of the same file(s) being checked in in multiple directories.

I expect the git repo size to be less than 7G. In addition, all our work is done on site on nfs, so we can use clone -s to avoid copying the whole 7G.

To sum it up, sparse cloning would not be important to me.

James

#10 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path?to do sparse clone

Quoted 4 lines · James Pickens

If you're referring to me (I mentioned a 144G CVS repo), then let me clarify a couple of things:
[...]
To sum it up, sparse cloning would not be important to me.

I was referring to you, so thanks for the clarification.

-Peff

#15 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Sverre Rabbelier

Hi,

Quoted 8 lines · Sverre Rabbelier
Quoted 4 lines · Petr Baudis

I don't follow how these two issues arise, if the server will do the
pruning for you. It will just skip entering some tree objects when doing object traversal; why opening the git protocol or faking commits? This would be a simple extra capability in the protocol.

Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a lot of overhead there I reckon.

So the server would _not_ have to deflate the objects to inspect them? I thought you knew more about Git's object database.

Quoted 6 lines
Quoted 4 lines

One question is what to do with delta chains including unwanted objects, but I think that given the objects' associativity for delta chains, this shouldn't be huge practical issues and it could be affordable in principle to include even unwanted objects.

Just keep them?

You'd still have to inspect the objects, which is way more work than the current code has to do. Remember: in the optimal case, upload-pack does not more than just serve the existing deltas/base objects.

Ciao,
Dscho

#16 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

Hi,

Quoted 13 lines · Johannes Schindelin
Quoted 8 lines · Sverre Rabbelier
Quoted 4 lines · Petr Baudis

I don't follow how these two issues arise, if the server will do the
pruning for you. It will just skip entering some tree objects when doing object traversal; why opening the git protocol or faking commits? This would be a simple extra capability in the protocol.

Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a lot of overhead there I reckon.

So the server would _not_ have to deflate the objects to inspect them? I thought you knew more about Git's object database.

..snip..

Quoted 3 lines

You'd still have to inspect the objects, which is way more work than the current code has to do. Remember: in the optimal case, upload-pack does not more than just serve the existing deltas/base objects.

then right now, exactly how does the server decide that the blob
7a7ff130 should be served along git.git HEAD? I still see upload-pack.c calling traverse_commit_list() that does process_tree() on every tree, etc. But the code is not straightforward, maybe I'm missing some shortcut?

Signature · 3 lines
				Petr "Pasky" Baudis
As in certain cults it is possible to kill a process if you know
its true name.  -- Ken Thompson and Dennis M. Ritchie
#11 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Jeff King
Quoted 7 lines · Jeff King
Quoted 5 lines · Johannes Schindelin
Quoted 4 lines

As a user, I would expect "sparse clone" to also be sparse on the fetching. That is, to not even bother fetching tree objects that we are not going to check out. But that is a whole other can of worms from local sparseness, so I think it is worth saving for a different series.

Quoted 8 lines
Quoted 3 lines

I think this is not even worth of a series. Sure, it would have benefits for those who want sparse checkouts. But it comes for a high price on everyone else:

I agree there are a lot of issues. I am just thinking of the person who said they had a >100G repository. But I am also not volunteering to do it, so I will let somebody who really cares about it try to defend the idea.

I think sparse fetch is a lot worse than grafts and shallow clones which are already bad. These are all ways to introduce local inconsistency at the object level and pretend everything is Ok, but the latter two do so only at commit boundary and it is somewhat more manageable (but we still do not handle it very well). With sparse fetch, you cannot even guarantee the integrity of individual commits with subtrees here and there missing.

I do think shallow checkout that says "I'll have the whole tree in the index but the work tree will have only these paths checked out" makes sense. You do not need a fully populated work tree to create commits or merges -- the only absolute minimum you need is a fully populated index.

In that sense, I think "protect index entries outside of these paths" (I remember that the first round of this series was done around that notion) is a wrong mentality to handle this. We should think of this as more like "you still populate the index with the whole tree, and you are free to update them in any way you want, but we do not touch work tree outside these areas".

This has a few ramifications:

- If the user can somehow check out a path outside the "sparse" area, it is perfectly fine for the user to edit and "git add" it. Such a method to check out a path outside the "sparse" area is a way to widen the "sparse" area the user originally set up;

- When the user runs "merge", and it needs to present the user a working tree file because of conflicts at the file level, the user has to agree to widen the "sparse" area before being able to do so. One way to do this is to refuse and fail the merge (and then the user needs to do that "unspecified way" of widening the "sparse" area first). Another way would be to automatically widen the "sparse" area to include such conflicting paths.

- And you would want to narrow it down after you do such a widening.

For many projects that has src/ and doc/ (git.git being one of them), it is perfectly valid for a code person and a doc person to work in tandem. In such a project, after the code person makes changes in her sparsely checked out repository and making changes only to the src/ area and pushes the results out, the doc person would run "git pull && git log -p ORIG_HEAD" and updates the documentation in his sparsely checked out repository that has only doc/ area. The two parts are tied together and they advance more or less in sync. I think sparse checkout would be a useful feature to help such a configuration.

Having said that, I however think that this can easily be misused as a CVS style "one CVSROOT houses millions of totally unrelated projects" layout. In CVS, the layout is perfectly fine because the system does not track changes at anything higher than the level of individual files, but when you naïvely map the layout to a system with tree-wide atomic commits, such as git, it will defeat the whole point of using such a system. The pace these millions of unrelated projects advance do not have any relationship with each other, but by tying them together in the same top-level tree, the layout is introducing an unnecessary ordering between their commits.

#12 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

Quoted 1 line · Junio C Hamano

For many projects that has src/ and doc/ (git.git being one of them), it

We are? That's great, that'd mean I can actually do a 'ls' in the git.git root! Oh wait...

$ ls | wc -l
693
Signature · 3 lines
Cheers,

Sverre Rabbelier
#17 lore raw

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

In reply to Johannes Schindelin
Quoted 6 lines · Johannes Schindelin
Quoted 2 lines · Sverre Rabbelier

Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a lot of overhead there I reckon.

So the server would _not_ have to deflate the objects to inspect them? I thought you knew more about Git's object database.

Nope, I did not know this. I thought that the server already had to do all that to decide what to send, since not every request asks for the same pack (someone might not have updated in 2 month, or someone might might be up to date, or anything in between).

Quoted 5 lines
Quoted 1 line

Just keep them?

You'd still have to inspect the objects, which is way more work than the current code has to do. Remember: in the optimal case, upload-pack does not more than just serve the existing deltas/base objects.

Ah, I can see how that would produce overhead then.

Signature · 3 lines
Cheers,

Sverre Rabbelier