Skip to contentSkip to conversation

ref-filter: add %(rest) atom

RFC PATCH Superseded ZheNing Hu via GitGitGadget 3 participants 13 messages Jun 8, 2021 – Jun 9, 2021 Documentation

lore
Share & tools
lore
#4 lore raw

PATCH 4/4 ref-filter: add %(rest) atom

In reply to ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

In order to let "cat-file --batch=%(rest)" use the ref-filter interface, add %(rest) atom for ref-filter. "git for-each-ref", "git branch", "git tag" and "git verify-tag" will reject %(rest) by default.

---
 ref-filter.c             | 21 +++++++++++++++++++++
 ref-filter.h             |  5 ++++-
 t/t3203-branch-output.sh |  4 ++++
 t/t6300-for-each-ref.sh  |  4 ++++
 t/t7004-tag.sh           |  4 ++++
 t/t7030-verify-tag.sh    |  4 ++++
 6 files changed, 41 insertions(+), 1 deletion(-)
ref-filter.c +21 -0
index af8c15aef44d..8868cf98f090 100644
@@ -157,6 +157,7 @@ enum atom_type {
 	ATOM_IF,
 	ATOM_THEN,
 	ATOM_ELSE,
+	ATOM_REST,
 };
 
 /*
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,
 	return 0;
 }
 
+static int rest_atom_parser(struct ref_format *format, struct used_atom *atom,
+			    const char *arg, struct strbuf *err)
+{
+	if (arg)
+		return strbuf_addf_ret(err, -1, _("%%(rest) does not take arguments"));
+	format->use_rest = 1;
+	return 0;
+}
+
 static int head_atom_parser(struct ref_format *format, struct used_atom *atom,
 			    const char *arg, struct strbuf *unused_err)
 {
@@ -615,6 +625,7 @@ static struct {
 	[ATOM_IF] = { "if", SOURCE_NONE, FIELD_STR, if_atom_parser },
 	[ATOM_THEN] = { "then", SOURCE_NONE },
 	[ATOM_ELSE] = { "else", SOURCE_NONE },
+	[ATOM_REST] = { "rest", SOURCE_NONE, FIELD_STR, rest_atom_parser },
 	/*
 	 * Please update $__git_ref_fieldlist in git-completion.bash
 	 * when you add new atoms
@@ -1006,6 +1017,9 @@ int verify_ref_format(struct ref_format *format)
 		at = parse_ref_filter_atom(format, sp + 2, ep, &err);
 		if (at < 0)
 			die("%s", err.buf);
+		if (used_atom[at].atom_type == ATOM_REST)
+			die("this command reject atom %%(%.*s)", (int)(ep - sp - 2), sp + 2);
+
 		if (format->quote_style && used_atom[at].atom_type == ATOM_RAW &&
 		    used_atom[at].u.raw_data.option == RAW_BARE)
 			die(_("--format=%.*s cannot be used with"
@@ -1920,6 +1934,12 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err)
 			v->handler = else_atom_handler;
 			v->s = xstrdup("");
 			continue;
+		} else if (atom_type == ATOM_REST) {
+			if (ref->rest)
+				v->s = xstrdup(ref->rest);
+			else
+				v->s = xstrdup("");
+			continue;
 		} else
 			continue;
 
@@ -2137,6 +2157,7 @@ static struct ref_array_item *new_ref_array_item(const char *refname,
 
 	FLEX_ALLOC_STR(ref, refname, refname);
 	oidcpy(&ref->objectname, oid);
+	ref->rest = NULL;
 
 	return ref;
 }
ref-filter.h +4 -1
index 74fb423fc89f..9dc07476a584 100644
@@ -38,6 +38,7 @@ struct ref_sorting {
 
 struct ref_array_item {
 	struct object_id objectname;
+	const char *rest;
 	int flag;
 	unsigned int kind;
 	const char *symref;
@@ -76,14 +77,16 @@ struct ref_format {
 	 * verify_ref_format() afterwards to finalize.
 	 */
 	const char *format;
+	const char *rest;
 	int quote_style;
+	int use_rest;
 	int use_color;
 
 	/* Internal state to ref-filter */
 	int need_color_reset_at_eol;
 };
 
-#define REF_FORMAT_INIT { NULL, 0, -1 }
+#define REF_FORMAT_INIT { NULL, NULL, 0, 0, -1 }
 
 /*  Macros for checking --merged and --no-merged options */
 #define _OPT_MERGED_NO_MERGED(option, filter, h) \
t/t3203-branch-output.sh +4 -0
index 5325b9f67a00..2780ec8803fd 100755
@@ -340,6 +340,10 @@ test_expect_success 'git branch --format option' '
 	test_cmp expect actual
 '
 
+test_expect_success 'git branch with --format=%(rest) must failed' '
+	test_must_fail git branch --format="%(rest)" >actual
+'
+
 test_expect_success 'worktree colors correct' '
 	cat >expect <<-EOF &&
 	* <GREEN>(HEAD detached from fromtag)<RESET>
t/t6300-for-each-ref.sh +4 -0
index b86950bb6e68..f59975a9b6af 100755
@@ -1187,6 +1187,10 @@ test_expect_success 'basic atom: head contents:trailers' '
 	test_cmp expect actual.clean
 '
 
+test_expect_success 'basic atom: rest must failed' '
+	test_must_fail git for-each-ref --format="%(rest)" refs/heads/main
+'
+
 test_expect_success 'trailer parsing not fooled by --- line' '
 	git commit --allow-empty -F - <<-\EOF &&
 	this is the subject
t/t7004-tag.sh +4 -0
index 2f72c5c6883e..9fc4c4323949 100755
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' '
 	test_cmp expect actual
 '
 
+test_expect_success 'git tag -l with --format="%(rest)" must failed' '
+	test_must_fail git tag -l --format="%(rest)" "v1*"
+'
+
 test_expect_success "set up color tests" '
 	echo "<RED>v1.0<RESET>" >expect.color &&
 	echo "v1.0" >expect.bare &&
t/t7030-verify-tag.sh +4 -0
index 3cefde9602bf..785b32eb88f9 100755
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' '
 	test_cmp expect actual
 '
 
+test_expect_success GPG 'verifying tag with --format="%(rest)" must failed' '
+	test_must_fail git verify-tag --format="%(rest)" "fourth-signed"
+'
+
 test_expect_success GPG 'verifying a forged tag with --format should fail silently' '
 	test_must_fail git verify-tag --format="tagname : %(tag)" $(cat forged1.tag) >actual-forged &&
 	test_must_be_empty actual-forged
Signature · 1 line
gitgitgadget
#1 lore raw

PATCH 1/4 ref-filter: add obj-type check in grab contents

In reply to ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

Only tag and commit objects use grab_sub_body_contents() to grab object contents in the current codebase. We want to teach the function to also handle blobs and trees to get their raw data, without parsing a blob (whose contents looks like a commit or a tag) incorrectly as a commit or a tag.

Skip the block of code that is specific to handling commits and tags early when the given object is of a wrong type to help later addition to handle other types of objects in this function.

---
 ref-filter.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)
ref-filter.c +15 -9
index 4db0e40ff4c6..5cee6512fbaf 100644
@@ -1356,11 +1356,12 @@ static void append_lines(struct strbuf *out, const char *buf, unsigned long size
 }
 
 /* See grab_values */
-static void grab_sub_body_contents(struct atom_value *val, int deref, void *buf)
+static void grab_sub_body_contents(struct atom_value *val, int deref, struct expand_data *data)
 {
 	int i;
 	const char *subpos = NULL, *bodypos = NULL, *sigpos = NULL;
 	size_t sublen = 0, bodylen = 0, nonsiglen = 0, siglen = 0;
+	void *buf = data->content;
 
 	for (i = 0; i < used_atom_cnt; i++) {
 		struct used_atom *atom = &used_atom[i];
@@ -1371,10 +1372,13 @@ static void grab_sub_body_contents(struct atom_value *val, int deref, void *buf)
 			continue;
 		if (deref)
 			name++;
-		if (strcmp(name, "body") &&
-		    !starts_with(name, "subject") &&
-		    !starts_with(name, "trailers") &&
-		    !starts_with(name, "contents"))
+
+		if ((data->type != OBJ_TAG &&
+		     data->type != OBJ_COMMIT) ||
+		    (strcmp(name, "body") &&
+		     !starts_with(name, "subject") &&
+		     !starts_with(name, "trailers") &&
+		     !starts_with(name, "contents")))
 			continue;
 		if (!subpos)
 			find_subpos(buf,
@@ -1438,17 +1442,19 @@ static void fill_missing_values(struct atom_value *val)
  * pointed at by the ref itself; otherwise it is the object the
  * ref (which is a tag) refers to.
  */
-static void grab_values(struct atom_value *val, int deref, struct object *obj, void *buf)
+static void grab_values(struct atom_value *val, int deref, struct object *obj, struct expand_data *data)
 {
+	void *buf = data->content;
+
 	switch (obj->type) {
 	case OBJ_TAG:
 		grab_tag_values(val, deref, obj);
-		grab_sub_body_contents(val, deref, buf);
+		grab_sub_body_contents(val, deref, data);
 		grab_person("tagger", val, deref, buf);
 		break;
 	case OBJ_COMMIT:
 		grab_commit_values(val, deref, obj);
-		grab_sub_body_contents(val, deref, buf);
+		grab_sub_body_contents(val, deref, data);
 		grab_person("author", val, deref, buf);
 		grab_person("committer", val, deref, buf);
 		break;
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj
 			return strbuf_addf_ret(err, -1, _("parse_object_buffer failed on %s for %s"),
 					       oid_to_hex(&oi->oid), ref->refname);
 		}
-		grab_values(ref->value, deref, *obj, oi->content);
+		grab_values(ref->value, deref, *obj, oi);
 	}
 
 	grab_common_values(ref->value, deref, oi);
Signature · 1 line
gitgitgadget
#2 lore raw

PATCH 2/4 ref-filter: add %(raw) atom

In reply to ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

Add new formatting option %(raw), which will print the raw object data without any changes. It will help further to migrate all cat-file formatting logic from cat-file to ref-filter.

The raw data of blob, tree objects may contain '\0', but most of the logic in ref-filter depends on the output of the atom being text (specifically, no embedded NULs in it).

E.g. quote_formatting() use strbuf_addstr() or *._quote_buf() add the data to the buffer. The raw data of a tree object is 100644 one\0..., only the 100644 one will be added to the buffer, which is incorrect.

Therefore, add a new member in struct atom_value: s_size, which can record raw object size, it can help us add raw object data to the buffer or compare two buffers which contain raw object data.

Beyond, --format=%(raw) cannot be used with --python, --shell, --tcl, --perl because if our binary raw data is passed to a variable in the host language, the host language may not support arbitrary binary data in the variables of its string type.

---
 Documentation/git-for-each-ref.txt |   9 ++
 ref-filter.c                       | 139 +++++++++++++++----
 t/t6300-for-each-ref.sh            | 207 +++++++++++++++++++++++++++++
 3 files changed, 328 insertions(+), 27 deletions(-)
Documentation/git-for-each-ref.txt +9 -0
index 2ae2478de706..7f1f0a1ca3b6 100644
@@ -235,6 +235,15 @@ and `date` to extract the named component.  For email fields (`authoremail`,
 without angle brackets, and `:localpart` to get the part before the `@` symbol
 out of the trimmed email.
 
+The raw data in an object is `raw`.
+
+raw:size::
+	The raw data size of the object.
+
+Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,
+`--perl` because the host language may not support arbitrary binary data in the
+variables of its string type.
+
 The message in a commit or a tag object is `contents`, from which
 `contents:<part>` can be used to extract various parts out of:
 
ref-filter.c +112 -27

293 lines not on this page · raw message

t/t6300-for-each-ref.sh +207 -0

249 lines not on this page · raw message

Signature · 1 line
gitgitgadget
#0 lore raw

cover ref-filter: add %(rest) atom

In order to let git cat-file --batch reuse ref-filter logic, in this version, the atom %(rest) is added to ref-filter.

Change from last version:

1. Deleted two commit about %(raw:textconv) and %(raw:filters), their construction will be after letting cat-file --batch uses ref-filter logic.
2. Exchange the order of the definition of "cmp_fn" and the declaration of "a_size" and "b_size" in cmp_ref_sorting(), to avoid breaking -Wdecl-after-stmt.
3. Delete --rest option, "git for-each-ref" family will reject %(rest) by default.
4. Add GPG prefix to some %(raw) tests.

ZheNing Hu (4):
[GSOC] ref-filter: add obj-type check in grab contents [GSOC] ref-filter: add %(raw) atom
[GSOC] ref-filter: use non-const ref_format in *_atom_parser() [GSOC] ref-filter: add %(rest) atom

 Documentation/git-for-each-ref.txt |   9 ++
 builtin/tag.c                      |   2 +-
 ref-filter.c                       | 226 +++++++++++++++++++++--------
 ref-filter.h                       |   9 +-
 t/t3203-branch-output.sh           |   4 +
 t/t6300-for-each-ref.sh            | 211 +++++++++++++++++++++++++++
 t/t7004-tag.sh                     |   4 +
 t/t7030-verify-tag.sh              |   4 +
 8 files changed, 408 insertions(+), 61 deletions(-)

base-commit: 1197f1a46360d3ae96bd9c15908a3a6f8e562207
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-976%2Fadlternative%2Fref-filter-texconv-filters-v2-v1
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-976/adlternative/ref-filter-texconv-filters-v2-v1
Pull-Request: https://github.com/gitgitgadget/git/pull/976

Signature · 1 line
gitgitgadget
#3 lore raw

PATCH 3/4 ref-filter: use non-const ref_format in *_atom_parser()

From: ZheNing Hu <[email protected]>

Use non-const ref_format in *_atom_parser(), which can help us modify the members of ref_format in *_atom_parser().

---
 builtin/tag.c |  2 +-
 ref-filter.c  | 44 ++++++++++++++++++++++----------------------
 ref-filter.h  |  4 ++--
 3 files changed, 25 insertions(+), 25 deletions(-)
builtin/tag.c +1 -1
index 82fcfc098242..452558ec9575 100644
@@ -146,7 +146,7 @@ static int verify_tag(const char *name, const char *ref,
 		      const struct object_id *oid, void *cb_data)
 {
 	int flags;
-	const struct ref_format *format = cb_data;
+	struct ref_format *format = cb_data;
 	flags = GPG_VERIFY_VERBOSE;
 
 	if (format->format)
ref-filter.c +22 -22

164 lines not on this page · raw message

ref-filter.h +2 -2
index baf72a718965..74fb423fc89f 100644
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);
 void ref_sorting_set_sort_flags_all(struct ref_sorting *sorting, unsigned int mask, int on);
 /*  Based on the given format and quote_style, fill the strbuf */
 int format_ref_array_item(struct ref_array_item *info,
-			  const struct ref_format *format,
+			  struct ref_format *format,
 			  struct strbuf *final_buf,
 			  struct strbuf *error_buf);
 /*  Parse a single sort specifier and add it to the list */
@@ -137,7 +137,7 @@ void setup_ref_filter_porcelain_msg(void);
  * name must be a fully qualified refname.
  */
 void pretty_print_ref(const char *name, const struct object_id *oid,
-		      const struct ref_format *format);
+		      struct ref_format *format);
 
 /*
  * Push a single ref onto the array; this can be used to construct your own
Signature · 1 line
gitgitgadget
#5 lore raw
In reply to ZheNing Hu via GitGitGadget
Quoted 15 lines · ZheNing Hu via GitGitGadget

In order to let git cat-file --batch reuse ref-filter logic, in this version, the atom %(rest) is added to ref-filter.

Change from last version:

1. Deleted two commit about %(raw:textconv) and %(raw:filters), their construction will be after letting cat-file --batch uses ref-filter logic.
2. Exchange the order of the definition of "cmp_fn" and the declaration of "a_size" and "b_size" in cmp_ref_sorting(), to avoid breaking -Wdecl-after-stmt.
3. Delete --rest option, "git for-each-ref" family will reject %(rest) by default.
4. Add GPG prefix to some %(raw) tests.

Quoted 5 lines

ZheNing Hu (4):
[GSOC] ref-filter: add obj-type check in grab contents [GSOC] ref-filter: add %(raw) atom
[GSOC] ref-filter: use non-const ref_format in *_atom_parser() [GSOC] ref-filter: add %(rest) atom

Thanks, will replace.

#6 lore raw
Quoted 18 lines · Junio C Hamano <[email protected]> 于2021年6月9日周三 上午11:39写道:
Quoted 15 lines · ZheNing Hu via GitGitGadget

In order to let git cat-file --batch reuse ref-filter logic, in this version, the atom %(rest) is added to ref-filter.

Change from last version:

1. Deleted two commit about %(raw:textconv) and %(raw:filters), their construction will be after letting cat-file --batch uses ref-filter logic.
2. Exchange the order of the definition of "cmp_fn" and the declaration of "a_size" and "b_size" in cmp_ref_sorting(), to avoid breaking -Wdecl-after-stmt.
3. Delete --rest option, "git for-each-ref" family will reject %(rest) by default.
4. Add GPG prefix to some %(raw) tests.

Quoted 7 lines
Quoted 5 lines

ZheNing Hu (4):
[GSOC] ref-filter: add obj-type check in grab contents [GSOC] ref-filter: add %(raw) atom
[GSOC] ref-filter: use non-const ref_format in *_atom_parser() [GSOC] ref-filter: add %(rest) atom

Thanks, will replace.

It’s worth mentioning that this patch still fails the github CI test. [1]

t6300-for-each-ref.sh:
274: Verify sorts with raw
275: Verify sorts with raw:size
278: empty raw refs with %(if)

There is no such breakage on my local machine.

[1] https://github.com/gitgitgadget/git/pull/976/checks?check_run_id=2774963705

#7 lore raw
Quoted 10 lines · ZheNing Hu

It’s worth mentioning that this patch still fails the github CI test. [1]

t6300-for-each-ref.sh:
274: Verify sorts with raw
275: Verify sorts with raw:size
278: empty raw refs with %(if)

There is no such breakage on my local machine.

[1] https://github.com/gitgitgadget/git/pull/976/checks?check_run_id=2774963705

It's one of the two 32-bit jobs, I think. If the other one fails the same way, that means the code does not work on 32-bit systems, most likely something in your code assumes certain types are 64-bit when they aren't.

#11 lore raw

PATCH v2 3/4 ref-filter: use non-const ref_format in *_atom_parser()

In reply to ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

Use non-const ref_format in *_atom_parser(), which can help us modify the members of ref_format in *_atom_parser().

---
 builtin/tag.c |  2 +-
 ref-filter.c  | 44 ++++++++++++++++++++++----------------------
 ref-filter.h  |  4 ++--
 3 files changed, 25 insertions(+), 25 deletions(-)
builtin/tag.c +1 -1
index 82fcfc098242..452558ec9575 100644
@@ -146,7 +146,7 @@ static int verify_tag(const char *name, const char *ref,
 		      const struct object_id *oid, void *cb_data)
 {
 	int flags;
-	const struct ref_format *format = cb_data;
+	struct ref_format *format = cb_data;
 	flags = GPG_VERIFY_VERBOSE;
 
 	if (format->format)
ref-filter.c +22 -22

164 lines not on this page · raw message

ref-filter.h +2 -2
index baf72a718965..74fb423fc89f 100644
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);
 void ref_sorting_set_sort_flags_all(struct ref_sorting *sorting, unsigned int mask, int on);
 /*  Based on the given format and quote_style, fill the strbuf */
 int format_ref_array_item(struct ref_array_item *info,
-			  const struct ref_format *format,
+			  struct ref_format *format,
 			  struct strbuf *final_buf,
 			  struct strbuf *error_buf);
 /*  Parse a single sort specifier and add it to the list */
@@ -137,7 +137,7 @@ void setup_ref_filter_porcelain_msg(void);
  * name must be a fully qualified refname.
  */
 void pretty_print_ref(const char *name, const struct object_id *oid,
-		      const struct ref_format *format);
+		      struct ref_format *format);
 
 /*
  * Push a single ref onto the array; this can be used to construct your own
Signature · 1 line
gitgitgadget
#8 lore raw

v2 cover ref-filter: add %(rest) atom

In reply to ZheNing Hu via GitGitGadget

In order to let git cat-file --batch reuse ref-filter logic, the atom %(rest) and %(raw) are added to ref-filter.

Change from last version: Use printf "xxx" instead of printf "%b" "xxx" in test, ensure that strings containing NUL will not be truncated, to avoid the breakage of the test.

ZheNing Hu (4):
[GSOC] ref-filter: add obj-type check in grab contents [GSOC] ref-filter: add %(raw) atom
[GSOC] ref-filter: use non-const ref_format in *_atom_parser() [GSOC] ref-filter: add %(rest) atom

 Documentation/git-for-each-ref.txt |   9 ++
 builtin/tag.c                      |   2 +-
 ref-filter.c                       | 226 +++++++++++++++++++++--------
 ref-filter.h                       |   9 +-
 t/t3203-branch-output.sh           |   4 +
 t/t6300-for-each-ref.sh            | 211 +++++++++++++++++++++++++++
 t/t7004-tag.sh                     |   4 +
 t/t7030-verify-tag.sh              |   4 +
 8 files changed, 408 insertions(+), 61 deletions(-)

base-commit: 1197f1a46360d3ae96bd9c15908a3a6f8e562207
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-976%2Fadlternative%2Fref-filter-texconv-filters-v2-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-976/adlternative/ref-filter-texconv-filters-v2-v2
Pull-Request: https://github.com/gitgitgadget/git/pull/976

Range-diff vs v1:

 1:  48d256db5c34 = 1:  48d256db5c34 [GSOC] ref-filter: add obj-type check in grab contents
 2:  428d105ebbad ! 2:  abee6a03becb [GSOC] ref-filter: add %(raw) atom
     @@ t/t6300-for-each-ref.sh: test_atom refs/myblobs/first contents:body ""
      +'
      +
      +test_expect_success 'set up refs pointing to binary blob' '
     -+	printf "%b" "a\0b\0c" >blob1 &&
     -+	printf "%b" "a\0c\0b" >blob2 &&
     -+	printf "%b" "\0a\0b\0c" >blob3 &&
     -+	printf "%b" "abc" >blob4 &&
     -+	printf "%b" "\0 \0 \0 " >blob5 &&
     -+	printf "%b" "\0 \0a\0 " >blob6 &&
     -+	printf "%b" "  " >blob7 &&
     ++	printf "a\0b\0c" >blob1 &&
     ++	printf "a\0c\0b" >blob2 &&
     ++	printf "\0a\0b\0c" >blob3 &&
     ++	printf "abc" >blob4 &&
     ++	printf "\0 \0 \0 " >blob5 &&
     ++	printf "\0 \0a\0 " >blob6 &&
     ++	printf "  " >blob7 &&
      +	>blob8 &&
      +	git hash-object blob1 -w | xargs git update-ref refs/myblobs/blob1 &&
      +	git hash-object blob2 -w | xargs git update-ref refs/myblobs/blob2 &&
 3:  e4a5aeec0409 = 3:  c99d1d070a18 [GSOC] ref-filter: use non-const ref_format in *_atom_parser()
 4:  5e95292e3de2 = 4:  5a5b5f78aeea [GSOC] ref-filter: add %(rest) atom
Signature · 1 line
gitgitgadget
#10 lore raw

PATCH v2 2/4 ref-filter: add %(raw) atom

From: ZheNing Hu <[email protected]>

Add new formatting option %(raw), which will print the raw object data without any changes. It will help further to migrate all cat-file formatting logic from cat-file to ref-filter.

The raw data of blob, tree objects may contain '\0', but most of the logic in ref-filter depends on the output of the atom being text (specifically, no embedded NULs in it).

E.g. quote_formatting() use strbuf_addstr() or *._quote_buf() add the data to the buffer. The raw data of a tree object is 100644 one\0..., only the 100644 one will be added to the buffer, which is incorrect.

Therefore, add a new member in struct atom_value: s_size, which can record raw object size, it can help us add raw object data to the buffer or compare two buffers which contain raw object data.

Beyond, --format=%(raw) cannot be used with --python, --shell, --tcl, --perl because if our binary raw data is passed to a variable in the host language, the host language may not support arbitrary binary data in the variables of its string type.

---
 Documentation/git-for-each-ref.txt |   9 ++
 ref-filter.c                       | 139 +++++++++++++++----
 t/t6300-for-each-ref.sh            | 207 +++++++++++++++++++++++++++++
 3 files changed, 328 insertions(+), 27 deletions(-)
Documentation/git-for-each-ref.txt +9 -0
index 2ae2478de706..7f1f0a1ca3b6 100644
@@ -235,6 +235,15 @@ and `date` to extract the named component.  For email fields (`authoremail`,
 without angle brackets, and `:localpart` to get the part before the `@` symbol
 out of the trimmed email.
 
+The raw data in an object is `raw`.
+
+raw:size::
+	The raw data size of the object.
+
+Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,
+`--perl` because the host language may not support arbitrary binary data in the
+variables of its string type.
+
 The message in a commit or a tag object is `contents`, from which
 `contents:<part>` can be used to extract various parts out of:
 
ref-filter.c +112 -27

293 lines not on this page · raw message

t/t6300-for-each-ref.sh +207 -0

249 lines not on this page · raw message

Signature · 1 line
gitgitgadget
#12 lore raw

PATCH v2 4/4 ref-filter: add %(rest) atom

In reply to ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

In order to let "cat-file --batch=%(rest)" use the ref-filter interface, add %(rest) atom for ref-filter. "git for-each-ref", "git branch", "git tag" and "git verify-tag" will reject %(rest) by default.

---
 ref-filter.c             | 21 +++++++++++++++++++++
 ref-filter.h             |  5 ++++-
 t/t3203-branch-output.sh |  4 ++++
 t/t6300-for-each-ref.sh  |  4 ++++
 t/t7004-tag.sh           |  4 ++++
 t/t7030-verify-tag.sh    |  4 ++++
 6 files changed, 41 insertions(+), 1 deletion(-)
ref-filter.c +21 -0
index af8c15aef44d..8868cf98f090 100644
@@ -157,6 +157,7 @@ enum atom_type {
 	ATOM_IF,
 	ATOM_THEN,
 	ATOM_ELSE,
+	ATOM_REST,
 };
 
 /*
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,
 	return 0;
 }
 
+static int rest_atom_parser(struct ref_format *format, struct used_atom *atom,
+			    const char *arg, struct strbuf *err)
+{
+	if (arg)
+		return strbuf_addf_ret(err, -1, _("%%(rest) does not take arguments"));
+	format->use_rest = 1;
+	return 0;
+}
+
 static int head_atom_parser(struct ref_format *format, struct used_atom *atom,
 			    const char *arg, struct strbuf *unused_err)
 {
@@ -615,6 +625,7 @@ static struct {
 	[ATOM_IF] = { "if", SOURCE_NONE, FIELD_STR, if_atom_parser },
 	[ATOM_THEN] = { "then", SOURCE_NONE },
 	[ATOM_ELSE] = { "else", SOURCE_NONE },
+	[ATOM_REST] = { "rest", SOURCE_NONE, FIELD_STR, rest_atom_parser },
 	/*
 	 * Please update $__git_ref_fieldlist in git-completion.bash
 	 * when you add new atoms
@@ -1006,6 +1017,9 @@ int verify_ref_format(struct ref_format *format)
 		at = parse_ref_filter_atom(format, sp + 2, ep, &err);
 		if (at < 0)
 			die("%s", err.buf);
+		if (used_atom[at].atom_type == ATOM_REST)
+			die("this command reject atom %%(%.*s)", (int)(ep - sp - 2), sp + 2);
+
 		if (format->quote_style && used_atom[at].atom_type == ATOM_RAW &&
 		    used_atom[at].u.raw_data.option == RAW_BARE)
 			die(_("--format=%.*s cannot be used with"
@@ -1920,6 +1934,12 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err)
 			v->handler = else_atom_handler;
 			v->s = xstrdup("");
 			continue;
+		} else if (atom_type == ATOM_REST) {
+			if (ref->rest)
+				v->s = xstrdup(ref->rest);
+			else
+				v->s = xstrdup("");
+			continue;
 		} else
 			continue;
 
@@ -2137,6 +2157,7 @@ static struct ref_array_item *new_ref_array_item(const char *refname,
 
 	FLEX_ALLOC_STR(ref, refname, refname);
 	oidcpy(&ref->objectname, oid);
+	ref->rest = NULL;
 
 	return ref;
 }
ref-filter.h +4 -1
index 74fb423fc89f..9dc07476a584 100644
@@ -38,6 +38,7 @@ struct ref_sorting {
 
 struct ref_array_item {
 	struct object_id objectname;
+	const char *rest;
 	int flag;
 	unsigned int kind;
 	const char *symref;
@@ -76,14 +77,16 @@ struct ref_format {
 	 * verify_ref_format() afterwards to finalize.
 	 */
 	const char *format;
+	const char *rest;
 	int quote_style;
+	int use_rest;
 	int use_color;
 
 	/* Internal state to ref-filter */
 	int need_color_reset_at_eol;
 };
 
-#define REF_FORMAT_INIT { NULL, 0, -1 }
+#define REF_FORMAT_INIT { NULL, NULL, 0, 0, -1 }
 
 /*  Macros for checking --merged and --no-merged options */
 #define _OPT_MERGED_NO_MERGED(option, filter, h) \
t/t3203-branch-output.sh +4 -0
index 5325b9f67a00..2780ec8803fd 100755
@@ -340,6 +340,10 @@ test_expect_success 'git branch --format option' '
 	test_cmp expect actual
 '
 
+test_expect_success 'git branch with --format=%(rest) must failed' '
+	test_must_fail git branch --format="%(rest)" >actual
+'
+
 test_expect_success 'worktree colors correct' '
 	cat >expect <<-EOF &&
 	* <GREEN>(HEAD detached from fromtag)<RESET>
t/t6300-for-each-ref.sh +4 -0
index e2867de791e7..8c97c3b877c6 100755
@@ -1187,6 +1187,10 @@ test_expect_success 'basic atom: head contents:trailers' '
 	test_cmp expect actual.clean
 '
 
+test_expect_success 'basic atom: rest must failed' '
+	test_must_fail git for-each-ref --format="%(rest)" refs/heads/main
+'
+
 test_expect_success 'trailer parsing not fooled by --- line' '
 	git commit --allow-empty -F - <<-\EOF &&
 	this is the subject
t/t7004-tag.sh +4 -0
index 2f72c5c6883e..9fc4c4323949 100755
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' '
 	test_cmp expect actual
 '
 
+test_expect_success 'git tag -l with --format="%(rest)" must failed' '
+	test_must_fail git tag -l --format="%(rest)" "v1*"
+'
+
 test_expect_success "set up color tests" '
 	echo "<RED>v1.0<RESET>" >expect.color &&
 	echo "v1.0" >expect.bare &&
t/t7030-verify-tag.sh +4 -0
index 3cefde9602bf..785b32eb88f9 100755
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' '
 	test_cmp expect actual
 '
 
+test_expect_success GPG 'verifying tag with --format="%(rest)" must failed' '
+	test_must_fail git verify-tag --format="%(rest)" "fourth-signed"
+'
+
 test_expect_success GPG 'verifying a forged tag with --format should fail silently' '
 	test_must_fail git verify-tag --format="tagname : %(tag)" $(cat forged1.tag) >actual-forged &&
 	test_must_be_empty actual-forged
Signature · 1 line
gitgitgadget
#9 lore raw

PATCH v2 1/4 ref-filter: add obj-type check in grab contents

In reply to ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

Only tag and commit objects use grab_sub_body_contents() to grab object contents in the current codebase. We want to teach the function to also handle blobs and trees to get their raw data, without parsing a blob (whose contents looks like a commit or a tag) incorrectly as a commit or a tag.

Skip the block of code that is specific to handling commits and tags early when the given object is of a wrong type to help later addition to handle other types of objects in this function.

---
 ref-filter.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)
ref-filter.c +15 -9
index 4db0e40ff4c6..5cee6512fbaf 100644
@@ -1356,11 +1356,12 @@ static void append_lines(struct strbuf *out, const char *buf, unsigned long size
 }
 
 /* See grab_values */
-static void grab_sub_body_contents(struct atom_value *val, int deref, void *buf)
+static void grab_sub_body_contents(struct atom_value *val, int deref, struct expand_data *data)
 {
 	int i;
 	const char *subpos = NULL, *bodypos = NULL, *sigpos = NULL;
 	size_t sublen = 0, bodylen = 0, nonsiglen = 0, siglen = 0;
+	void *buf = data->content;
 
 	for (i = 0; i < used_atom_cnt; i++) {
 		struct used_atom *atom = &used_atom[i];
@@ -1371,10 +1372,13 @@ static void grab_sub_body_contents(struct atom_value *val, int deref, void *buf)
 			continue;
 		if (deref)
 			name++;
-		if (strcmp(name, "body") &&
-		    !starts_with(name, "subject") &&
-		    !starts_with(name, "trailers") &&
-		    !starts_with(name, "contents"))
+
+		if ((data->type != OBJ_TAG &&
+		     data->type != OBJ_COMMIT) ||
+		    (strcmp(name, "body") &&
+		     !starts_with(name, "subject") &&
+		     !starts_with(name, "trailers") &&
+		     !starts_with(name, "contents")))
 			continue;
 		if (!subpos)
 			find_subpos(buf,
@@ -1438,17 +1442,19 @@ static void fill_missing_values(struct atom_value *val)
  * pointed at by the ref itself; otherwise it is the object the
  * ref (which is a tag) refers to.
  */
-static void grab_values(struct atom_value *val, int deref, struct object *obj, void *buf)
+static void grab_values(struct atom_value *val, int deref, struct object *obj, struct expand_data *data)
 {
+	void *buf = data->content;
+
 	switch (obj->type) {
 	case OBJ_TAG:
 		grab_tag_values(val, deref, obj);
-		grab_sub_body_contents(val, deref, buf);
+		grab_sub_body_contents(val, deref, data);
 		grab_person("tagger", val, deref, buf);
 		break;
 	case OBJ_COMMIT:
 		grab_commit_values(val, deref, obj);
-		grab_sub_body_contents(val, deref, buf);
+		grab_sub_body_contents(val, deref, data);
 		grab_person("author", val, deref, buf);
 		grab_person("committer", val, deref, buf);
 		break;
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj
 			return strbuf_addf_ret(err, -1, _("parse_object_buffer failed on %s for %s"),
 					       oid_to_hex(&oi->oid), ref->refname);
 		}
-		grab_values(ref->value, deref, *obj, oi->content);
+		grab_values(ref->value, deref, *obj, oi);
 	}
 
 	grab_common_values(ref->value, deref, oi);
Signature · 1 line
gitgitgadget