Skip to contentSkip to conversation

scipts/tags.sh: Add custom sort order

PATCH New [email protected] 2 participants 9 messages Aug 5, 2020 – Sep 4, 2020 Kbuild

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 scipts/tags.sh: Add custom sort order

One long standing annoyance I have with using vim-tags is that our tags file is not properly sorted. That is, the sorting exhuberant Ctags does is only on the tag itself.

The problem with that is that, for example, the tag 'mutex' appears a mere 505 times, 492 of those are structure members. However it is _far_ more likely that someone wants the struct definition when looking for the mutex tag than any of those members. However, due to the nature of the sorting, the struct definition will not be first.

So add a script that does a custom sort of the tags file, taking the tag kind into account.

The kind ordering is roughly: 'type', 'function', 'macro', 'enum', rest.

---
 scripts/sort-tags.awk | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/tags.sh       |  8 +++++-
 2 files changed, 86 insertions(+), 1 deletion(-)
scripts/sort-tags.awk new file +79 -0
new file mode 100755
index 000000000000..1eb50406c9d3
@@ -0,0 +1,79 @@
+#!/usr/bin/awk -f
+
+# $ ctags --list-kinds
+# C
+#   c  classes
+#   s  structure names
+#   t  typedefs
+#   g  enumeration names
+#   u  union names
+#   n  namespaces
+
+#   f  function definitions
+#   p  function prototypes [off]
+#   d  macro definitions
+
+#   e  enumerators (values inside an enumeration)
+#   m  class, struct, and union members
+#   v  variable definitions
+
+#   l  local variables [off]
+#   x  external and forward variable declarations [off]
+
+BEGIN {
+	FS = "\t"
+
+	sort = "LC_ALL=C sort"
+
+	# our sort order for C kinds:
+	order["c"] = "A"
+	order["s"] = "B"
+	order["t"] = "C"
+	order["g"] = "D"
+	order["u"] = "E"
+	order["n"] = "F"
+	order["f"] = "G"
+	order["p"] = "H"
+	order["d"] = "I"
+	order["e"] = "J"
+	order["m"] = "K"
+	order["v"] = "L"
+	order["l"] = "M"
+	order["x"] = "N"
+}
+
+# pass through header
+/^!_TAG/ {
+	print $0
+	next
+}
+
+{
+	# find 'kinds'
+	for (i = 1; i <= NF; i++) {
+		if ($i ~ /;"$/) {
+			kind = $(i+1)
+			break;
+		}
+	}
+
+	# create sort key
+	if (order[kind])
+		key = $1 order[kind];
+	else
+		key = $1 "Z";
+
+	# get it sorted
+	print key "\t" $0 |& sort
+}
+
+END {
+	close(sort, "to")
+	while ((sort |& getline) > 0) {
+		# strip key
+		sub(/[^[:space:]]*[[:space:]]*/, "")
+		print $0
+	}
+	close(sort)
+}
+
scripts/tags.sh +7 -1
index 4e18ae5282a6..93d729392a7b 100755
@@ -251,6 +251,8 @@ setup_regex()
 
 exuberant()
 {
+	rm -f tags.unsorted
+
 	setup_regex exuberant asm c
 	all_target_sources | xargs $1 -a                        \
 	-I __initdata,__exitdata,__initconst,__ro_after_init	\
@@ -266,12 +268,16 @@ exuberant()
 	-I DEFINE_TRACE,EXPORT_TRACEPOINT_SYMBOL,EXPORT_TRACEPOINT_SYMBOL_GPL \
 	-I static,const						\
 	--extra=+fq --c-kinds=+px --fields=+iaS --langmap=c:+.h \
+	--sort=no -o tags.unsorted				\
 	"${regex[@]}"
 
 	setup_regex exuberant kconfig
 	all_kconfigs | xargs $1 -a                              \
-	--langdef=kconfig --language-force=kconfig "${regex[@]}"
+	--langdef=kconfig --language-force=kconfig --sort=no	\
+	-o tags.unsorted "${regex[@]}"
 
+	scripts/sort-tags.awk tags.unsorted > tags
+	rm -f tags.unsorted
 }
 
 emacs()
#1 lore raw

PATCH 1/1 scipts/tags.sh: Add custom sort order

One long standing annoyance I have with using vim-tags is that our tags file is not properly sorted. That is, the sorting exhuberant Ctags does is only on the tag itself.

The problem with that is that, for example, the tag 'mutex' appears a mere 505 times, 492 of those are structure members. However it is _far_ more likely that someone wants the struct definition when looking for the mutex tag than any of those members. However, due to the nature of the sorting, the struct definition will not be first.

So add a script that does a custom sort of the tags file, taking the tag kind into account.

The kind ordering is roughly: 'type', 'function', 'macro', 'enum', rest.

---
Changes since v1:
- removed the need for tags.unsorted by using a pipe

Due to this change 'make tags' is now actually faster than it was before due to less sorting.

 scripts/sort-tags.awk | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/tags.sh       | 11 +++++--
 2 files changed, 87 insertions(+), 3 deletions(-)
scripts/sort-tags.awk new file +79 -0
new file mode 100755
index 000000000000..1eb50406c9d3
@@ -0,0 +1,79 @@
+#!/usr/bin/awk -f
+
+# $ ctags --list-kinds
+# C
+#   c  classes
+#   s  structure names
+#   t  typedefs
+#   g  enumeration names
+#   u  union names
+#   n  namespaces
+
+#   f  function definitions
+#   p  function prototypes [off]
+#   d  macro definitions
+
+#   e  enumerators (values inside an enumeration)
+#   m  class, struct, and union members
+#   v  variable definitions
+
+#   l  local variables [off]
+#   x  external and forward variable declarations [off]
+
+BEGIN {
+	FS = "\t"
+
+	sort = "LC_ALL=C sort"
+
+	# our sort order for C kinds:
+	order["c"] = "A"
+	order["s"] = "B"
+	order["t"] = "C"
+	order["g"] = "D"
+	order["u"] = "E"
+	order["n"] = "F"
+	order["f"] = "G"
+	order["p"] = "H"
+	order["d"] = "I"
+	order["e"] = "J"
+	order["m"] = "K"
+	order["v"] = "L"
+	order["l"] = "M"
+	order["x"] = "N"
+}
+
+# pass through header
+/^!_TAG/ {
+	print $0
+	next
+}
+
+{
+	# find 'kinds'
+	for (i = 1; i <= NF; i++) {
+		if ($i ~ /;"$/) {
+			kind = $(i+1)
+			break;
+		}
+	}
+
+	# create sort key
+	if (order[kind])
+		key = $1 order[kind];
+	else
+		key = $1 "Z";
+
+	# get it sorted
+	print key "\t" $0 |& sort
+}
+
+END {
+	close(sort, "to")
+	while ((sort |& getline) > 0) {
+		# strip key
+		sub(/[^[:space:]]*[[:space:]]*/, "")
+		print $0
+	}
+	close(sort)
+}
+
scripts/tags.sh +8 -3
index 4e18ae5282a6..51087c3d8b1e 100755
@@ -251,8 +251,10 @@ setup_regex()
 
 exuberant()
 {
+	(
+
 	setup_regex exuberant asm c
-	all_target_sources | xargs $1 -a                        \
+	all_target_sources | xargs $1				\
 	-I __initdata,__exitdata,__initconst,__ro_after_init	\
 	-I __initdata_memblock					\
 	-I __refdata,__attribute,__maybe_unused,__always_unused \
@@ -266,12 +268,15 @@ exuberant()
 	-I DEFINE_TRACE,EXPORT_TRACEPOINT_SYMBOL,EXPORT_TRACEPOINT_SYMBOL_GPL \
 	-I static,const						\
 	--extra=+fq --c-kinds=+px --fields=+iaS --langmap=c:+.h \
+	--sort=no -o -						\
 	"${regex[@]}"
 
 	setup_regex exuberant kconfig
-	all_kconfigs | xargs $1 -a                              \
-	--langdef=kconfig --language-force=kconfig "${regex[@]}"
+	all_kconfigs | xargs $1					\
+	--langdef=kconfig --language-force=kconfig --sort=no	\
+	-o - "${regex[@]}"
 
+	) | scripts/sort-tags.awk > tags
 }
 
 emacs()
#2 lore raw
Quoted 11 lines · On Thu, Aug 06, 2020 at 02:04:38PM +0200, [email protected] wrote:

One long standing annoyance I have with using vim-tags is that our tags file is not properly sorted. That is, the sorting exhuberant Ctags does is only on the tag itself.

The problem with that is that, for example, the tag 'mutex' appears a mere 505 times, 492 of those are structure members. However it is _far_ more likely that someone wants the struct definition when looking for the mutex tag than any of those members. However, due to the nature of the sorting, the struct definition will not be first.

Quoted 7 lines

So add a script that does a custom sort of the tags file, taking the tag kind into account.

The kind ordering is roughly: 'type', 'function', 'macro', 'enum', rest.

---

ping?

Reviewed-by Peter Zijlstra (Intel) on v1 1/1
#3 lore raw
In reply to [email protected]
Quoted 138 lines · On Fri, Aug 7, 2020 at 2:28 AM <[email protected]> wrote:

One long standing annoyance I have with using vim-tags is that our tags file is not properly sorted. That is, the sorting exhuberant Ctags does is only on the tag itself.

The problem with that is that, for example, the tag 'mutex' appears a mere 505 times, 492 of those are structure members. However it is _far_ more likely that someone wants the struct definition when looking for the mutex tag than any of those members. However, due to the nature of the sorting, the struct definition will not be first.

So add a script that does a custom sort of the tags file, taking the tag kind into account.

The kind ordering is roughly: 'type', 'function', 'macro', 'enum', rest.

---
Changes since v1:
- removed the need for tags.unsorted by using a pipe

Due to this change 'make tags' is now actually faster than it was before due to less sorting.

 scripts/sort-tags.awk | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/tags.sh       | 11 +++++--
 2 files changed, 87 insertions(+), 3 deletions(-)
scripts/sort-tags.awk new file +79 -0
new file mode 100755
index 000000000000..1eb50406c9d3
@@ -0,0 +1,79 @@
+#!/usr/bin/awk -f
+
+# $ ctags --list-kinds
+# C
+#   c  classes
+#   s  structure names
+#   t  typedefs
+#   g  enumeration names
+#   u  union names
+#   n  namespaces
+
+#   f  function definitions
+#   p  function prototypes [off]
+#   d  macro definitions
+
+#   e  enumerators (values inside an enumeration)
+#   m  class, struct, and union members
+#   v  variable definitions
+
+#   l  local variables [off]
+#   x  external and forward variable declarations [off]
+
+BEGIN {
+       FS = "\t"
+
+       sort = "LC_ALL=C sort"
+
+       # our sort order for C kinds:
+       order["c"] = "A"
+       order["s"] = "B"
+       order["t"] = "C"
+       order["g"] = "D"
+       order["u"] = "E"
+       order["n"] = "F"
+       order["f"] = "G"
+       order["p"] = "H"
+       order["d"] = "I"
+       order["e"] = "J"
+       order["m"] = "K"
+       order["v"] = "L"
+       order["l"] = "M"
+       order["x"] = "N"
+}
+
+# pass through header
+/^!_TAG/ {
+       print $0
+       next
+}
+
+{
+       # find 'kinds'
+       for (i = 1; i <= NF; i++) {
+               if ($i ~ /;"$/) {
+                       kind = $(i+1)
+                       break;
+               }
+       }
+
+       # create sort key
+       if (order[kind])
+               key = $1 order[kind];
+       else
+               key = $1 "Z";
+
+       # get it sorted
+       print key "\t" $0 |& sort
+}
+
+END {
+       close(sort, "to")
+       while ((sort |& getline) > 0) {
+               # strip key
+               sub(/[^[:space:]]*[[:space:]]*/, "")
+               print $0
+       }
+       close(sort)
+}
+
scripts/tags.sh +4 -1
index 4e18ae5282a6..51087c3d8b1e 100755
@@ -251,8 +251,10 @@ setup_regex()
 
 exuberant()
 {
+       (
+
        setup_regex exuberant asm c
-       all_target_sources | xargs $1 -a                        \
+       all_target_sources | xargs $1                           \
        -I __initdata,__exitdata,__initconst,__ro_after_init    \
        -I __initdata_memblock                                  \
        -I __refdata,__attribute,__maybe_unused,__always_unused \
@@ -266,12 +268,15 @@ exuberant()
        -I DEFINE_TRACE,EXPORT_TRACEPOINT_SYMBOL,EXPORT_TRACEPOINT_SYMBOL_GPL \
        -I static,const                                         \
        --extra=+fq --c-kinds=+px --fields=+iaS --langmap=c:+.h \
+       --sort=no -o -                                          \
        "${regex[@]}"
 
        setup_regex exuberant kconfig

scripts/tags.sh exuberant() · line 282

Quoted 10 lines
scripts/tags.sh +4 -2
-       all_kconfigs | xargs $1 -a                              \
-       --langdef=kconfig --language-force=kconfig "${regex[@]}"
+       all_kconfigs | xargs $1                                 \
+       --langdef=kconfig --language-force=kconfig --sort=no    \
+       -o - "${regex[@]}"
 
+       ) | scripts/sort-tags.awk > tags
 }
 
 emacs()

Sorry for the long delay.

First, this patch breaks 'make TAGS'
if 'etags' is a symlink to exuberant ctags.

masahiro@oscar:~/ref/linux$ etags --version
Exuberant Ctags 5.9~svn20110310, Copyright (C) 1996-2009 Darren Hiebert
  Addresses: <[email protected]>, http://ctags.sourceforge.net
  Optional compiled features: +wildcards, +regex
masahiro@oscar:~/ref/linux$ make TAGS
  GEN     TAGS
etags: Warning: include/linux/seqlock.h:738: null expansion of name pattern "\2"
sed: can't read TAGS: No such file or directory
make: *** [Makefile:1820: TAGS] Error 2

The reason is the hard-coded ' > tags',
and easy to fix.

But, honestly, I am not super happy about this patch.

Reason 1
In my understanding, sorting by the tag kind only works for ctags. My favorite editor is emacs.
(Do not get me wrong. I do not intend emacs vs vi war). So, I rather do 'make TAGS' instead of 'make tags', but this solution would not work for etags because etags has a different format.
So, I'd rather want to see a more general solution.

Reason 2
We would have more messy code, mixing two files/languages

When is it useful to tag structure members?

If they are really annoying, why don't we delete them instead of moving them to the bottom of the tag file?

I attached an alternative solution,
and wrote up my thoughts in the log.

What do you think?

Signature · 2 lines
Best Regards
Masahiro Yamada

From 1a003fce7e4f8460ef3256fb5d958fb5c6cc631e Mon Sep 17 00:00:00 2001
From: Masahiro Yamada <[email protected]>
Date: Wed, 2 Sep 2020 21:49:59 +0900
Subject: [PATCH] scripts/tags.sh: remove m, v, x tag kinds from exuberant tags

Exuberant Ctags supports the following kinds of tags:

  $ ctags --list-kinds=c
  c  classes
  d  macro definitions
  e  enumerators (values inside an enumeration)
  f  function definitions
  g  enumeration names
  l  local variables [off]
  m  class, struct, and union members
  n  namespaces
  p  function prototypes [off]
  s  structure names
  t  typedefs
  u  union names
  v  variable definitions
  x  external and forward variable declarations [off]

This commit excludes 'm', 'v', and 'x'.

Peter Zijlstra states:
"The problem with that is that, for example, the tag 'mutex' appears a mere 505 times, 492 of those are structure members. However it is _far_ more likely that someone wants the struct definition when looking for the mutex tag than any of those members."
(https://lkml.org/lkml/2020/8/6/512")

So, 'm' is rather annoying than useful. For the same reason, it seems better to turn off 'v'.

You may argue about the criteria, but we need to draw a line somewhere to make it reasonable for the majority of people.

We flipped 'p' and 'x' in the past:

[1] commit f6333eb4e788 ("kbuild: Add ctags support for function prototypes and external variable declarations") added 'p' and 'x', but did not explain when they are actually useful.

[2] commit 7db86dc97fb0 ("ctags: usability fix") removed 'p' and 'x', stating both of them make no real sense.

[3] commit 0a18a9386c05 ("tags: put function prototypes back!") re-added 'p' and 'x', but the commit log only mentioned 'p'.

OK, [3] clearly explained why 'p' is useful, but turned --c-kinds=+px into --c-kinds=-px. So, 'x' was also (accidentally?) disabled. I think it should have been --c-kinds=+p-x, or more simply --c-kinds=+p since 'x' is off by default.

It seems a bug of [3], so I disabled 'x' to get back the pre-[3] behavior.

'make tags' and 'make TAGS' will run faster, create much smaller tags if Ctags is exuberant.

---
 scripts/tags.sh | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)
scripts/tags.sh +1 -14
index 32d3f53af10b..440f4ecad43b 100755
@@ -179,19 +179,6 @@ regex_c=(
 	'/^DEF_PCI_AC_\(\|NO\)RET(\([[:alnum:]_]*\).*/\2/'
 	'/^PCI_OP_READ(\(\w*\).*[1-4])/pci_bus_read_config_\1/'
 	'/^PCI_OP_WRITE(\(\w*\).*[1-4])/pci_bus_write_config_\1/'
-	'/\<DEFINE_\(RT_MUTEX\|MUTEX\|SEMAPHORE\|SPINLOCK\)(\([[:alnum:]_]*\)/\2/v/'
-	'/\<DEFINE_\(RAW_SPINLOCK\|RWLOCK\|SEQLOCK\)(\([[:alnum:]_]*\)/\2/v/'
-	'/\<DECLARE_\(RWSEM\|COMPLETION\)(\([[:alnum:]_]\+\)/\2/v/'
-	'/\<DECLARE_BITMAP(\([[:alnum:]_]*\)/\1/v/'
-	'/\(^\|\s\)\(\|L\|H\)LIST_HEAD(\([[:alnum:]_]*\)/\3/v/'
-	'/\(^\|\s\)RADIX_TREE(\([[:alnum:]_]*\)/\2/v/'
-	'/\<DEFINE_PER_CPU([^,]*, *\([[:alnum:]_]*\)/\1/v/'
-	'/\<DEFINE_PER_CPU_SHARED_ALIGNED([^,]*, *\([[:alnum:]_]*\)/\1/v/'
-	'/\<DECLARE_WAIT_QUEUE_HEAD(\([[:alnum:]_]*\)/\1/v/'
-	'/\<DECLARE_\(TASKLET\|WORK\|DELAYED_WORK\)(\([[:alnum:]_]*\)/\2/v/'
-	'/\(^\s\)OFFSET(\([[:alnum:]_]*\)/\2/v/'
-	'/\(^\s\)DEFINE(\([[:alnum:]_]*\)/\2/v/'
-	'/\<\(DEFINE\|DECLARE\)_HASHTABLE(\([[:alnum:]_]*\)/\2/v/'
 	'/\<DEFINE_ID\(R\|A\)(\([[:alnum:]_]\+\)/\2/'
 	'/\<DEFINE_WD_CLASS(\([[:alnum:]_]\+\)/\1/'
 	'/\<ATOMIC_NOTIFIER_HEAD(\([[:alnum:]_]\+\)/\1/'
@@ -255,7 +242,7 @@ exuberant()
 	-I EXPORT_SYMBOL,EXPORT_SYMBOL_GPL,ACPI_EXPORT_SYMBOL   \
 	-I DEFINE_TRACE,EXPORT_TRACEPOINT_SYMBOL,EXPORT_TRACEPOINT_SYMBOL_GPL \
 	-I static,const						\
-	--extra=+fq --c-kinds=+px --fields=+iaS --langmap=c:+.h \
+	--extra=+fq --c-kinds=+p-mv --fields=+iaS --langmap=c:+.h \
 	"${regex[@]}"
 
 	setup_regex exuberant kconfig
Signature · 1 line
2.25.1
#4 lore raw
Quoted 200 lines · Masahiro Yamada
Quoted 138 lines · On Fri, Aug 7, 2020 at 2:28 AM <[email protected]> wrote:

One long standing annoyance I have with using vim-tags is that our tags file is not properly sorted. That is, the sorting exhuberant Ctags does is only on the tag itself.

The problem with that is that, for example, the tag 'mutex' appears a mere 505 times, 492 of those are structure members. However it is _far_ more likely that someone wants the struct definition when looking for the mutex tag than any of those members. However, due to the nature of the sorting, the struct definition will not be first.

So add a script that does a custom sort of the tags file, taking the tag kind into account.

The kind ordering is roughly: 'type', 'function', 'macro', 'enum', rest.

---
Changes since v1:
- removed the need for tags.unsorted by using a pipe

Due to this change 'make tags' is now actually faster than it was before due to less sorting.

 scripts/sort-tags.awk | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/tags.sh       | 11 +++++--
 2 files changed, 87 insertions(+), 3 deletions(-)
scripts/sort-tags.awk new file +79 -0
new file mode 100755
index 000000000000..1eb50406c9d3
@@ -0,0 +1,79 @@
+#!/usr/bin/awk -f
+
+# $ ctags --list-kinds
+# C
+#   c  classes
+#   s  structure names
+#   t  typedefs
+#   g  enumeration names
+#   u  union names
+#   n  namespaces
+
+#   f  function definitions
+#   p  function prototypes [off]
+#   d  macro definitions
+
+#   e  enumerators (values inside an enumeration)
+#   m  class, struct, and union members
+#   v  variable definitions
+
+#   l  local variables [off]
+#   x  external and forward variable declarations [off]
+
+BEGIN {
+       FS = "\t"
+
+       sort = "LC_ALL=C sort"
+
+       # our sort order for C kinds:
+       order["c"] = "A"
+       order["s"] = "B"
+       order["t"] = "C"
+       order["g"] = "D"
+       order["u"] = "E"
+       order["n"] = "F"
+       order["f"] = "G"
+       order["p"] = "H"
+       order["d"] = "I"
+       order["e"] = "J"
+       order["m"] = "K"
+       order["v"] = "L"
+       order["l"] = "M"
+       order["x"] = "N"
+}
+
+# pass through header
+/^!_TAG/ {
+       print $0
+       next
+}
+
+{
+       # find 'kinds'
+       for (i = 1; i <= NF; i++) {
+               if ($i ~ /;"$/) {
+                       kind = $(i+1)
+                       break;
+               }
+       }
+
+       # create sort key
+       if (order[kind])
+               key = $1 order[kind];
+       else
+               key = $1 "Z";
+
+       # get it sorted
+       print key "\t" $0 |& sort
+}
+
+END {
+       close(sort, "to")
+       while ((sort |& getline) > 0) {
+               # strip key
+               sub(/[^[:space:]]*[[:space:]]*/, "")
+               print $0
+       }
+       close(sort)
+}
+
scripts/tags.sh +4 -1
index 4e18ae5282a6..51087c3d8b1e 100755
@@ -251,8 +251,10 @@ setup_regex()
 
 exuberant()
 {
+       (
+
        setup_regex exuberant asm c
-       all_target_sources | xargs $1 -a                        \
+       all_target_sources | xargs $1                           \
        -I __initdata,__exitdata,__initconst,__ro_after_init    \
        -I __initdata_memblock                                  \
        -I __refdata,__attribute,__maybe_unused,__always_unused \
@@ -266,12 +268,15 @@ exuberant()
        -I DEFINE_TRACE,EXPORT_TRACEPOINT_SYMBOL,EXPORT_TRACEPOINT_SYMBOL_GPL \
        -I static,const                                         \
        --extra=+fq --c-kinds=+px --fields=+iaS --langmap=c:+.h \
+       --sort=no -o -                                          \
        "${regex[@]}"
 
        setup_regex exuberant kconfig

scripts/tags.sh exuberant() · line 282

Quoted 10 lines
scripts/tags.sh +4 -2
-       all_kconfigs | xargs $1 -a                              \
-       --langdef=kconfig --language-force=kconfig "${regex[@]}"
+       all_kconfigs | xargs $1                                 \
+       --langdef=kconfig --language-force=kconfig --sort=no    \
+       -o - "${regex[@]}"
 
+       ) | scripts/sort-tags.awk > tags
 }
 
 emacs()

Sorry for the long delay.

First, this patch breaks 'make TAGS'
if 'etags' is a symlink to exuberant ctags.

masahiro@oscar:~/ref/linux$ etags --version
Exuberant Ctags 5.9~svn20110310, Copyright (C) 1996-2009 Darren Hiebert
  Addresses: <[email protected]>, http://ctags.sourceforge.net
  Optional compiled features: +wildcards, +regex
masahiro@oscar:~/ref/linux$ make TAGS
  GEN     TAGS
etags: Warning: include/linux/seqlock.h:738: null expansion of name pattern "\2"
sed: can't read TAGS: No such file or directory
make: *** [Makefile:1820: TAGS] Error 2

The reason is the hard-coded ' > tags',
and easy to fix.

But, honestly, I am not super happy about this patch.

Reason 1
In my understanding, sorting by the tag kind only works for ctags. My favorite editor is emacs.
(Do not get me wrong. I do not intend emacs vs vi war). So, I rather do 'make TAGS' instead of 'make tags', but this solution would not work for etags because etags has a different format.
So, I'd rather want to see a more general solution.

Reason 2
We would have more messy code, mixing two files/languages

When is it useful to tag structure members?

If they are really annoying, why don't we delete them instead of moving them to the bottom of the tag file?

Quoted 5 lines

I attached an alternative solution,
and wrote up my thoughts in the log.

What do you think?

Sorry, the commit log of the attachment was wrong.

The correct sentence is:

"OK, [3] clearly explained why 'p' is useful, but turned --c-kinds=-px into --c-kinds=+px. So, 'x' was also (accidentally?) enabled."

Signature · 2 lines
Best Regards
Masahiro Yamada
#5 lore raw
In reply to Masahiro Yamada
Quoted 11 lines · Masahiro Yamada

Sorry for the long delay.

First, this patch breaks 'make TAGS'
if 'etags' is a symlink to exuberant ctags.

masahiro@oscar:~/ref/linux$ etags --version
Exuberant Ctags 5.9~svn20110310, Copyright (C) 1996-2009 Darren Hiebert
  Addresses: <[email protected]>, http://ctags.sourceforge.net
  Optional compiled features: +wildcards, +regex
Quoted 8 lines
masahiro@oscar:~/ref/linux$ make TAGS
  GEN     TAGS
etags: Warning: include/linux/seqlock.h:738: null expansion of name pattern "\2"
sed: can't read TAGS: No such file or directory
make: *** [Makefile:1820: TAGS] Error 2

The reason is the hard-coded ' > tags',
and easy to fix.

Ah, my bad, I forgot to check.

Quoted 10 lines

But, honestly, I am not super happy about this patch.

Reason 1
In my understanding, sorting by the tag kind only works for ctags. My favorite editor is emacs.
(Do not get me wrong. I do not intend emacs vs vi war). So, I rather do 'make TAGS' instead of 'make tags', but this solution would not work for etags because etags has a different format.
So, I'd rather want to see a more general solution.

It might be possible that emacs' tags implementation can already do this natively. Initially I tried to fix this in vim, with a macro, but I couldn't get access to the 'kind' tag.

Quoted 2 lines

Reason 2
We would have more messy code, mixing two files/languages

I could try and write the whole thing in bash I suppose.

Quoted 1 line

When is it useful to tag structure members?

Often, just not when there is a naming conflict.

Quoted 2 lines

If they are really annoying, why don't we delete them instead of moving them to the bottom of the tag file?

Because they're really useful :-)

Quoted 23 lines

I attached an alternative solution,
and wrote up my thoughts in the log.

What do you think?

Exuberant Ctags supports the following kinds of tags:

  $ ctags --list-kinds=c
  c  classes
  d  macro definitions
  e  enumerators (values inside an enumeration)
  f  function definitions
  g  enumeration names
  l  local variables [off]
  m  class, struct, and union members
  n  namespaces
  p  function prototypes [off]
  s  structure names
  t  typedefs
  u  union names
  v  variable definitions
  x  external and forward variable declarations [off]
Quoted 1 line

This commit excludes 'm', 'v', and 'x'.

So my main beef is with m vs s conflicts (they're pretty prevalent), removing v is insane, but even removing m is undesired IMO.

Quoted 1 line

Very much not I'm afraid. I really do like my tags, it's just that I'd like to have a set precedence when there's a naming conflict.

My claim is that a structure definition is more interesting than a member variable, not that member variables are not interesting.

#6 lore raw
Quoted 77 lines · On Thu, Sep 3, 2020 at 1:26 AM <[email protected]> wrote:
Quoted 11 lines · Masahiro Yamada

Sorry for the long delay.

First, this patch breaks 'make TAGS'
if 'etags' is a symlink to exuberant ctags.

masahiro@oscar:~/ref/linux$ etags --version
Exuberant Ctags 5.9~svn20110310, Copyright (C) 1996-2009 Darren Hiebert
  Addresses: <[email protected]>, http://ctags.sourceforge.net
  Optional compiled features: +wildcards, +regex
Quoted 8 lines
masahiro@oscar:~/ref/linux$ make TAGS
  GEN     TAGS
etags: Warning: include/linux/seqlock.h:738: null expansion of name pattern "\2"
sed: can't read TAGS: No such file or directory
make: *** [Makefile:1820: TAGS] Error 2

The reason is the hard-coded ' > tags',
and easy to fix.

Ah, my bad, I forgot to check.

Quoted 10 lines

But, honestly, I am not super happy about this patch.

Reason 1
In my understanding, sorting by the tag kind only works for ctags. My favorite editor is emacs.
(Do not get me wrong. I do not intend emacs vs vi war). So, I rather do 'make TAGS' instead of 'make tags', but this solution would not work for etags because etags has a different format.
So, I'd rather want to see a more general solution.

It might be possible that emacs' tags implementation can already do this natively. Initially I tried to fix this in vim, with a macro, but I couldn't get access to the 'kind' tag.

Quoted 2 lines

Reason 2
We would have more messy code, mixing two files/languages

I could try and write the whole thing in bash I suppose.

Quoted 1 line

When is it useful to tag structure members?

Often, just not when there is a naming conflict.

Quoted 2 lines

If they are really annoying, why don't we delete them instead of moving them to the bottom of the tag file?

Because they're really useful :-)

Quoted 23 lines

I attached an alternative solution,
and wrote up my thoughts in the log.

What do you think?

Exuberant Ctags supports the following kinds of tags:

  $ ctags --list-kinds=c
  c  classes
  d  macro definitions
  e  enumerators (values inside an enumeration)
  f  function definitions
  g  enumeration names
  l  local variables [off]
  m  class, struct, and union members
  n  namespaces
  p  function prototypes [off]
  s  structure names
  t  typedefs
  u  union names
  v  variable definitions
  x  external and forward variable declarations [off]
Quoted 6 lines
Quoted 1 line

This commit excludes 'm', 'v', and 'x'.

So my main beef is with m vs s conflicts (they're pretty prevalent), removing v is insane, but even removing m is undesired IMO.

Quoted 1 line

Sorry, I intended Reported-by, not Reviewed-by.

Quoted 5 lines

Very much not I'm afraid. I really do like my tags, it's just that I'd like to have a set precedence when there's a naming conflict.

My claim is that a structure definition is more interesting than a member variable, not that member variables are not interesting.

OK, but is there any idea
to make the code cleaner and easier to maintain?

People play with whatever they want to do in this script.

f81b1be40c44b33b9706d64c117edd29e627ad12
introduced file-level ordering.

4f491bb6ea2aef2f5b184f385904a73796d98554
broke it.
(I pointed it out in the review,
but akpm picked it anyway.)

Now, here is the tag-level ordering
(only for exuberant ctags).

Contributors stop caring after their code is merged, but maintaining it is tiring.

Will re-implementing your sorting logic
in bash look cleaner?

Or, in hindsight, we should have used python or perl?

--
Best Regards
Masahiro Yamada

#7 lore raw
Quoted 2 lines · Masahiro Yamada

Contributors stop caring after their code is merged, but maintaining it is tiring.

This seems to hold in general :/

Quoted 2 lines

Will re-implementing your sorting logic
in bash look cleaner?

Possibly, I can try, we'll see.

Quoted 1 line

Or, in hindsight, we should have used python or perl?

I don't speak either :-/.

I googled to see if there is a python/perl ctags implementation we can 'borrow' and found https://github.com/universal-ctags/ctags instead. That seems to be a continuation of exhuberant ctags, I can also try if they're interested in --sort-kinds or something like that.

#8 lore raw
Quoted 6 lines · On Thu, Sep 03, 2020 at 09:26:04AM +0200, [email protected] wrote:
Quoted 2 lines · Masahiro Yamada

Will re-implementing your sorting logic
in bash look cleaner?

Possibly, I can try, we'll see.

It is somewhat cleaner, but it is _abysmally_ slow. Bash sucks :-(

It is still broken in all the same ways as before, I figured I'd get it 'working' first.

---

scripts/tags.sh +61 -2
index 32d3f53af10b..ec2688b3441a 100755
@@ -239,10 +239,65 @@ setup_regex()
 	done
 }
 
+sort_tags()
+{
+	export LC_ALL=C
+
+	# start concurrent sort
+	coproc sort
+	# HACK, clone sort output into 3 to ensure we can still read it
+	# after sort terminates
+	exec 3<&${COPROC[0]}
+
+	while read tag file rest;
+	do
+		local tmp=${rest#*;\"}
+
+		case "${tmp:1:1}" in # Precedence for 'C' kinds
+
+		c) order="A";; # classes
+		s) order="B";; # structure names
+		t) order="C";; # typedefs
+		g) order="D";; # enumeration names
+		u) order="E";; # union names
+		n) order="F";; # namespaces
+
+		f) order="G";; # function definitions
+		p) order="H";; # function prototypes
+		d) order="I";; # macro definitions
+
+		e) order="J";; # enumerators (values inside an enumeration)
+		m) order="K";; # class, struct and union members
+		v) order="L";; # variable definitions
+
+		l) order="M";; # local variables [off]
+		x) order="N";; # external and forward variable declarations
+
+		*) order="Z";;
+
+		esac
+
+		# write to sort with a new sort-key prepended
+		echo "${tag}${order}	${tag}	${file}	${rest}" >&${COPROC[1]}
+	done
+
+	# close sort input
+	exec {COPROC[1]}>&-
+
+	# consume sort output
+	while read -u 3 key line;
+	do
+		# strip the sort-key
+		echo "${line}"
+	done
+}
+
 exuberant()
 {
+	(
+
 	setup_regex exuberant asm c
-	all_target_sources | xargs $1 -a                        \
+	all_target_sources | xargs $1                           \
 	-I __initdata,__exitdata,__initconst,__ro_after_init	\
 	-I __initdata_memblock					\
 	-I __refdata,__attribute,__maybe_unused,__always_unused \
@@ -256,12 +311,16 @@ exuberant()
 	-I DEFINE_TRACE,EXPORT_TRACEPOINT_SYMBOL,EXPORT_TRACEPOINT_SYMBOL_GPL \
 	-I static,const						\
 	--extra=+fq --c-kinds=+px --fields=+iaS --langmap=c:+.h \
+	--sort=no -o -						\
 	"${regex[@]}"
 
 	setup_regex exuberant kconfig
-	all_kconfigs | xargs $1 -a                              \
+	all_kconfigs | xargs $1                                 \
+	--sort=no -o -						\
 	--langdef=kconfig --language-force=kconfig "${regex[@]}"
 
+	) | sort_tags > tags
+
 }
 
 emacs()