coccicheck: extend with parmap and indexing heuristics

index aa5e78fba270..02602381be59 100755
@@ -12,8 +12,8 @@ if [ ! -x "$SPATCH" ]; then
     exit 1
 fi
 
-trap kill_running SIGTERM SIGINT
-declare -a SPATCH_PID
+USE_JOBS="no"
+$SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes"
 
 # The verbosity may be set by the environmental parameter V=
 # as for example with 'make V=1 coccicheck'

@@ -82,7 +82,21 @@ if [ "$ONLINE" = "0" ] ; then
     echo ''
 fi
 
-run_cmd() {
+if [ "$USE_JOBS" = "no" ]; then
+	trap kill_running SIGTERM SIGINT
+	declare -a SPATCH_PID
+else
+	OPTIONS="$OPTIONS --jobs $NPROC --chunksize 1"
+fi
+
+run_cmd_parmap() {
+	if [ $VERBOSE -ne 0 ] ; then
+		echo "Running ($NPROC in parallel): $@"
+	fi
+	$@ 2> ${DIR}/coccicheck.$$.err
+}
+
+run_cmd_old() {
 	local i
 	if [ $VERBOSE -ne 0 ] ; then
 		echo "Running ($NPROC in parallel): $@"

@@ -97,6 +111,14 @@ run_cmd() {
 	wait
 }
 
+run_cmd() {
+	if [ "$USE_JOBS" = "yes" ]; then
+		run_cmd_parmap $@
+	else
+		run_cmd_old $@
+	fi
+}
+
 kill_running() {
 	for i in $(seq 0 $(( NPROC - 1 )) ); do
 		if [ $VERBOSE -eq 2 ] ; then

new file mode 100755
index 000000000000..5fd24e49f31b
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+DIR=$(dirname $(readlink -f $0))
+DIR="${DIR}/../"
+
+GLIMPSEINDEX="`which ${GLIMPSEINDEX:=glimpseindex}`"
+if [ ! -x "$GLIMPSEINDEX" ]; then
+	echo 'glimpseindex can be obtained at https://github.com/mcgrof/glimpse.git'
+	exit 1
+fi
+
+find $DIR/* -name "*.[ch]" | $GLIMPSEINDEX -o -H . -F

index 02602381be59..6d75fd8580a6 100755
@@ -5,6 +5,8 @@
 # version 1.0.0-rc11.
 #
 
+DIR=$(dirname $(readlink -f $0))
+DIR="${DIR}/../"
 SPATCH="`which ${SPATCH:=spatch}`"
 
 if [ ! -x "$SPATCH" ]; then

@@ -15,6 +17,80 @@ fi
 USE_JOBS="no"
 $SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes"
 
+# Indexing USE_* optimizations heuristics.
+#
+# Linux runs on git (TM). However, if you have supplemental indexing options
+# you may use them to help Coccinelle further. If you are using Coccinelle
+# within a target git tree --use-gitrep will be used, and this should
+# suffice for most uses. If you however want optimal performance do
+# consider embracing a supplemental indexing as part of your development.
+# For instance glimpse, and idutils can be used, however you should
+# be sure to update the indexes as often as you update your git tree to
+# ensure your indexes are not stale.
+#
+# idutils is currently not as efficient as glimpse because the query language
+# for glimpse is simpler, so when idutils is used more filtering has to be
+# done at the ocaml level within Coccinelle. Glimpse allows queries that are
+# arbitrary formulas, up to a limited level of complexity, involving both
+# && and ||. For idutils, Coccinelle runs lid intersections on the result.
+#
+# You can override these heuristics with COCCI_INDEX="--use-gitgrep" for
+# example. This will force to use --use-gitgrep even if you have a glimpse
+# index. Other examples:
+#
+# $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-coccigrep"
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-idutils ID"
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-glimpse"
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-gitgrep"
+#
+# The order of heuristics for indexing used by coccicheck is listed below.
+#
+# 0. Glimpse currently should outperform all indexing options. so if a glimpse
+#    index is used we use it. Refer to Linux scripts/glimpse.sh for details.
+#    If you think you should be getting better performance with glimpse than
+#    what you would expect inspect the stderr log file for cocciecheck:
+#
+#    ${DIR}/coccicheck.$$.err
+#
+#    If glimpse is running correctly there should be very few occurrences
+#    of "Skipping", also coccinelle will inform you if it could not use
+#    glimpse. As an example an output of the following would indicate glimpse
+#    was properly used on ${DIR}/coccicheck.$$.err :
+#
+#    There are matches to 1252 out of 47281 files
+#    glimpse request = request_threaded_irq
+#
+# 1. Use idutils next. You'll need to generate an index using either of these:
+#
+#	a) mkid -s
+#       By default this dumps the index into ./ID
+#
+#	b) mkid -i C --output .id-utils.index *
+#       This method is provided with coccinelle repo on
+#       scripts/idutils_index.sh
+#
+# 2. Next best is --use-gitgrep and if you are working within a git tree
+#    this will be used by default.
+#
+# 3. By default coccinelle internally uses --use-coccigrep if no indexing
+#    options are requested and your version of coccinelle supports it so we
+#    do not need to be specific about requesting that as a fallback mechanism.
+#    Use of --use-coccigrep is comparable to --use-gitgrep.
+#
+# XXX: Glimpse is not well maintained. See if we can add similar indexing
+# features and query language glimpse supports to git.
+if [ "$COCCI_INDEX" = "" ] ; then
+	USE_GLIMPSE="no"
+	$SPATCH --help | grep "\-\-use\-glimpse" > /dev/null && [ -f $DIR/.glimpse_index ] && USE_GLIMPSE="yes"
+
+	USE_IDUTILS="no"
+	$SPATCH --help | grep "\-\-use\-idutils" > /dev/null && [ -f $DIR/ID -o -f $DIR/.id-utils.index ] && USE_IDUTILS="yes"
+
+	USE_GITGREP="no"
+	$SPATCH --help | grep "\-\-use\-gitgrep" > /dev/null && [ -d $DIR/.git ] && USE_GITGREP="yes"
+fi
+
 # The verbosity may be set by the environmental parameter V=
 # as for example with 'make V=1 coccicheck'
 

@@ -89,6 +165,27 @@ else
 	OPTIONS="$OPTIONS --jobs $NPROC --chunksize 1"
 fi
 
+# Check COCCI_INDEX first to manual override, otherwise rely on
+# internal heuristics documented above.
+if [ "$COCCI_INDEX" != "" ] ; then
+	OPTIONS="$OPTIONS $COCCI_INDEX"
+elif [ "$USE_GLIMPSE" = "yes" ]; then
+	OPTIONS="$OPTIONS --use-glimpse"
+elif [ "$USE_IDUTILS" = "yes" ]; then
+	index=""
+	if [ -f $DIR/ID ]; then
+		index="$DIR/ID"
+	elif [ -f $DIR/.id-utils.index ]; then
+		index="$DIR/.id-utils.index"
+	else
+		echo "idutils index not found, expected: $DIR/ID or $DIR/.id-utils.index"
+		exit 1
+	fi
+	OPTIONS="$OPTIONS --use-idutils $index"
+elif [ "$USE_GITGREP" = "yes" ]; then
+	OPTIONS="$OPTIONS --use-gitgrep"
+fi
+
 run_cmd_parmap() {
 	if [ $VERBOSE -ne 0 ] ; then
 		echo "Running ($NPROC in parallel): $@"

index 6d75fd8580a6..c6440138c6ea 100755
@@ -106,7 +106,26 @@ else
 	NPROC="$J"
 fi
 
-FLAGS="$SPFLAGS --very-quiet"
+# You can use SPFLAGS to append extra arguments to coccicheck.
+# A good example is if you want to debug your cocci script, you can
+# for instance use the following:
+#
+# $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci
+# $ time make coccicheck MODE=report SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c
+#
+# "--show-trying" should show you what rule is being processed as it goes, if
+# you have issues with a rule getting stuck you can then inspect the file:
+#
+# ${DIR}/coccicheck.$$.err
+#
+# which will have the profile output.
+#
+# --profile will not output if --very-quiet is used, so avoid it.
+if [[ $SPFLAGS  == *"--profile"* ]]; then
+	FLAGS="$SPFLAGS"
+else
+	FLAGS="$SPFLAGS --very-quiet"
+fi
 
 # spatch only allows include directories with the syntax "-I include"
 # while gcc also allows "-Iinclude" and "-include include"

index c6440138c6ea..37db7ac1dfae 100755
@@ -1,9 +1,14 @@
 #!/bin/bash
-
+# Linux kernel coccicheck
+#
+# For more detailed documentation refer to:
+#
+# https://bottest.wiki.kernel.org/coccicheck
+#
+# This documentation always refers to the linux-next version of the script.
 #
 # This script requires at least spatch
 # version 1.0.0-rc11.
-#
 
 DIR=$(dirname $(readlink -f $0))
 DIR="${DIR}/../"

index dd85a455b2ba..aa5e78fba270 100755
@@ -7,6 +7,11 @@
 
 SPATCH="`which ${SPATCH:=spatch}`"
 
+if [ ! -x "$SPATCH" ]; then
+    echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/'
+    exit 1
+fi
+
 trap kill_running SIGTERM SIGINT
 declare -a SPATCH_PID
 

@@ -51,11 +56,6 @@ if [ "$KBUILD_EXTMOD" != "" ] ; then
     OPTIONS="--patch $srctree $OPTIONS"
 fi
 
-if [ ! -x "$SPATCH" ]; then
-    echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/'
-    exit 1
-fi
-
 if [ "$MODE" = "" ] ; then
     if [ "$ONLINE" = "0" ] ; then
 	echo 'You have not explicitly specified the mode to use. Using default "report" mode.'

index aa5e78fba270..02602381be59 100755
@@ -12,8 +12,8 @@ if [ ! -x "$SPATCH" ]; then
     exit 1
 fi
 
-trap kill_running SIGTERM SIGINT
-declare -a SPATCH_PID
+USE_JOBS="no"
+$SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes"
 
 # The verbosity may be set by the environmental parameter V=
 # as for example with 'make V=1 coccicheck'

@@ -82,7 +82,21 @@ if [ "$ONLINE" = "0" ] ; then
     echo ''
 fi
 
-run_cmd() {
+if [ "$USE_JOBS" = "no" ]; then
+	trap kill_running SIGTERM SIGINT
+	declare -a SPATCH_PID
+else
+	OPTIONS="$OPTIONS --jobs $NPROC --chunksize 1"
+fi
+
+run_cmd_parmap() {
+	if [ $VERBOSE -ne 0 ] ; then
+		echo "Running ($NPROC in parallel): $@"
+	fi
+	$@ 2> ${DIR}/coccicheck.$$.err
+}
+
+run_cmd_old() {
 	local i
 	if [ $VERBOSE -ne 0 ] ; then
 		echo "Running ($NPROC in parallel): $@"

@@ -97,6 +111,14 @@ run_cmd() {
 	wait
 }
 
+run_cmd() {
+	if [ "$USE_JOBS" = "yes" ]; then
+		run_cmd_parmap $@
+	else
+		run_cmd_old $@
+	fi
+}
+
 kill_running() {
 	for i in $(seq 0 $(( NPROC - 1 )) ); do
 		if [ $VERBOSE -eq 2 ] ; then

new file mode 100755
index 000000000000..5fd24e49f31b
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+DIR=$(dirname $(readlink -f $0))
+DIR="${DIR}/../"
+
+GLIMPSEINDEX="`which ${GLIMPSEINDEX:=glimpseindex}`"
+if [ ! -x "$GLIMPSEINDEX" ]; then
+	echo 'glimpseindex can be obtained at https://github.com/mcgrof/glimpse.git'
+	exit 1
+fi
+
+find $DIR/* -name "*.[ch]" | $GLIMPSEINDEX -o -H . -F

index 02602381be59..6d75fd8580a6 100755
@@ -5,6 +5,8 @@
 # version 1.0.0-rc11.
 #
 
+DIR=$(dirname $(readlink -f $0))
+DIR="${DIR}/../"
 SPATCH="`which ${SPATCH:=spatch}`"
 
 if [ ! -x "$SPATCH" ]; then

@@ -15,6 +17,80 @@ fi
 USE_JOBS="no"
 $SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes"
 
+# Indexing USE_* optimizations heuristics.
+#
+# Linux runs on git (TM). However, if you have supplemental indexing options
+# you may use them to help Coccinelle further. If you are using Coccinelle
+# within a target git tree --use-gitrep will be used, and this should
+# suffice for most uses. If you however want optimal performance do
+# consider embracing a supplemental indexing as part of your development.
+# For instance glimpse, and idutils can be used, however you should
+# be sure to update the indexes as often as you update your git tree to
+# ensure your indexes are not stale.
+#
+# idutils is currently not as efficient as glimpse because the query language
+# for glimpse is simpler, so when idutils is used more filtering has to be
+# done at the ocaml level within Coccinelle. Glimpse allows queries that are
+# arbitrary formulas, up to a limited level of complexity, involving both
+# && and ||. For idutils, Coccinelle runs lid intersections on the result.
+#
+# You can override these heuristics with COCCI_INDEX="--use-gitgrep" for
+# example. This will force to use --use-gitgrep even if you have a glimpse
+# index. Other examples:
+#
+# $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-coccigrep"
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-idutils ID"
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-glimpse"
+# $ make coccicheck V=1 MODE=report COCCI_INDEX="--use-gitgrep"
+#
+# The order of heuristics for indexing used by coccicheck is listed below.
+#
+# 0. Glimpse currently should outperform all indexing options. so if a glimpse
+#    index is used we use it. Refer to Linux scripts/glimpse.sh for details.
+#    If you think you should be getting better performance with glimpse than
+#    what you would expect inspect the stderr log file for cocciecheck:
+#
+#    ${DIR}/coccicheck.$$.err
+#
+#    If glimpse is running correctly there should be very few occurrences
+#    of "Skipping", also coccinelle will inform you if it could not use
+#    glimpse. As an example an output of the following would indicate glimpse
+#    was properly used on ${DIR}/coccicheck.$$.err :
+#
+#    There are matches to 1252 out of 47281 files
+#    glimpse request = request_threaded_irq
+#
+# 1. Use idutils next. You'll need to generate an index using either of these:
+#
+#	a) mkid -s
+#       By default this dumps the index into ./ID
+#
+#	b) mkid -i C --output .id-utils.index *
+#       This method is provided with coccinelle repo on
+#       scripts/idutils_index.sh
+#
+# 2. Next best is --use-gitgrep and if you are working within a git tree
+#    this will be used by default.
+#
+# 3. By default coccinelle internally uses --use-coccigrep if no indexing
+#    options are requested and your version of coccinelle supports it so we
+#    do not need to be specific about requesting that as a fallback mechanism.
+#    Use of --use-coccigrep is comparable to --use-gitgrep.
+#
+# XXX: Glimpse is not well maintained. See if we can add similar indexing
+# features and query language glimpse supports to git.
+if [ "$COCCI_INDEX" = "" ] ; then
+	USE_GLIMPSE="no"
+	$SPATCH --help | grep "\-\-use\-glimpse" > /dev/null && [ -f $DIR/.glimpse_index ] && USE_GLIMPSE="yes"
+
+	USE_IDUTILS="no"
+	$SPATCH --help | grep "\-\-use\-idutils" > /dev/null && [ -f $DIR/ID -o -f $DIR/.id-utils.index ] && USE_IDUTILS="yes"
+
+	USE_GITGREP="no"
+	$SPATCH --help | grep "\-\-use\-gitgrep" > /dev/null && [ -d $DIR/.git ] && USE_GITGREP="yes"
+fi
+
 # The verbosity may be set by the environmental parameter V=
 # as for example with 'make V=1 coccicheck'
 

@@ -89,6 +165,27 @@ else
 	OPTIONS="$OPTIONS --jobs $NPROC --chunksize 1"
 fi
 
+# Check COCCI_INDEX first to manual override, otherwise rely on
+# internal heuristics documented above.
+if [ "$COCCI_INDEX" != "" ] ; then
+	OPTIONS="$OPTIONS $COCCI_INDEX"
+elif [ "$USE_GLIMPSE" = "yes" ]; then
+	OPTIONS="$OPTIONS --use-glimpse"
+elif [ "$USE_IDUTILS" = "yes" ]; then
+	index=""
+	if [ -f $DIR/ID ]; then
+		index="$DIR/ID"
+	elif [ -f $DIR/.id-utils.index ]; then
+		index="$DIR/.id-utils.index"
+	else
+		echo "idutils index not found, expected: $DIR/ID or $DIR/.id-utils.index"
+		exit 1
+	fi
+	OPTIONS="$OPTIONS --use-idutils $index"
+elif [ "$USE_GITGREP" = "yes" ]; then
+	OPTIONS="$OPTIONS --use-gitgrep"
+fi
+
 run_cmd_parmap() {
 	if [ $VERBOSE -ne 0 ] ; then
 		echo "Running ($NPROC in parallel): $@"

index 6d75fd8580a6..c6440138c6ea 100755
@@ -106,7 +106,26 @@ else
 	NPROC="$J"
 fi
 
-FLAGS="$SPFLAGS --very-quiet"
+# You can use SPFLAGS to append extra arguments to coccicheck.
+# A good example is if you want to debug your cocci script, you can
+# for instance use the following:
+#
+# $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci
+# $ time make coccicheck MODE=report SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c
+#
+# "--show-trying" should show you what rule is being processed as it goes, if
+# you have issues with a rule getting stuck you can then inspect the file:
+#
+# ${DIR}/coccicheck.$$.err
+#
+# which will have the profile output.
+#
+# --profile will not output if --very-quiet is used, so avoid it.
+if [[ $SPFLAGS  == *"--profile"* ]]; then
+	FLAGS="$SPFLAGS"
+else
+	FLAGS="$SPFLAGS --very-quiet"
+fi
 
 # spatch only allows include directories with the syntax "-I include"
 # while gcc also allows "-Iinclude" and "-include include"

index c6440138c6ea..37db7ac1dfae 100755
@@ -1,9 +1,14 @@
 #!/bin/bash
-
+# Linux kernel coccicheck
+#
+# For more detailed documentation refer to:
+#
+# https://bottest.wiki.kernel.org/coccicheck
+#
+# This documentation always refers to the linux-next version of the script.
 #
 # This script requires at least spatch
 # version 1.0.0-rc11.
-#
 
 DIR=$(dirname $(readlink -f $0))
 DIR="${DIR}/../"