Skip to contentSkip to conversation

powerpc: Instrument Hypervisor Calls

PATCH In review Mike Kravetz 3 participants 8 messages Jun 22, 2006 – Jul 12, 2006

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover powerpc: Instrument Hypervisor Calls

Here is an updated version of the hcall instrumentation patches. Unfortunately, I have not been able to spend as much time as I would like on this (and it seems that will continue).

This version addresses all comments received except Arnd's issue with an #ifdef for each function in the assembly file. I like Jimi's suggestion for creating macros(which could also minimize the #ifdefs), but have not got around to implementing this. Suggestions of others way to accomplish this are welcome.

Statistic files are moved to debugfs, and remain split out by CPU. After some thought, disable/enable of preemption was added around the statistic gathering(mostly for timing).

Signature · 1 line
Mike
#1 lore raw

PATCH 1/3 powerpc: Instrument Hypervisor Calls: merge headers

Move all the Hypervisor call definitions to to a single header file.

--

drivers/net/ibmveth.h +0 -10
diff -Naupr linux-2.6.17.1/drivers/net/ibmveth.h linux-2.6.17.1.work/drivers/net/ibmveth.h
@@ -41,16 +41,6 @@
 #define IbmVethMcastRemoveFilter     0x2UL
 #define IbmVethMcastClearFilterTable 0x3UL
 
-/* hcall numbers */
-#define H_VIO_SIGNAL             0x104
-#define H_REGISTER_LOGICAL_LAN   0x114
-#define H_FREE_LOGICAL_LAN       0x118
-#define H_ADD_LOGICAL_LAN_BUFFER 0x11C
-#define H_SEND_LOGICAL_LAN       0x120
-#define H_MULTICAST_CTRL         0x130
-#define H_CHANGE_LOGICAL_LAN_MAC 0x14C
-#define H_FREE_LOGICAL_LAN_BUFFER 0x1D4
-
 /* hcall macros */
 #define h_register_logical_lan(ua, buflst, rxq, fltlst, mac) \
   plpar_hcall_norets(H_REGISTER_LOGICAL_LAN, ua, buflst, rxq, fltlst, mac)
include/asm-powerpc/hvcall.h +9 -0
diff -Naupr linux-2.6.17.1/include/asm-powerpc/hvcall.h linux-2.6.17.1.work/include/asm-powerpc/hvcall.h
@@ -155,9 +155,15 @@
 #define H_VIO_SIGNAL		0x104
 #define H_SEND_CRQ		0x108
 #define H_COPY_RDMA		0x110
+#define H_REGISTER_LOGICAL_LAN	0x114
+#define H_FREE_LOGICAL_LAN	0x118
+#define H_ADD_LOGICAL_LAN_BUFFER 0x11C
+#define H_SEND_LOGICAL_LAN	0x120
+#define H_MULTICAST_CTRL	0x130
 #define H_SET_XDABR		0x134
 #define H_STUFF_TCE		0x138
 #define H_PUT_TCE_INDIRECT	0x13C
+#define H_CHANGE_LOGICAL_LAN_MAC 0x14C
 #define H_VTERM_PARTNER_INFO	0x150
 #define H_REGISTER_VTERM	0x154
 #define H_FREE_VTERM		0x158
@@ -187,11 +193,14 @@
 #define H_GET_HCA_INFO          0x1B8
 #define H_GET_PERF_COUNT        0x1BC
 #define H_MANAGE_TRACE          0x1C0
+#define H_FREE_LOGICAL_LAN_BUFFER 0x1D4
 #define H_QUERY_INT_STATE       0x1E4
 #define H_POLL_PENDING		0x1D8
 #define H_JOIN			0x298
 #define H_ENABLE_CRQ		0x2B0
 
+#define MAX_HCALL_OPCODES	(H_ENABLE_CRQ >> 2)
+
 #ifndef __ASSEMBLY__
 
 /* plpar_hcall() -- Generic call interface using above opcodes
#2 lore raw

PATCH 2/3 powerpc: Instrument Hypervisor Calls: add wrappers

In reply to Mike Kravetz

Add wrappers which perform the actual hypervisor call instrumentation.

--

arch/powerpc/Kconfig.debug +13 -0
diff -Naupr linux-2.6.17.1/arch/powerpc/Kconfig.debug linux-2.6.17.1.work/arch/powerpc/Kconfig.debug
@@ -18,6 +18,19 @@ config DEBUG_STACK_USAGE
 
 	  This option will slow down process creation somewhat.
 
+config HCALL_STATS
+	bool "Hypervisor call instrumentation"
+	depends on PPC_PSERIES && DEBUG_FS
+	help
+	  Adds code to keep track of the number of hypervisor calls made and
+	  the amount of time spent in hypervisor calls.  A directory named
+	  hcall_inst is added at the root of the debugfs filesystem.  Within
+	  the hcall_inst directory are files that contain CPU specific call
+	  statistics.
+
+	  This option will add a small amount of overhead to all hypervisor
+	  calls.
+
 config DEBUGGER
 	bool "Enable debugger hooks"
 	depends on DEBUG_KERNEL
arch/powerpc/platforms/pseries/Makefile +1 -0
diff -Naupr linux-2.6.17.1/arch/powerpc/platforms/pseries/Makefile linux-2.6.17.1.work/arch/powerpc/platforms/pseries/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_EEH)	+= eeh.o eeh_cache.o e
 
 obj-$(CONFIG_HVC_CONSOLE)	+= hvconsole.o
 obj-$(CONFIG_HVCS)		+= hvcserver.o
+obj-$(CONFIG_HCALL_STATS)	+= hvCall_inst.o
arch/powerpc/platforms/pseries/hvCall.S +32 -0
diff -Naupr linux-2.6.17.1/arch/powerpc/platforms/pseries/hvCall.S linux-2.6.17.1.work/arch/powerpc/platforms/pseries/hvCall.S
@@ -25,7 +25,11 @@
 			unsigned long *out2,		R9
 			unsigned long *out3);		R10
  */
+#ifdef CONFIG_HCALL_STATS
+_GLOBAL(plpar_hcall_base)
+#else
 _GLOBAL(plpar_hcall)
+#endif
 	HMT_MEDIUM
 
 	mfcr	r0
@@ -52,7 +56,19 @@ _GLOBAL(plpar_hcall)
 
 
 /* Simple interface with no output values (other than status) */
+#ifdef CONFIG_HCALL_STATS
+/*
+ * A special 'indirect' call to a C based wrapper if statistics are desired.
+ * See plpar_hcall_norets_C function header for more details.
+ */
+_GLOBAL(plpar_hcall_norets)
+	b	plpar_hcall_norets_C
+
+
+_GLOBAL(plpar_hcall_norets_base)
+#else
 _GLOBAL(plpar_hcall_norets)
+#endif
 	HMT_MEDIUM
 
 	mfcr	r0
@@ -76,7 +92,11 @@ _GLOBAL(plpar_hcall_norets)
 			unsigned long arg8,		112(R1)
 			unsigned long *out1);		120(R1)
  */
+#ifdef CONFIG_HCALL_STATS
+_GLOBAL(plpar_hcall_8arg_2ret_base)
+#else
 _GLOBAL(plpar_hcall_8arg_2ret)
+#endif
 	HMT_MEDIUM
 
 	mfcr	r0
@@ -102,7 +122,11 @@ _GLOBAL(plpar_hcall_8arg_2ret)
 		 	unsigned long *out3,		R10
 		 	unsigned long *out4);		112(R1)
  */
+#ifdef CONFIG_HCALL_STATS
+_GLOBAL(plpar_hcall_4out_base)
+#else
 _GLOBAL(plpar_hcall_4out)
+#endif
 	HMT_MEDIUM
 
 	mfcr	r0
@@ -144,7 +168,11 @@ _GLOBAL(plpar_hcall_4out)
 			 unsigned long *out6,		102(R1)
 			 unsigned long *out7);		100(R1)
 */
+#ifdef CONFIG_HCALL_STATS
+_GLOBAL(plpar_hcall_7arg_7ret_base)
+#else
 _GLOBAL(plpar_hcall_7arg_7ret)
+#endif
 	HMT_MEDIUM
 
 	mfcr	r0
@@ -193,7 +221,11 @@ _GLOBAL(plpar_hcall_7arg_7ret)
 			 unsigned long *out8,		 94(R1)
 		         unsigned long *out9,            92(R1)
 */
+#ifdef CONFIG_HCALL_STATS
+_GLOBAL(plpar_hcall_9arg_9ret_base)
+#else
 _GLOBAL(plpar_hcall_9arg_9ret)
+#endif
 	HMT_MEDIUM
 
 	mfcr	r0
arch/powerpc/platforms/pseries/hvCall_inst.c +219 -0

219 lines not on this page · raw message

include/asm-powerpc/hvcall.h +80 -0

86 lines not on this page · raw message

#3 lore raw

PATCH 3/3 powerpc: Instrument Hypervisor Calls: add debugfs files

In reply to Mike Kravetz

Make statistics available via files in debugfs.

--

arch/powerpc/platforms/pseries/hvCall_inst.c +91 -0
diff -Naupr linux-2.6.17.1/arch/powerpc/platforms/pseries/hvCall_inst.c linux-2.6.17.1.work/arch/powerpc/platforms/pseries/hvCall_inst.c
@@ -217,3 +217,94 @@ long plpar_hcall_9arg_9ret(unsigned long
 	preempt_enable();
 	return rc;
 }
+
+/*
+ * Routines for displaying the statistics in debugfs
+ */
+static void *hc_start(struct seq_file *m, loff_t *pos)
+{
+	if ((int)*pos < MAX_HCALL_OPCODES)
+		return (void *)(unsigned long)(*pos + 1);
+
+	return NULL;
+}
+
+static void *hc_next(struct seq_file *m, void *p, loff_t * pos)
+{
+	++*pos;
+
+	return hc_start(m, pos);
+}
+
+static void hc_stop(struct seq_file *m, void *p)
+{
+}
+
+static int hc_show(struct seq_file *m, void *p)
+{
+	unsigned long h_num = (unsigned long)p;
+	struct hcall_stats *hs = (struct hcall_stats *)m->private;
+
+	if (hs[h_num].num_calls)
+		seq_printf(m, "%lu %lu %lu\n", h_num<<2, hs[h_num].num_calls,
+			   hs[h_num].total_time);
+
+	return 0;
+}
+
+static struct seq_operations hcall_inst_seq_ops = {
+        .start = hc_start,
+        .next  = hc_next,
+        .stop  = hc_stop,
+        .show  = hc_show
+};
+
+static int hcall_inst_seq_open(struct inode *inode, struct file *file)
+{
+	int rc;
+	struct seq_file *seq;
+
+	rc = seq_open(file, &hcall_inst_seq_ops);
+	seq = file->private_data;
+	seq->private = file->f_dentry->d_inode->u.generic_ip;
+
+	return rc;
+}
+
+static struct file_operations hcall_inst_seq_fops = {
+	.open = hcall_inst_seq_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = seq_release,
+};
+
+#define	HCALL_ROOT_DIR		"hcall_inst"
+#define CPU_NAME_BUF_SIZE	32
+
+static int __init hcall_inst_init(void)
+{
+	struct dentry *hcall_root;
+	struct dentry *hcall_file;
+	char cpu_name_buf[CPU_NAME_BUF_SIZE];
+	int cpu;
+
+	if (!firmware_has_feature(FW_FEATURE_LPAR))
+		return 0;
+
+	hcall_root = debugfs_create_dir(HCALL_ROOT_DIR, NULL);
+	if (!hcall_root)
+		return -ENOMEM;
+
+	for_each_cpu(cpu) {
+		snprintf(cpu_name_buf, CPU_NAME_BUF_SIZE, "cpu%d", cpu);
+		hcall_file = debugfs_create_file(cpu_name_buf, S_IRUGO,
+						 hcall_root,
+						 per_cpu(hcall_stats, cpu),
+						 &hcall_inst_seq_fops);
+		if (!hcall_file)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+__initcall(hcall_inst_init);
#4 lore raw
In reply to Mike Kravetz
Quoted 1 line

Statistic files are moved to debugfs, and remain split out by CPU.

Having done some if this myself long ago, I really like to see these per-cpu statistics. "Not useful" in real life, but for most testcases they're invaluable. It would be useful to also add a "total" column as well, of course, but it can also be done in userland.

Segher

#5 lore raw
In reply to Mike Kravetz
Quoted 2 lines · Mike Kravetz

This version addresses all comments received except Arnd's issue with an #ifdef for each function in the assembly file.

I was thinking of changing the names of all the assembly routines from plpar_hcall_*() to plpar_hcall_*_asm(). The instrumented version of the routines would be named plpar_hcall_*_inst(). Then, the header file would contain definitions such as:

#ifdef CONFIG_HCALL_STATS
#define plpar_hcall_*() plpar_hcall_*_inst()
.
#else
#define plpar_hcall_*() plpar_hcall_*_asm()
.
#endif

Is that any better than all the individual #ifdefs in the .S file? Is it still too ugly?

I'm open to any suggestions.

Thanks,

Signature · 1 line
Mike
#6 lore raw
Quoted 17 lines · Mike Kravetz
Quoted 2 lines · Mike Kravetz

This version addresses all comments received except Arnd's issue with an #ifdef for each function in the assembly file.

I was thinking of changing the names of all the assembly routines from plpar_hcall_*() to plpar_hcall_*_asm(). The instrumented version of the routines would be named plpar_hcall_*_inst(). Then, the header file would contain definitions such as:

#ifdef CONFIG_HCALL_STATS
#define plpar_hcall_*() plpar_hcall_*_inst()
.
#else
#define plpar_hcall_*() plpar_hcall_*_asm()
.
#endif

Quoted 3 lines

Is that any better than all the individual #ifdefs in the .S file? Is it still too ugly?

I guess it's better to have the #ifdef in the header file, but then again, you could just as well save some source lines doing

#ifndef CONFIG_HCALL_STATS
#define plpar_hcalldef(x) plpar_call_ ## x ## _asm #else
#define plpar_hcalldef(x) plpar_call_ ## x ## _inst #endif

#define plpar_call_foo plpar_hcalldef(foo)

	Arnd <><
#7 lore raw
In reply to Mike Kravetz
Quoted 2 lines · Mike Kravetz

#ifdef CONFIG_HCALL_STATS
#define plpar_hcall_*() plpar_hcall_*_inst()

To answer my own question, this won't work because plpar_hcall_*() routines are exported. So, marcros can't be used. Still thinking about the best way to juggle function names for instrumentation.

Signature · 1 line
Mike