Skip to contentSkip to conversation

xen arch header patches

RFC PATCH Applied [email protected] 7 participants 23 messages Oct 5, 2006 – Oct 17, 2006

lore
Share & tools
lore
#0 lore raw

cover xen arch header patches

This patchset reworks xen arch-*.h headers, so multiple of them can be included at the same time. x86_32 and x86_64 only for now. See individual patches for more info.

#1 lore raw

RFC PATCH 1/3 xen arch header rework.

This patch makes arch-specific types and structs be postfixed by the architecture name. The native ones are defined to the non-postfixed versions. Also changed struct elements to use uint{32,64}_t everywhere so we get the correct sizes no matter what sizeof(long) happens to be.

The goal of this work is to be able to include foreign arch headers within one source file and be able to deal with foreign structs in the xen tools.

For now I've started with x86_32 and x86_64, we'll need that first when supporting 32-on-64 paravirtualized guests.

---
 xen/include/public/arch-x86_32.h |  207 ++++++++++++++++++++++-----------------
 xen/include/public/arch-x86_64.h |  181 ++++++++++++++++++++--------------
 2 files changed, 227 insertions(+), 161 deletions(-)
xen/include/public/arch-x86_32.h +120 -87
Index: build-64-unstable-11624/xen/include/public/arch-x86_32.h
===================================================================
@@ -43,20 +43,6 @@
 #define get_xen_guest_handle(val, hnd)  do { val = (hnd).p; } while (0)
 #endif
 
-#ifndef __ASSEMBLY__
-/* Guest handles for primitive C types. */
-__DEFINE_XEN_GUEST_HANDLE(uchar, unsigned char);
-__DEFINE_XEN_GUEST_HANDLE(uint,  unsigned int);
-__DEFINE_XEN_GUEST_HANDLE(ulong, unsigned long);
-DEFINE_XEN_GUEST_HANDLE(char);
-DEFINE_XEN_GUEST_HANDLE(int);
-DEFINE_XEN_GUEST_HANDLE(long);
-DEFINE_XEN_GUEST_HANDLE(void);
-
-typedef unsigned long xen_pfn_t;
-DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
-#endif
-
 /*
  * SEGMENT DESCRIPTOR TABLES
  */
@@ -82,44 +68,21 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
 #define FLAT_RING3_DS 0xe033    /* GDT index 262 */
 #define FLAT_RING3_SS 0xe033    /* GDT index 262 */
 
-#define FLAT_KERNEL_CS FLAT_RING1_CS
-#define FLAT_KERNEL_DS FLAT_RING1_DS
-#define FLAT_KERNEL_SS FLAT_RING1_SS
-#define FLAT_USER_CS    FLAT_RING3_CS
-#define FLAT_USER_DS    FLAT_RING3_DS
-#define FLAT_USER_SS    FLAT_RING3_SS
-
-/*
- * Virtual addresses beyond this are not modifiable by guest OSes. The 
- * machine->physical mapping table starts at this address, read-only.
- */
-#ifdef CONFIG_X86_PAE
-#define __HYPERVISOR_VIRT_START 0xF5800000
-#define __MACH2PHYS_VIRT_START  0xF5800000
-#define __MACH2PHYS_VIRT_END    0xF6800000
-#else
-#define __HYPERVISOR_VIRT_START 0xFC000000
-#define __MACH2PHYS_VIRT_START  0xFC000000
-#define __MACH2PHYS_VIRT_END    0xFC400000
-#endif
-
-#ifndef HYPERVISOR_VIRT_START
-#define HYPERVISOR_VIRT_START mk_unsigned_long(__HYPERVISOR_VIRT_START)
-#endif
-
-#define MACH2PHYS_VIRT_START  mk_unsigned_long(__MACH2PHYS_VIRT_START)
-#define MACH2PHYS_VIRT_END    mk_unsigned_long(__MACH2PHYS_VIRT_END)
-#define MACH2PHYS_NR_ENTRIES  ((MACH2PHYS_VIRT_END-MACH2PHYS_VIRT_START)>>2)
-#ifndef machine_to_phys_mapping
-#define machine_to_phys_mapping ((unsigned long *)MACH2PHYS_VIRT_START)
-#endif
+#define FLAT_KERNEL_CS_X86_32  FLAT_RING1_CS
+#define FLAT_KERNEL_DS_X86_32  FLAT_RING1_DS
+#define FLAT_KERNEL_SS_X86_32  FLAT_RING1_SS
+#define FLAT_USER_CS_X86_32    FLAT_RING3_CS
+#define FLAT_USER_DS_X86_32    FLAT_RING3_DS
+#define FLAT_USER_SS_X86_32    FLAT_RING3_SS
 
 /* Maximum number of virtual CPUs in multi-processor guests. */
 #define MAX_VIRT_CPUS 32
 
 #ifndef __ASSEMBLY__
 
-typedef unsigned long xen_ulong_t;
+typedef uint32_t xen_ulong_x86_32_t;
+typedef uint32_t xen_pfn_x86_32_t;
+typedef uint64_t tsc_timestamp_x86_32_t; /* RDTSC timestamp */
 
 /*
  * Send an array of these to HYPERVISOR_set_trap_table()
@@ -128,16 +91,15 @@ typedef unsigned long xen_ulong_t;
 #define TI_GET_IF(_ti)       ((_ti)->flags & 4)
 #define TI_SET_DPL(_ti,_dpl) ((_ti)->flags |= (_dpl))
 #define TI_SET_IF(_ti,_if)   ((_ti)->flags |= ((!!(_if))<<2))
-struct trap_info {
+struct trap_info_x86_32 {
     uint8_t       vector;  /* exception vector                              */
     uint8_t       flags;   /* 0-3: privilege level; 4: clear event enable?  */
     uint16_t      cs;      /* code selector                                 */
-    unsigned long address; /* code offset                                   */
+    uint32_t      address; /* code offset                                   */
 };
-typedef struct trap_info trap_info_t;
-DEFINE_XEN_GUEST_HANDLE(trap_info_t);
+typedef struct trap_info_x86_32 trap_info_x86_32_t;
 
-struct cpu_user_regs {
+struct cpu_user_regs_x86_32 {
     uint32_t ebx;
     uint32_t ecx;
     uint32_t edx;
@@ -159,16 +121,13 @@ struct cpu_user_regs {
     uint16_t fs, _pad4;
     uint16_t gs, _pad5;
 };
-typedef struct cpu_user_regs cpu_user_regs_t;
-DEFINE_XEN_GUEST_HANDLE(cpu_user_regs_t);
-
-typedef uint64_t tsc_timestamp_t; /* RDTSC timestamp */
+typedef struct cpu_user_regs_x86_32 cpu_user_regs_x86_32_t;
 
 /*
  * The following is all CPU context. Note that the fpu_ctxt block is filled 
  * in by FXSAVE if the CPU has feature FXSR; otherwise FSAVE is used.
  */
-struct vcpu_guest_context {
+struct vcpu_guest_context_x86_32 {
     /* FPU registers come first so they can be aligned for FXSAVE/FXRSTOR. */
     struct { char x[512]; } fpu_ctxt;       /* User-level FPU registers     */
 #define VGCF_I387_VALID                (1<<0)
@@ -182,54 +141,128 @@ struct vcpu_guest_context {
 #define VGCF_in_kernel                 (1<<_VGCF_in_kernel)
 #define _VGCF_failsafe_disables_events 3
 #define VGCF_failsafe_disables_events  (1<<_VGCF_failsafe_disables_events)
-    unsigned long flags;                    /* VGCF_* flags                 */
-    struct cpu_user_regs user_regs;         /* User-level CPU registers     */
-    struct trap_info trap_ctxt[256];        /* Virtual IDT                  */
-    unsigned long ldt_base, ldt_ents;       /* LDT (linear address, # ents) */
-    unsigned long gdt_frames[16], gdt_ents; /* GDT (machine frames, # ents) */
-    unsigned long kernel_ss, kernel_sp;     /* Virtual TSS (only SS1/SP1)   */
-    unsigned long ctrlreg[8];               /* CR0-CR7 (control registers)  */
-    unsigned long debugreg[8];              /* DB0-DB7 (debug registers)    */
-    unsigned long event_callback_cs;        /* CS:EIP of event callback     */
-    unsigned long event_callback_eip;
-    unsigned long failsafe_callback_cs;     /* CS:EIP of failsafe callback  */
-    unsigned long failsafe_callback_eip;
-    unsigned long vm_assist;                /* VMASST_TYPE_* bitmap */
+    uint32_t flags;                         /* VGCF_* flags                 */
+    struct cpu_user_regs_x86_32 user_regs;  /* User-level CPU registers     */
+    struct trap_info_x86_32 trap_ctxt[256]; /* Virtual IDT                  */
+    uint32_t ldt_base, ldt_ents;            /* LDT (linear address, # ents) */
+    uint32_t gdt_frames[16], gdt_ents;      /* GDT (machine frames, # ents) */
+    uint32_t kernel_ss, kernel_sp;          /* Virtual TSS (only SS1/SP1)   */
+    uint32_t ctrlreg[8];                    /* CR0-CR7 (control registers)  */
+    uint32_t debugreg[8];                   /* DB0-DB7 (debug registers)    */
+    uint32_t event_callback_cs;             /* CS:EIP of event callback     */
+    uint32_t event_callback_eip;
+    uint32_t failsafe_callback_cs;          /* CS:EIP of failsafe callback  */
+    uint32_t failsafe_callback_eip;
+    uint32_t vm_assist;                     /* VMASST_TYPE_* bitmap */
 };
-typedef struct vcpu_guest_context vcpu_guest_context_t;
-DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t);
+typedef struct vcpu_guest_context_x86_32 vcpu_guest_context_x86_32_t;
 
 /*
  * Page-directory addresses above 4GB do not fit into architectural %cr3.
  * When accessing %cr3, or equivalent field in vcpu_guest_context, guests
  * must use the following accessor macros to pack/unpack valid MFNs.
  */
-#define xen_pfn_to_cr3(pfn) (((unsigned)(pfn) << 12) | ((unsigned)(pfn) >> 20))
-#define xen_cr3_to_pfn(cr3) (((unsigned)(cr3) >> 12) | ((unsigned)(cr3) << 20))
+#define xen_pfn_to_cr3_x86_32(pfn) (((uint32_t)(pfn) << 12) | ((uint32_t)(pfn) >> 20))
+#define xen_cr3_to_pfn_x86_32(cr3) (((uint32_t)(cr3) >> 12) | ((uint32_t)(cr3) << 20))
 
-struct arch_shared_info {
-    unsigned long max_pfn;                  /* max pfn that appears in table */
+struct arch_shared_info_x86_32 {
+    uint32_t max_pfn;                  /* max pfn that appears in table */
     /* Frame containing list of mfns containing list of mfns containing p2m. */
-    xen_pfn_t     pfn_to_mfn_frame_list_list;
-    unsigned long nmi_reason;
-    uint64_t pad[32];
+    xen_pfn_x86_32_t pfn_to_mfn_frame_list_list;
+    uint32_t nmi_reason;
+    uint32_t pad[64];
 };
-typedef struct arch_shared_info arch_shared_info_t;
+typedef struct arch_shared_info_x86_32 arch_shared_info_x86_32_t;
 
-struct arch_vcpu_info {
-    unsigned long cr2;
-    unsigned long pad[5]; /* sizeof(vcpu_info_t) == 64 */
+struct arch_vcpu_info_x86_32 {
+    uint32_t cr2;
+    uint32_t pad[5]; /* sizeof(vcpu_info_t) == 64 */
 };
-typedef struct arch_vcpu_info arch_vcpu_info_t;
+typedef struct arch_vcpu_info_x86_32 arch_vcpu_info_x86_32_t;
 
-struct xen_callback {
-    unsigned long cs;
-    unsigned long eip;
+struct xen_callback_x86_32 {
+    uint32_t cs;
+    uint32_t eip;
 };
-typedef struct xen_callback xen_callback_t;
+typedef struct xen_callback_x86_32 xen_callback_x86_32_t;
 
 #endif /* !__ASSEMBLY__ */
 
+#ifdef __i386__
+/* make also available without _x86_32 postfix when native */
+
+#ifndef __ASSEMBLY__
+
+__DEFINE_XEN_GUEST_HANDLE(uchar, unsigned char);
+__DEFINE_XEN_GUEST_HANDLE(uint,  unsigned int);
+__DEFINE_XEN_GUEST_HANDLE(ulong, unsigned long);
+DEFINE_XEN_GUEST_HANDLE(char);
+DEFINE_XEN_GUEST_HANDLE(int);
+DEFINE_XEN_GUEST_HANDLE(long);
+DEFINE_XEN_GUEST_HANDLE(void);
+
+__DEFINE_XEN_GUEST_HANDLE(xen_pfn_t, xen_pfn_x86_32_t);
+__DEFINE_XEN_GUEST_HANDLE(trap_info_t, trap_info_x86_32_t);
+__DEFINE_XEN_GUEST_HANDLE(cpu_user_regs_t, cpu_user_regs_x86_32_t);
+__DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t, vcpu_guest_context_x86_32_t);
+
+#define xen_pfn_t             xen_pfn_x86_32_t
+#define PRIpfn                PRIx32
+
+#define xen_ulong_t           xen_ulong_x86_32_t
+#define tsc_timestamp_t       tsc_timestamp_x86_32_t
+#define trap_info             trap_info_x86_32
+#define trap_info_t           trap_info_x86_32_t
+#define cpu_user_regs         cpu_user_regs_x86_32
+#define cpu_user_regs_t       cpu_user_regs_x86_32_t
+#define vcpu_guest_context    vcpu_guest_context_x86_32
+#define vcpu_guest_context_t  vcpu_guest_context_x86_32_t
+#define arch_shared_info      arch_shared_info_x86_32
+#define arch_shared_info_t    arch_shared_info_x86_32_t
+#define arch_vcpu_info        arch_vcpu_info_x86_32
+#define arch_vcpu_info_t      arch_vcpu_info_x86_32_t
+#define xen_callback          xen_callback_x86_32
+#define xen_callback_t        xen_callback_x86_32_t
+
+#define xen_pfn_to_cr3        xen_pfn_to_cr3_x86_32
+#define xen_cr3_to_pfn        xen_cr3_to_pfn_x86_32
+
+#endif /* !__ASSEMBLY__ */
+
+#define FLAT_KERNEL_DS        FLAT_KERNEL_DS_X86_32
+#define FLAT_KERNEL_CS        FLAT_KERNEL_CS_X86_32
+#define FLAT_KERNEL_SS        FLAT_KERNEL_SS_X86_32
+#define FLAT_USER_DS          FLAT_USER_DS_X86_32
+#define FLAT_USER_CS          FLAT_USER_CS_X86_32
+#define FLAT_USER_SS          FLAT_USER_SS_X86_32
+
+/*
+ * Virtual addresses beyond this are not modifiable by guest OSes. The
+ * machine->physical mapping table starts at this address, read-only.
+ */
+#ifdef CONFIG_X86_PAE
+#define __HYPERVISOR_VIRT_START 0xF5800000
+#define __MACH2PHYS_VIRT_START  0xF5800000
+#define __MACH2PHYS_VIRT_END    0xF6800000
+#else
+#define __HYPERVISOR_VIRT_START 0xFC000000
+#define __MACH2PHYS_VIRT_START  0xFC000000
+#define __MACH2PHYS_VIRT_END    0xFC400000
+#endif
+
+#ifndef HYPERVISOR_VIRT_START
+#define HYPERVISOR_VIRT_START mk_unsigned_long(__HYPERVISOR_VIRT_START)
+#endif
+
+#define MACH2PHYS_VIRT_START  mk_unsigned_long(__MACH2PHYS_VIRT_START)
+#define MACH2PHYS_VIRT_END    mk_unsigned_long(__MACH2PHYS_VIRT_END)
+#define MACH2PHYS_NR_ENTRIES  ((MACH2PHYS_VIRT_END-MACH2PHYS_VIRT_START)>>2)
+#ifndef machine_to_phys_mapping
+#define machine_to_phys_mapping ((unsigned long *)MACH2PHYS_VIRT_START)
+#endif
+
+#endif /* i386 native */
+
 /*
  * Prefix forces emulation of some non-trapping instructions.
  * Currently only CPUID.
xen/include/public/arch-x86_64.h +107 -74

261 lines not on this page · raw message

--

#2 lore raw

Re: [patch rfc 1/3] xen arch header rework.

I'm not sure I like the tagging (as it'll likely result in even more overriding in the 32-on-64 patches), but I understand the motivation.

Quoted 1 line

+ uint32_t unused; /* alignment */

Could you use _pad[0-9]* here as is done elsewhere, so that scripts can easily recognize the field as not needing copying (and namely not needing matching source and destination fields) when translating structures between architectures?

Jan

#3 lore raw

Re: [patch rfc 1/3] xen arch header rework.

Quoted 9 lines · Jan Beulich

I'm not sure I like the tagging (as it'll likely result in even more overriding in the 32-on-64 patches), but I understand the motivation.

Quoted 1 line

+ uint32_t unused; /* alignment */

Could you use _pad[0-9]* here as is done elsewhere, so that scripts can easily recognize the field as not needing copying (and namely not needing matching source and destination fields) when translating structures between architectures?

Right now I'm looking at your patches posted yesterday, especially the "compatibility_header_generation" one, and see if that works out better.

The "just fixing up arch-${name}.h" approach has its limits. In the end I'll need a arch-specific xen.h too. Due to longs being in quite some structs, also due to "struct arch_foo" being element of "struct foo", even the structs outside arch-${name}.h end up being quite different on different archs ...

cheers,

Gerd

Signature · 2 lines
Gerd Hoffmann <[email protected]>
http://www.suse.de/~kraxel/julika-dora.jpeg
#4 lore raw

Re: [patch rfc 1/3] xen arch header rework.

In reply to Jan Beulich
Quoted 3 lines · Jan Beulich
Quoted 1 line

+ uint32_t unused; /* alignment */

Could you use _pad[0-9]* here as is done elsewhere, so that scripts

Fixed using "_attribute__((aligned (8)))" instead.

I've switched over to generating headers containing the architecture-specific structures and defines for the tools, current patch is at
http://www.suse.de/~kraxel/patches/kraxel-unstable-master-xen-hg11731-quilt/tools-domain-builder-header-gen.diff

cheers,

Gerd

Signature · 2 lines
Gerd Hoffmann <[email protected]>
http://www.suse.de/~kraxel/julika-dora.jpeg
#5 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 6 lines · Gerd Hoffmann
Quoted 4 lines · Jan Beulich
Quoted 2 lines

+ uint32_t unused; /* alignment
*/

Could you use _pad[0-9]* here as is done elsewhere, so that scripts

Quoted 7 lines

Fixed using "_attribute__((aligned (8)))" instead.

I've switched over to generating headers containing the architecture-specific structures and defines for the tools, current patch is at
http://www.suse.de/~kraxel/patches/kraxel-unstable-master-xen-hg11731-quilt/to
ols-domain-builder-header-gen.diff

I like this approach (although I don't like Perl, but at least it's better than reams of bash, which is definitely the pits ;-). The Solaris guys won't like the gcc attributes, but they can add some flexibility to the script to target the pragmas/extensions of their own compiler.

A generic script that can read in public header files, parse according to the rules of one architectural ABI and spit out header files matching the rules of another ABI is definitely a good thing. If it can also generate the offset/size statistics for any given ABI then that's even better. Basically, it would be good to centralise all this scripting intelligence in one place, rather than scatter hacks all over the place. This script looks like a plausible starting point.

-- Keir

#6 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 4 lines · Keir Fraser

I like this approach (although I don't like Perl, but at least it's better than reams of bash, which is definitely the pits ;-). The Solaris guys won't like the gcc attributes, but they can add some flexibility to the script to target the pragmas/extensions of their own compiler.

Xen seems committed to using Python already (not that I am too thrilled about that, but at least Python code is readable to the occasional hacker), could we try to limit the amount of arcane scripting languages that have to be supported when building Xen, i.e. not add Perl as a build-time dependency?

Thanks,
Jacob

#7 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 10 lines · Jacob Gorm Hansen
Quoted 4 lines

I like this approach (although I don't like Perl, but at least it's better than reams of bash, which is definitely the pits ;-). The Solaris guys won't like the gcc attributes, but they can add some flexibility to the script to target the pragmas/extensions of their own compiler.

Xen seems committed to using Python already (not that I am too thrilled about that, but at least Python code is readable to the occasional hacker), could we try to limit the amount of arcane scripting languages that have to be supported when building Xen, i.e. not add Perl as a build-time dependency?

I'd certainly rather see Python used, unless there's a really good reason not to. I'm not sure if building Linux already requires Perl? It looks to me like it doesn't, unless you build the raid6 drivers.

-- Keir

#8 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 9 lines · Keir Fraser
Quoted 5 lines

Xen seems committed to using Python already (not that I am too thrilled about that, but at least Python code is readable to the occasional hacker), could we try to limit the amount of arcane scripting languages that have to be supported when building Xen, i.e. not add Perl as a build-time dependency?

I'd certainly rather see Python used, unless there's a really good reason not to. I'm not sure if building Linux already requires Perl? It looks to me like it doesn't, unless you build the raid6 drivers.

I'm using perl just because perl is _the_ classic language for text processing jobs like that one and I've already hacked up perl scripts before python started to exist ;)

These days every programming language on earth has bindings for the PCRE library, so switching to something else should be possible without too much trouble. It wouldn't be /me writing that script though.

I don't think adding perl as build dependency is a big problem, almost everyone has it on the machine anyway. Try "rpm -e perl" on any linux distro and watch the error message with the long list of stuff which depends on perl.

Wrt. readability of the scripts: That is IMO more a matter of the programming style than of the programming language. Sure you can easily write unreadable perl code, but you don't have to. And you better shouldn't, just in case you have to touch the scripts again one year later.

cheers,

Gerd

Signature · 2 lines
Gerd Hoffmann <[email protected]>
http://www.suse.de/~kraxel/julika-dora.jpeg
#9 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 3 lines · Gerd Hoffmann

I'm using perl just because perl is _the_ classic language for text processing jobs like that one and I've already hacked up perl scripts before python started to exist ;)

We all have our reasons. I gave up on BASIC a while ago though, and my 68K skill have not been very much in demand either lately.

Quoted 8 lines

These days every programming language on earth has bindings for the PCRE library, so switching to something else should be possible without too much trouble. It wouldn't be /me writing that script though.

I don't think adding perl as build dependency is a big problem, almost everyone has it on the machine anyway. Try "rpm -e perl" on any linux distro and watch the error message with the long list of stuff which depends on perl.

Yes it's a terrible world already, lets make things worse. I think for a project like Xen to be successful in the long term, it is necessary to abide to a set of rules or a 'taste' of what should and should not be allowed. For a while there was a tendency in Xen to fault in any odd piece of software that some developer had fallen in love with (like Twisted). Most of that stuff is thankfully gone now, because someone (other than the people pushing for the inclusion originally I am sure) went through the hard work of rewriting things. The problem with Perl is that quite often people tend to depend on various modules, and the poor (Perl-hating) user ends up spending hours fetching stuff from CPAN just to make the life of the (Perl-loving and lazy) developer a little bit easier. If you don't want to spend the time necessary to port your scripts to Xen's scripting language of choice, and noone else can be convinced to do it for you, I think your patch should simply be rejected.

Jacob

#10 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 8 lines · Jacob Gorm Hansen
Quoted 4 lines · Gerd Hoffmann

I don't think adding perl as build dependency is a big problem, almost everyone has it on the machine anyway. Try "rpm -e perl" on any linux distro and watch the error message with the long list of stuff which depends on perl.

Yes it's a terrible world already, lets make things worse.

Even you hate perl, it is still on almost every unix machine in the world. Which is the point of the paragraph above. perl is so established in the unix world that I wouldn't count that as extra dependency.

Quoted 5 lines

The problem with Perl is
that quite often people tend to depend on various modules, and the poor (Perl-hating) user ends up spending hours fetching stuff from CPAN just to make the life of the (Perl-loving and lazy) developer a little bit easier.

That again is a matter of programming style, not the language. It's not my fault if other people are doing that. The script in question doesn't.

We had such issues with python in the xen tree too btw. One offender was pygrub, which used to depend on a redhat/fedora-specific module. Which was even more annoying than having to fetch stuff from CPAN. CPAN provides at least a simple way to find and fetch the stuff by module name, which is much easier that guessing which of the redhat source rpms you need ...

cheers,

Gerd

Signature · 2 lines
Gerd Hoffmann <[email protected]>
http://www.suse.de/~kraxel/julika-dora.jpeg
#11 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

In reply to Gerd Hoffmann
Quoted 4 lines · Gerd Hoffmann

I don't think adding perl as build dependency is a big problem, almost everyone has it on the machine anyway. Try "rpm -e perl" on any linux distro and watch the error message with the long list of stuff which depends on perl.

While I realise this is drifting from the topic somewhat, if I could just say:

jade:~# rpm -e perl
rpm: To install rpm packages on Debian systems, use alien. See README.Debian.
error: cannot open Packages index using db3 - No such file or directory (2)
error: cannot open /var/lib/rpm/packages.rpm

Perl is most certainly on that system, but 'perl is everywhere' is a little like 'rpm is everywhere'.

Quoted 4 lines

Wrt. readability of the scripts: That is IMO more a matter of the programming style than of the programming language. Sure you can easily write unreadable perl code, but you don't have to. And you better shouldn't, just in case you have to touch the scripts again one year later.

I spend far more time with python than I do perl, I confess, however one of my colleagues spends no time with either and has far less trouble understanding python than perl. In terms of novice-readability perl has a whole bunch of (really useful) line noise constructs. =~, say.

In many cases having a tool in an 'odd' language is better than no tool at all, but in general the fewer dependencies the better.

Signature · 8 lines
Julian Davison
Note: 1) This may have come from an address @cbhs.school.nz
         but isn't necessarily the (or even an) official view
         of Christchurch Boys' High School
      2) While replying to this address may get into my mailbox
         it will almost certainly be filtered into a mailing list
         folder. To actually reach actual me, strip off the bit
         after the '-' in the name.
#12 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 8 lines · Julian Davison
Quoted 4 lines · Gerd Hoffmann

I don't think adding perl as build dependency is a big problem, almost everyone has it on the machine anyway. Try "rpm -e perl" on any linux distro and watch the error message with the long list of stuff which depends on perl.

While I realise this is drifting from the topic somewhat, if I could just say:

Quoted 7 lines
jade:~# rpm -e perl
rpm: To install rpm packages on Debian systems, use alien. See README.Debian.
error: cannot open Packages index using db3 - No such file or directory (2)
error: cannot open /var/lib/rpm/packages.rpm

Perl is most certainly on that system, but 'perl is everywhere' is a little like 'rpm is everywhere'.

OK, then try this:

# apt-get remove perl
Reading package lists... Done
Building dependency tree... Done
The following extra packages will be installed:
  gaim-data gimp-data wesnoth-data
Recommended packages:
  wesnoth-utbs wesnoth-ttb wesnoth-tsg
The following packages will be REMOVED
  abiword-common abiword-gnome acroread acroread-plugins akregator 
alien amor
  analog anjuta apache apache-common ark arts artsbuilder atlantik
  atlantikdesigner auctex autoconf autodia automake1.4 beagle 
binfmt-support

[ snip lots of packages ]

gnome-system-tools gnome-terminal gnome-themes gnome-themes-extras gnome-utils gnome2-user-guide gnomemeeting gnumeric gpdf grepmail gs

[ snip some more ]
kcharselect kcoloredit kcontrol kcron kdat kde-core
kdeaddons-kfile-plugins
kdeadmin kdeadmin-kfile-plugins kdeartwork kdeartwork-style kdeartwork-theme-window kdebase kdebase-bin kdebase-kio-plugins kdegames kdegraphics kdegraphics-kfile-plugins kdelibs kdelibs4c2a kdelirc kdemultimedia-kappfinder-data kdemultimedia-kfile-plugins

[ snip lots and lots more ]

websvn wesnoth wesnoth-ei wesnoth-httt wesnoth-sotbe wesnoth-trow x-ttcidfont-conf xawtv xbase-clients xchm xemacs21 xemacs21-basesupport xemacs21-bin xemacs21-mule xemacs21-support xml-core xpdf xpdf-reader xprint xprint-common xprt-xprintorg xqf xsane xscreensaver
xscreensaver-gl
xserver-xfree86 xserver-xorg xterm xxdiff yaird yelp zenity
The following packages will be upgraded:
gaim-data gimp-data wesnoth-data
3 upgraded, 0 newly installed, 702 to remove and 129 not upgraded. Need to get 34.2MB of archives.
After unpacking 1837MB disk space will be freed.
Do you want to continue [Y/n]? n
Abort.

So, Gerd's example of using rpm was bad, and I wouldn't compare having perl to having rpm.

Quoted 11 lines
Quoted 5 lines

Wrt. readability of the scripts: That is IMO more a matter of the programming style than of the programming language. Sure you can easily write unreadable perl code, but you don't have to. And you better shouldn't, just in case you have to touch the scripts again one year later.

I spend far more time with python than I do perl, I confess, however one of my colleagues spends no time with either and has far less trouble understanding python than perl. In terms of novice-readability perl has a whole bunch of (really useful) line noise constructs. =~, say.

But that is an extremely more powerful tool than anything in python.

Quoted 5 lines

In many cases having a tool in an 'odd' language is better than no tool at all, but in general the fewer dependencies the better.

I find python annoying to work with. Is it spaces or tabs?

if a:
<do something>
if b:
<do this>
if c:
<do that>
if d:
<do another>
if e:
<do whatever>
<and more>
<and that>
<and here>
<and there>
if f:
<do this too>
if g:
<do that as well>
<Now what if am I on???>

Using columns instead of blocks has brought me back to the old days on the old IBM Mainframes. It sometimes gets confusing to know where you're at. And I haven't found emacs or vi helpful in telling you which "if" you are working under. (both easily show which { matches which } ).

But, OK, I like C better than most languages. So if I need something done I usually write it in C. But if it has to do with string manipulation and file parsing, I'll write it in perl.

Yes perl can be obfuscated, and hard to read if someone writes it that way. But I have written lots of perl scripts where I need to work with them after a year or two, and it's never been a problem with me to understand what I did. I just write my perl scripts like I write C and it never has been a problem.

I picked up perl a lot faster than I picked up python, and I still don't have a strong urge to go the python route.

Anyway, perl vs python is just like emacs vs vi, or gnome vs kde.

But just for comparison's sake:

# apt-get remove python
Reading package lists... Done
Building dependency tree... Done
The following packages will be REMOVED
  abiword-gnome alsa-utils anjuta beagle bluefish bug-buddy bum 
capplets-data
  devhelp dia dia-common dia-gnome eog epiphany-browser evolution
  evolution-data-server file-roller galeon galeon-common gcalctool
  gconf-editor gconf2 gdm gdm-themes gedit gksu gnome-about gnome-alsamixer
  gnome-applets gnome-applets-data gnome-control-center gnome-core
  gnome-cups-manager gnome-doc-utils gnome-games gnome-games-data gnome-gv
  gnome-media gnome-media-common gnome-menus gnome-netstatus-applet
  gnome-panel gnome-panel-data gnome-session gnome-spell 
gnome-system-monitor
  gnome-system-tools gnome-terminal gnome-terminal-data gnome-utils
  gnomemeeting gnumeric gpdf gstreamer0.10-gnomevfs
  gstreamer0.10-plugins-good gstreamer0.8-gnomevfs gstreamer0.8-misc
  gstreamer0.8-vorbis gtkhtml3.2 gtkhtml3.6 gtkhtml3.8 gtkorphan 
gtranslator
  gucharmap hplip knewsticker-scripts konq-plugins libapache2-mod-python
  libbonoboui2-0 libcamel1.2-0 libcamel1.2-dev libdevhelp-1-0 libebook1.2-3
  libebook1.2-5 libebook1.2-dev libebook8 libecal1.2-6 libecal1.2-dev
  libecal6 libedata-book1 libedata-book1.2-2 libedata-cal1.2-5 
libedata-cal5
  libedataserver1.2-4 libedataserver1.2-dev libedataserver3
  libedataserverui1.2-6 libedataserverui1.2-dev libeel2-2 
libegroupwise1.2-5
  libegroupwise6 libevolution-cil libexchange-storage1.2-1 libgal2.2-1
  libgal2.2-common libgconf2-dev libgksu2-0 libglade2-dev 
libgnome-desktop-2
  libgnome-media0 libgnome-menu2 libgnome-window-settings1 libgnome2-0
  libgnome2-common libgnome2-dev libgnome2.0-cil libgnomecupsui1.0-1c2a
  libgnomedb2-4 libgnomedb2-common libgnomeui-0 libgnomevfs2-0
  libgnomevfs2-common libgnomevfs2-dev libgnomevfs2-extra libgoffice-1-2
  libgsf-gnome-1 libgsf-gnome-1-114 libgstreamer-gconf0.8-0 
libgtkhtml3.2-11
  libgtkhtml3.6-18 libgtkhtml3.8-15 libgucharmap4 libmetacity0
  libnautilus-extension1 libpanel-applet2-0 libtotem-plparser1 lsb lsb-core
  lsb-cxx lsb-desktop lsb-graphics lsb-qt4 lsb-release mail-notification
  mercurial metacity metacity-common nautilus nautilus-data
  notification-daemon python python-cairo python-central python-dev
  python-foomatic python-glade2 python-gmenu python-gtk2 python-ipy
  python-libxml2 python-newt python-numeric python-qt3 python-subversion
  python-support python-uno rdiff-backup rhythmbox sgt-puzzles slbackup
  subversion-tools svn-buildpackage timer-applet vim-latexsuite vino yelp
0 upgraded, 0 newly installed, 166 to remove and 239 not upgraded.
Need to get 0B of archives.
After unpacking 369MB disk space will be freed.
Do you want to continue [Y/n]? n
Abort.

No snipping needed :-)

-- Steve

#13 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 27 lines · Steven Rostedt
Quoted 8 lines · Julian Davison
Quoted 4 lines · Gerd Hoffmann

I don't think adding perl as build dependency is a big problem, almost everyone has it on the machine anyway. Try "rpm -e perl" on any linux distro and watch the error message with the long list of stuff which depends on perl.

While I realise this is drifting from the topic somewhat, if I could just say:

Quoted 7 lines
jade:~# rpm -e perl
rpm: To install rpm packages on Debian systems, use alien. See README.Debian.
error: cannot open Packages index using db3 - No such file or directory (2)
error: cannot open /var/lib/rpm/packages.rpm

Perl is most certainly on that system, but 'perl is everywhere' is a little like 'rpm is everywhere'.

OK, then try this:

# apt-get remove perl
Reading package lists... Done
Building dependency tree... Done
The following extra packages will be installed:
  gaim-data gimp-data wesnoth-data
Recommended packages:
  wesnoth-utbs wesnoth-ttb wesnoth-tsg
The following packages will be REMOVED

[snip]

Quoted 4 lines

Need to get 34.2MB of archives.
After unpacking 1837MB disk space will be freed.
Do you want to continue [Y/n]? n
Abort.

jade:~# apt-get remove perl
Reading Package Lists... Done
Building Dependency Tree... Done
The following packages will be REMOVED:
  alien build-essential debhelper defoma dpkg-dev ethereal fontconfig
  intltool-debian libcompress-zlib-perl libfontconfig1 libft-perl
  libgtk2.0-0 libgtk2.0-bin libgtk2.0-common libmail-sendmail-perl
  libpango1.0-0 libpango1.0-common libxft2 ntp ntp-server ntp-simple
  perl perl-doc perl-modules po-debconf rpm ttf-bitstream-vera xterm
0 upgraded, 0 newly installed, 28 to remove and 0 not upgraded.
Need to get 0B of archives.
After unpacking 59.8MB disk space will be freed.
Do you want to continue? [Y/n] n
Abort.

Of course the exact list all depends on what the box does :) (Incidentally, those that read carefully may have noticed 'rpm'
now appears to be in the list, how bizarre)

Quoted 1 line

So, Gerd's example of using rpm was bad, and I wouldn't compare having perl to having rpm.

In the grand scheme of things neither would I. (Hence the 'little like' :) I was aiming at highlighting the fact that what was ultimately happening here (IMHO) was broad assumptions/assertions were being made based on what's true in the individuals surrounding environment.

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines

Wrt. readability of the scripts: That is IMO more a matter of the programming style than of the programming language. Sure you can easily write unreadable perl code, but you don't have to. And you better shouldn't, just in case you have to touch the scripts again one year later.

Quoted 7 lines
Quoted 5 lines

I spend far more time with python than I do perl, I confess, however one of my colleagues spends no time with either and has far less trouble understanding python than perl. In terms of novice-readability perl has a whole bunch of (really useful) line noise constructs. =~, say.

But that is an extremely more powerful tool than anything in python.

Undoubtedly.

Quoted 6 lines
Quoted 4 lines

In many cases having a tool in an 'odd' language is better than no tool at all, but in general the fewer dependencies the better.

I find python annoying to work with. Is it spaces or tabs?

Spaces :)

Quoted 22 lines

if a:
<do something>
if b:
<do this>
if c:
<do that>
if d:
<do another>
if e:
<do whatever>
<and more>
<and that>
<and here>
<and there>
if f:
<do this too>
if g:
<do that as well>
<Now what if am I on???>

Quoted 1 line

Using columns instead of blocks has brought me back to the old days on the old IBM Mainframes. It sometimes gets confusing to know where you're at. And I haven't found emacs or vi helpful in telling you which "if" you are working under. (both easily show which { matches which } ).

I've most definitely got a list of peeves about python too, don't get me wrong. When I first heard about python I thought the 'blocks-by-indentation' was completely absurd. But you get used to it, even if you don't ever like it.

I'm hesitant to say it (as I hate people who say this sort of thing) but, use something that isn't emacs or vi :)

Quoted 1 line

But, OK, I like C better than most languages. So if I need something done I usually write it in C. But if it has to do with string manipulation and file parsing, I'll write it in perl.

Sounds familiar :)

Quoted 1 line

Yes perl can be obfuscated, and hard to read if someone writes it that way. But I have written lots of perl scripts where I need to work with them after a year or two, and it's never been a problem with me to understand what I did. I just write my perl scripts like I write C and it never has been a problem.

Though I was looking at *someone else* working with the code. If you can't read your own code then there's a fairly serious problem with the way you write. If someone who has been using the language for several years can't read your code, that's also a style problem.
A python expert may well not be able to comprehend perl, while a perl expert has a far greater chance of comprehending a python script.

Quoted 1 line

I picked up perl a lot faster than I picked up python, and I still don't have a strong urge to go the python route.

While I don't mind python (or perl) the primary reason I've done so much in it is due to the environment here at work. The intranet is running on Plone/Zope/Python, which leads to python being a better (and easier) choice than perl. Which was a point in this discussion that I thought quite important - the more things that are the same the easier it is to get a handle on the whole (once you can grasp a part)

Quoted 1 line

Anyway, perl vs python is just like emacs vs vi, or gnome vs kde.

Nah, there are lots of things perl's far better suited for than python (and vice versa).
emacs/vi and gnome/kde arguments are just a bunch of lunatics that should get on with using the thing instead of evangelising :)

Quoted 5 lines

But just for comparison's sake:

# apt-get remove python
Reading package lists... Done
Building dependency tree... Done

[snip]

Quoted 2 lines

Do you want to continue [Y/n]? n
Abort.

jade:~# apt-get remove python
Reading Package Lists... Done
Building Dependency Tree... Done
Package python is not installed, so not removed
0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
Quoted 1 line

No snipping needed :-)

Indeed :)

Signature · 8 lines
Julian Davison
Note: 1) This may have come from an address @cbhs.school.nz
         but isn't necessarily the (or even an) official view
         of Christchurch Boys' High School
      2) While replying to this address may get into my mailbox
         it will almost certainly be filtered into a mailing list
         folder. To actually reach actual me, strip off the bit
         after the '-' in the name.
#14 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

[this is so off topic, that this is my last reply :-)]

Quoted 33 lines · Julian Davison
Quoted 11 lines

OK, then try this:

# apt-get remove perl
Reading package lists... Done
Building dependency tree... Done
The following extra packages will be installed:
  gaim-data gimp-data wesnoth-data
Recommended packages:
  wesnoth-utbs wesnoth-ttb wesnoth-tsg
The following packages will be REMOVED

[snip]

Quoted 4 lines

Need to get 34.2MB of archives.
After unpacking 1837MB disk space will be freed.
Do you want to continue [Y/n]? n
Abort.

jade:~# apt-get remove perl
Reading Package Lists... Done
Building Dependency Tree... Done
The following packages will be REMOVED:
  alien build-essential debhelper defoma dpkg-dev ethereal fontconfig
  intltool-debian libcompress-zlib-perl libfontconfig1 libft-perl
  libgtk2.0-0 libgtk2.0-bin libgtk2.0-common libmail-sendmail-perl
  libpango1.0-0 libpango1.0-common libxft2 ntp ntp-server ntp-simple
  perl perl-doc perl-modules po-debconf rpm ttf-bitstream-vera xterm
0 upgraded, 0 newly installed, 28 to remove and 0 not upgraded.
Need to get 0B of archives.
After unpacking 59.8MB disk space will be freed.
Do you want to continue? [Y/n] n
Abort.
Quoted 3 lines

Of course the exact list all depends on what the box does :) (Incidentally, those that read carefully may have noticed 'rpm'
now appears to be in the list, how bizarre)

Wow, there must not be a lot done on that box. Well, I did my apt-get on a box I use as an office box (lots of apps).

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines

In many cases having a tool in an 'odd' language is better than no tool at all, but in general the fewer dependencies the better.

I find python annoying to work with. Is it spaces or tabs?

Spaces :)

Well, at least for Xen development :)

Quoted 25 lines
Quoted 23 lines

if a:
<do something>
if b:
<do this>
if c:
<do that>
if d:
<do another>
if e:
<do whatever>
<and more>
<and that>
<and here>
<and there>
if f:
<do this too>
if g:
<do that as well>
<Now what if am I on???>

Quoted 1 line

Using columns instead of blocks has brought me back to the old days on the old IBM Mainframes. It sometimes gets confusing to know where you're at. And I haven't found emacs or vi helpful in telling you which "if" you are working under. (both easily show which { matches which } ).

Quoted 7 lines

I've most definitely got a list of peeves about python too, don't get me wrong. When I first heard about python I thought the 'blocks-by-indentation' was completely absurd. But you get used to it, even if you don't ever like it.

I'm hesitant to say it (as I hate people who say this sort of thing) but, use something that isn't emacs or vi :)

Joe?

Quoted 5 lines
Quoted 2 lines

But, OK, I like C better than most languages. So if I need something done I usually write it in C. But if it has to do with string manipulation and file parsing, I'll write it in perl.

Sounds familiar :)

Quoted 7 lines
Quoted 1 line

Yes perl can be obfuscated, and hard to read if someone writes it that way. But I have written lots of perl scripts where I need to work with them after a year or two, and it's never been a problem with me to understand what I did. I just write my perl scripts like I write C and it never has been a problem.

Though I was looking at *someone else* working with the code. If you can't read your own code then there's a fairly serious problem with the way you write. If someone who has been using the language for several years can't read your code, that's also a style problem.

That's a very good point. Most perl scripts that I download, I have no problem reading. Or even reading the code for the modules. But for the perl obfuscated tests, they are like doing complex Sudoku puzzles!

Quoted 3 lines

A python expert may well not be able to comprehend perl, while a perl expert has a far greater chance of comprehending a python script.

Hmm, never thought of it that way, and it may very well be true.

Quoted 6 lines
Quoted 1 line

Anyway, perl vs python is just like emacs vs vi, or gnome vs kde.

Nah, there are lots of things perl's far better suited for than python (and vice versa).
emacs/vi and gnome/kde arguments are just a bunch of lunatics that should get on with using the thing instead of evangelising :)

You're right. I shouldn't compare it to those other flame fests. Although, reading some mailing lists, it looks the same. But as I mentioned before, I do most my work with C, and when I need to do stuff with string manipulation, I use perl. But there has been a few times that I needed to do number crunching with very large numbers, and python was ideal for that. So where I would use perl, I would not use python, but there does indeed exists tasks that I would use python and not perl for.

I like to push, use the tool that's best for the job at hand, and the only time that python and perl fight is where either can do the job with the only difference being which is more preferred by the developer. For example, I would have written much of Xen in perl, but those that wrote that code preferred python. But so be it.

I don't get into the emacs vs vi fights. I develop in emacs, and edit in vi. That is, if I'm writing a lot of code and compiling a lot, I do that work in emacs. If I need to make edits to a file, or short quick changes to lots of files, I use vi. For those purposes I find that vi and emacs complement each other.

Quoted 7 lines
Quoted 6 lines

But just for comparison's sake:

# apt-get remove python
Reading package lists... Done
Building dependency tree... Done

[snip]

Quoted 8 lines
Quoted 2 lines

Do you want to continue [Y/n]? n
Abort.

jade:~# apt-get remove python
Reading Package Lists... Done
Building Dependency Tree... Done
Package python is not installed, so not removed
0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.

Ah, still less than perl :-)

-- Steve

#15 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 11 lines · Steven Rostedt
Quoted 11 lines

The following packages will be REMOVED:
alien build-essential debhelper defoma dpkg-dev ethereal fontconfig intltool-debian libcompress-zlib-perl libfontconfig1 libft-perl libgtk2.0-0 libgtk2.0-bin libgtk2.0-common libmail-sendmail-perl libpango1.0-0 libpango1.0-common libxft2 ntp ntp-server ntp-simple perl perl-doc perl-modules po-debconf rpm ttf-bitstream-vera xterm
0 upgraded, 0 newly installed, 28 to remove and 0 not upgraded. Need to get 0B of archives.
After unpacking 59.8MB disk space will be freed.
Do you want to continue? [Y/n] n
Abort.

I guess I just cannot fathom the infinite wisdom that leads to a widget toolkit or a truetype font requiring a script language interpreter to run. This example describes perfectly why Linux will never be a serious threat to any of the real desktop platforms out there, most of the application code is so tangled up in external dependencies that packaging up software stand-alone is practially impossible, version conflicts (two applications requiring different version of say, perl) are inevitable, and performance just plain sucks, because developers prefer comfortable (perl, python) to fast and predictable (C, C++). Oh well.

Xen already requires python, so using python instead of perl in the build script does not further bloat the requirements list.

Jacob

#16 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 14 lines · Jacob Gorm Hansen
Quoted 11 lines · Steven Rostedt
Quoted 11 lines

The following packages will be REMOVED:
alien build-essential debhelper defoma dpkg-dev ethereal fontconfig intltool-debian libcompress-zlib-perl libfontconfig1 libft-perl libgtk2.0-0 libgtk2.0-bin libgtk2.0-common libmail-sendmail-perl libpango1.0-0 libpango1.0-common libxft2 ntp ntp-server ntp-simple perl perl-doc perl-modules po-debconf rpm ttf-bitstream-vera xterm
0 upgraded, 0 newly installed, 28 to remove and 0 not upgraded. Need to get 0B of archives.
After unpacking 59.8MB disk space will be freed.
Do you want to continue? [Y/n] n
Abort.

Quoted 10 lines

I guess I just cannot fathom the infinite wisdom that leads to a widget toolkit or a truetype font requiring a script language interpreter to run. This example describes perfectly why Linux will never be a serious threat to any of the real desktop platforms out there, most of the application code is so tangled up in external dependencies that packaging up software stand-alone is practially impossible, version conflicts (two applications requiring different version of say, perl) are inevitable, and performance just plain sucks, because developers prefer comfortable (perl, python) to fast and predictable (C, C++). Oh well.

I'm with Steven in that we're getting well off topic here, but I'm not in any way convinced this is unique to Linux. Nor am I certain that's an accurate evaluation of those languages :)
But this isn't the place.

Quoted 2 lines

Xen already requires python, so using python instead of perl in the build script does not further bloat the requirements list.

I think ultimately this is what it comes down to. The more consistent the code/utility base the better it is for maintainers and builders.

Signature · 8 lines
Julian Davison
Note: 1) This may have come from an address @cbhs.school.nz
         but isn't necessarily the (or even an) official view
         of Christchurch Boys' High School
      2) While replying to this address may get into my mailbox
         it will almost certainly be filtered into a mailing list
         folder. To actually reach actual me, strip off the bit
         after the '-' in the name.
#17 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

In reply to Keir Fraser

Hi,

Quoted 7 lines

A generic script that can read in public header files, parse according to the rules of one architectural ABI and spit out header files matching the rules of another ABI is definitely a good thing. If it can also generate the offset/size statistics for any given ABI then that's even better. Basically, it would be good to centralise all this scripting intelligence in one place, rather than scatter hacks all over the place. This script looks like a plausible starting point.

It's intentionally not generic because I'll intend to translate only stuff which needs translation and not pipe everything through the script. One can try to move these bits (i.e. the @structs array) to some configuration file, I'm not sure that is worth the trouble though.

I can fit it into the xen/include/public subtree so the generated header files are available to everyone and not just private to libxc if that is the way we'll plan to go.

cheers,

Gerd

Signature · 2 lines
Gerd Hoffmann <[email protected]>
http://www.suse.de/~kraxel/julika-dora.jpeg
#18 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

Quoted 4 lines · Gerd Hoffmann

It's intentionally not generic because I'll intend to translate only stuff which needs translation and not pipe everything through the script. One can try to move these bits (i.e. the @structs array) to some configuration file, I'm not sure that is worth the trouble though.

Well, generic-ish. :-) It ought to be generic enough we don't end doing this kind of stuff in multiple places. It doesn't need to work on arbitrary C header files.

Quoted 3 lines

I can fit it into the xen/include/public subtree so the generated header files are available to everyone and not just private to libxc if that is the way we'll plan to go.

That would make sense. Xen itself will probably need these headers, right? At least for the hypercalls where we reimplement/recompile rather than hide the conversion in a copying shim?

And for implementation language: are you really stuck on Perl?

-- Keir

#19 lore raw

Re: SPAM: Re: Re: [patch rfc 1/3] xen arch header rework.

Hi,

Quoted 7 lines
Quoted 3 lines

I can fit it into the xen/include/public subtree so the generated header files are available to everyone and not just private to libxc if that is the way we'll plan to go.

That would make sense. Xen itself will probably need these headers, right? At least for the hypercalls where we reimplement/recompile rather than hide the conversion in a copying shim?

Right now Jan automatically translates the complete header set, a copy of the patch is here:

http://www.suse.de/~kraxel/patches/kraxel-unstable-master-xen-hg11731-quilt/32-on-64-compatibility-header-generation.diff

That is the only one from Jan's 32-on-64 patches I've inspected in detail. I've played around with that one too. One of the problems I've ran into quickly is that including headers for multiple architectures into one source file doesn't work. Thats why I've decided to cherry-pick the stuff which needs translation. Not sure whenever that can work for Jan's xen kernel bits too.

cheers,

Gerd

Signature · 2 lines
Gerd Hoffmann <[email protected]>
http://www.suse.de/~kraxel/julika-dora.jpeg
#20 lore raw

Re: Re: [patch rfc 1/3] xen arch header rework.

In reply to Keir Fraser
Quoted 4 lines · Keir Fraser

I like this approach (although I don't like Perl, but at least it's better than reams of bash, which is definitely the pits ;-). The Solaris guys won't like the gcc attributes, but they can add some flexibility to the script to target the pragmas/extensions of their own compiler.

GCC is fine for the tools and Xen itself; we don't anticipate using anything other than GCC for those.

We only need the public header files to be compiler-agnostic (or, at least, support both GCC and Sun cc).

regards
john

#21 lore raw

RFC PATCH 2/3 xen arch header rework, fixups.

In reply to [email protected]

Th theory the arch header rework should have been fully transparent to the users of the file. In practice it isn't, mostly due to "int" not "long" being used for int32_t on x86_32. This patch makes unstable build again.

---
 linux-2.6-xen-sparse/arch/x86_64/kernel/traps-xen.c |   38 ++++++++++----------
 tools/ioemu/vl.h                                    |    4 +-
 tools/libxc/xc_hvm_build.c                          |    4 +-
 tools/libxc/xc_linux_restore.c                      |    2 -
 tools/libxc/xc_linux_save.c                         |    8 ++--
 tools/libxc/xc_ptrace.c                             |    2 -
 xen/arch/x86/domctl.c                               |    2 -
 xen/arch/x86/hvm/svm/svm.c                          |    2 -
 xen/arch/x86/hvm/vmx/vmx.c                          |    2 -
 xen/arch/x86/mm.c                                   |    4 +-
 xen/arch/x86/x86_32/mm.c                            |    2 -
 xen/arch/x86/x86_32/traps.c                         |    4 +-
 xen/include/asm-x86/processor.h                     |    2 -
 13 files changed, 38 insertions(+), 38 deletions(-)
xen/arch/x86/domctl.c +1 -1
Index: build-32-unstable-11625/xen/arch/x86/domctl.c
===================================================================
@@ -214,7 +214,7 @@ long arch_do_domctl(
         int i;
         struct domain *d = find_domain_by_id(domctl->domain);
         unsigned long max_pfns = domctl->u.getmemlist.max_pfns;
-        unsigned long mfn, gmfn;
+        xen_pfn_t mfn, gmfn;
         struct list_head *list_ent;
 
         ret = -EINVAL;
xen/arch/x86/hvm/svm/svm.c +1 -1
Index: build-32-unstable-11625/xen/arch/x86/hvm/svm/svm.c
===================================================================
@@ -223,7 +223,7 @@ static void stop_svm(void)
 
 
 static void svm_store_cpu_guest_regs(
-    struct vcpu *v, struct cpu_user_regs *regs, unsigned long *crs)
+    struct vcpu *v, struct cpu_user_regs *regs, xen_ulong_t *crs)
 {
     struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
 
xen/arch/x86/hvm/vmx/vmx.c +1 -1
Index: build-32-unstable-11625/xen/arch/x86/hvm/vmx/vmx.c
===================================================================
@@ -501,7 +501,7 @@ void vmx_migrate_timers(struct vcpu *v)
 }
 
 static void vmx_store_cpu_guest_regs(
-    struct vcpu *v, struct cpu_user_regs *regs, unsigned long *crs)
+    struct vcpu *v, struct cpu_user_regs *regs, xen_ulong_t *crs)
 {
     vmx_vmcs_enter(v);
 
xen/arch/x86/mm.c +2 -2
Index: build-32-unstable-11625/xen/arch/x86/mm.c
===================================================================
@@ -2755,7 +2755,7 @@ void destroy_gdt(struct vcpu *v)
 
 
 long set_gdt(struct vcpu *v, 
-             unsigned long *frames,
+             xen_ulong_t *frames,
              unsigned int entries)
 {
     struct domain *d = v->domain;
@@ -2798,7 +2798,7 @@ long set_gdt(struct vcpu *v, 
 long do_set_gdt(XEN_GUEST_HANDLE(ulong) frame_list, unsigned int entries)
 {
     int nr_pages = (entries + 511) / 512;
-    unsigned long frames[16];
+    xen_ulong_t frames[16];
     long ret;
 
     /* Rechecked in set_gdt, but ensures a sane limit for copy_from_user(). */
xen/arch/x86/x86_32/mm.c +1 -1
Index: build-32-unstable-11625/xen/arch/x86/x86_32/mm.c
===================================================================
@@ -188,7 +188,7 @@ void subarch_init_memory(void)
 long subarch_memory_op(int op, XEN_GUEST_HANDLE(void) arg)
 {
     struct xen_machphys_mfn_list xmml;
-    unsigned long mfn;
+    xen_ulong_t mfn;
     unsigned int i, max;
     long rc = 0;
 
xen/arch/x86/x86_32/traps.c +2 -2
Index: build-32-unstable-11625/xen/arch/x86/x86_32/traps.c
===================================================================
@@ -42,7 +42,7 @@ static void print_xen_info(void)
 void show_registers(struct cpu_user_regs *regs)
 {
     struct cpu_user_regs fault_regs = *regs;
-    unsigned long fault_crs[8];
+    xen_ulong_t fault_crs[8];
     const char *context;
 
     if ( hvm_guest(current) && guest_mode(regs) )
@@ -84,7 +84,7 @@ void show_registers(struct cpu_user_regs
            fault_regs.eax, fault_regs.ebx, fault_regs.ecx, fault_regs.edx);
     printk("esi: %08x   edi: %08x   ebp: %08x   esp: %08x\n",
            fault_regs.esi, fault_regs.edi, fault_regs.ebp, fault_regs.esp);
-    printk("cr0: %08lx   cr4: %08lx   cr3: %08lx   cr2: %08lx\n",
+    printk("cr0: %08x   cr4: %08x   cr3: %08x   cr2: %08x\n",
            fault_crs[0], fault_crs[4], fault_crs[3], fault_crs[2]);
     printk("ds: %04x   es: %04x   fs: %04x   gs: %04x   "
            "ss: %04x   cs: %04x\n",
xen/include/asm-x86/processor.h +1 -1
Index: build-32-unstable-11625/xen/include/asm-x86/processor.h
===================================================================
@@ -475,7 +475,7 @@ extern void write_ptbase(struct vcpu *v)
 
 void destroy_gdt(struct vcpu *d);
 long set_gdt(struct vcpu *d, 
-             unsigned long *frames, 
+             xen_ulong_t *frames,
              unsigned int entries);
 
 long set_debugreg(struct vcpu *p, int reg, unsigned long value);
tools/libxc/xc_linux_restore.c +1 -1
Index: build-32-unstable-11625/tools/libxc/xc_linux_restore.c
===================================================================
@@ -634,7 +634,7 @@ int xc_linux_restore(int xc_handle, int 
     /* Get the list of PFNs that are not in the psuedo-phys map */
     {
         unsigned int count;
-        unsigned long *pfntab;
+        xen_ulong_t *pfntab;
         int rc;
 
         if (!read_exact(io_fd, &count, sizeof(count))) {
tools/libxc/xc_linux_save.c +4 -4
Index: build-32-unstable-11625/tools/libxc/xc_linux_save.c
===================================================================
@@ -805,7 +805,7 @@ int xc_linux_save(int xc_handle, int io_
 
             mfn = live_p2m[i];
             if((mfn != INVALID_P2M_ENTRY) && (mfn_to_pfn(mfn) != i)) {
-                DPRINTF("i=0x%x mfn=%lx live_m2p=%lx\n", i,
+                DPRINTF("i=0x%x mfn=%lx live_m2p=%" PRIpfn "\n", i,
                         mfn, mfn_to_pfn(mfn));
                 err++;
             }
@@ -889,7 +889,7 @@ int xc_linux_save(int xc_handle, int io_
                 int n = permute(N, max_pfn, order_nr);
 
                 if (debug) {
-                    DPRINTF("%d pfn= %08lx mfn= %08lx %d  [mfn]= %08lx\n",
+                    DPRINTF("%d pfn= %08lx mfn= %08" PRIpfn " %d  [mfn]= %08" PRIpfn "\n",
                             iter, (unsigned long)n, live_p2m[n],
                             test_bit(n, to_send),
                             mfn_to_pfn(live_p2m[n]&0xFFFFF));
@@ -941,7 +941,7 @@ int xc_linux_save(int xc_handle, int io_
                 goto skip; /* vanishingly unlikely... */
 
             if ((region_base = xc_map_foreign_batch(
-                     xc_handle, dom, PROT_READ, pfn_type, batch)) == 0) {
+                     xc_handle, dom, PROT_READ, /* FIXME */ (xen_pfn_t*)pfn_type, batch)) == 0) {
                 ERR("map batch failed");
                 goto out;
             }
@@ -962,7 +962,7 @@ int xc_linux_save(int xc_handle, int io_
                 }
 
                 if (debug)
-                    DPRINTF("%d pfn= %08lx mfn= %08lx [mfn]= %08lx"
+                    DPRINTF("%d pfn= %08lx mfn= %08lx [mfn]= %08" PRIpfn
                             " sum= %08lx\n",
                             iter,
                             (pfn_type[j] & XEN_DOMCTL_PFINFO_LTAB_MASK) |
tools/libxc/xc_ptrace.c +1 -1
Index: build-32-unstable-11625/tools/libxc/xc_ptrace.c
===================================================================
@@ -156,7 +156,7 @@ online_vcpus_changed(uint64_t cpumap)
 /* --------------------- */
 /* XXX application state */
 static long                     nr_pages = 0;
-static unsigned long           *page_array = NULL;
+static xen_ulong_t              *page_array = NULL;
 
 
 /*
linux-2.6-xen-sparse/arch/x86_64/kernel/traps-xen.c +19 -19
Index: build-32-unstable-11625/linux-2.6-xen-sparse/arch/x86_64/kernel/traps-xen.c
===================================================================
@@ -957,28 +957,28 @@ asmlinkage void math_state_restore(void)
  * specify <dpl>|4 in the second field.
  */
 static trap_info_t trap_table[] = {
-        {  0, 0|4, __KERNEL_CS, (unsigned long)divide_error               },
-        {  1, 0|4, __KERNEL_CS, (unsigned long)debug                      },
-        {  3, 3|4, __KERNEL_CS, (unsigned long)int3                       },
-        {  4, 3|4, __KERNEL_CS, (unsigned long)overflow                   },
-        {  5, 0|4, __KERNEL_CS, (unsigned long)bounds                     },
-        {  6, 0|4, __KERNEL_CS, (unsigned long)invalid_op                 },
-        {  7, 0|4, __KERNEL_CS, (unsigned long)device_not_available       },
-        {  9, 0|4, __KERNEL_CS, (unsigned long)coprocessor_segment_overrun},
-        { 10, 0|4, __KERNEL_CS, (unsigned long)invalid_TSS                },
-        { 11, 0|4, __KERNEL_CS, (unsigned long)segment_not_present        },
-        { 12, 0|4, __KERNEL_CS, (unsigned long)stack_segment              },
-        { 13, 0|4, __KERNEL_CS, (unsigned long)general_protection         },
-        { 14, 0|4, __KERNEL_CS, (unsigned long)page_fault                 },
-        { 15, 0|4, __KERNEL_CS, (unsigned long)spurious_interrupt_bug     },
-        { 16, 0|4, __KERNEL_CS, (unsigned long)coprocessor_error          },
-        { 17, 0|4, __KERNEL_CS, (unsigned long)alignment_check            },
+        {  0, 0|4, __KERNEL_CS, .address = (unsigned long)divide_error               },
+        {  1, 0|4, __KERNEL_CS, .address = (unsigned long)debug                      },
+        {  3, 3|4, __KERNEL_CS, .address = (unsigned long)int3                       },
+        {  4, 3|4, __KERNEL_CS, .address = (unsigned long)overflow                   },
+        {  5, 0|4, __KERNEL_CS, .address = (unsigned long)bounds                     },
+        {  6, 0|4, __KERNEL_CS, .address = (unsigned long)invalid_op                 },
+        {  7, 0|4, __KERNEL_CS, .address = (unsigned long)device_not_available       },
+        {  9, 0|4, __KERNEL_CS, .address = (unsigned long)coprocessor_segment_overrun},
+        { 10, 0|4, __KERNEL_CS, .address = (unsigned long)invalid_TSS                },
+        { 11, 0|4, __KERNEL_CS, .address = (unsigned long)segment_not_present        },
+        { 12, 0|4, __KERNEL_CS, .address = (unsigned long)stack_segment              },
+        { 13, 0|4, __KERNEL_CS, .address = (unsigned long)general_protection         },
+        { 14, 0|4, __KERNEL_CS, .address = (unsigned long)page_fault                 },
+        { 15, 0|4, __KERNEL_CS, .address = (unsigned long)spurious_interrupt_bug     },
+        { 16, 0|4, __KERNEL_CS, .address = (unsigned long)coprocessor_error          },
+        { 17, 0|4, __KERNEL_CS, .address = (unsigned long)alignment_check            },
 #ifdef CONFIG_X86_MCE
-        { 18, 0|4, __KERNEL_CS, (unsigned long)machine_check              },
+        { 18, 0|4, __KERNEL_CS, .address = (unsigned long)machine_check              },
 #endif
-        { 19, 0|4, __KERNEL_CS, (unsigned long)simd_coprocessor_error     },
+        { 19, 0|4, __KERNEL_CS, .address = (unsigned long)simd_coprocessor_error     },
 #ifdef CONFIG_IA32_EMULATION
-	{ IA32_SYSCALL_VECTOR, 3|4, __KERNEL_CS, (unsigned long)ia32_syscall},
+	{ IA32_SYSCALL_VECTOR, 3|4, __KERNEL_CS, .address = (unsigned long)ia32_syscall},
 #endif
         {  0, 0,           0, 0                                              }
 };
tools/ioemu/vl.h +2 -2
Index: build-32-unstable-11625/tools/ioemu/vl.h
===================================================================
@@ -146,9 +146,9 @@ extern int reset_requested;
 void main_loop_wait(int timeout);
 
 int unset_mm_mapping(int xc_handle, uint32_t domid, unsigned long nr_pages,
-                     unsigned int address_bits, unsigned long *extent_start);
+                     unsigned int address_bits, xen_pfn_t *extent_start);
 int set_mm_mapping(int xc_handle, uint32_t domid, unsigned long nr_pages,
-                   unsigned int address_bits, unsigned long *extent_start);
+                   unsigned int address_bits, xen_pfn_t *extent_start);
 
 extern void *shared_vram;
 
tools/libxc/xc_hvm_build.c +2 -2
Index: build-32-unstable-11625/tools/libxc/xc_hvm_build.c
===================================================================
@@ -23,7 +23,7 @@ parseelfimage(
     char *elfbase, unsigned long elfsize, struct domain_setup_info *dsi);
 static int
 loadelfimage(
-    char *elfbase, int xch, uint32_t dom, unsigned long *parray,
+    char *elfbase, int xch, uint32_t dom, xen_pfn_t *parray,
     struct domain_setup_info *dsi);
 
 static void xc_set_hvm_param(int handle,
@@ -587,7 +587,7 @@ static int parseelfimage(char *elfbase,
 
 static int
 loadelfimage(
-    char *elfbase, int xch, uint32_t dom, unsigned long *parray,
+    char *elfbase, int xch, uint32_t dom, xen_pfn_t *parray,
     struct domain_setup_info *dsi)
 {
     Elf32_Ehdr *ehdr = (Elf32_Ehdr *)elfbase;

--

#22 lore raw

RFC PATCH 3/3 xen arch header rework, check utility.

In reply to [email protected]

This adds a tiny utility to verify the patch is correct, by printing the sizes of the structures. Compile once on 32bit and once on 64bit, compare the results, it they don't match something is wrong ...

---
 tools/xcutils/Makefile        |    1 +
 tools/xcutils/check-structs.c |   32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
tools/xcutils/Makefile +1 -0
Index: build-64-unstable-11624/tools/xcutils/Makefile
===================================================================
@@ -27,6 +27,7 @@ CFLAGS += -Wp,-MD,.$(@F).d
 PROG_DEP = .*.d
 
 PROGRAMS		= xc_restore xc_save readnotes
+PROGRAMS		+= check-structs
 
 LDLIBS			= -L$(XEN_LIBXC) -lxenguest -lxenctrl
 
tools/xcutils/check-structs.c new file +32 -0
Index: build-64-unstable-11624/tools/xcutils/check-structs.c
===================================================================
@@ -0,0 +1,32 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <xenctrl.h>
+#include <xen/arch-x86_32.h>
+#include <xen/arch-x86_64.h>
+
+int main(int argc, char *argv[])
+{
+	printf("%-20s %-6s %-6s\n", "struct", "x86_32", "x86_64");
+	printf("----------------------------------\n");
+	printf("%-20s %6zd %6zd\n", "trap_info_t",
+	       sizeof(trap_info_x86_32_t),
+	       sizeof(trap_info_x86_64_t));
+	printf("%-20s %6zd %6zd\n", "cpu_user_regs_t",
+	       sizeof(cpu_user_regs_x86_32_t),
+	       sizeof(cpu_user_regs_x86_64_t));
+	printf("%-20s %6zd %6zd\n", "vcpu_guest_context_t",
+	       sizeof(vcpu_guest_context_x86_32_t),
+	       sizeof(vcpu_guest_context_x86_64_t));
+	printf("%-20s %6zd %6zd\n", "arch_shared_info_t",
+	       sizeof(arch_shared_info_x86_32_t),
+	       sizeof(arch_shared_info_x86_64_t));
+	printf("%-20s %6zd %6zd\n", "arch_vcpu_info_t",
+	       sizeof(arch_vcpu_info_x86_32_t),
+	       sizeof(arch_vcpu_info_x86_64_t));
+	printf("%-20s %6zd %6zd\n", "xen_callback_t",
+	       sizeof(xen_callback_x86_32_t),
+	       sizeof(xen_callback_x86_64_t));
+	return 0;
+}

--