this_cpu_cmpxchg_double support
Subsystems (4)
v2 cover this_cpu_cmpxchg_double support
This patch series introduces this_cpu_cmpxchg_double().
x86 cpus support cmpxchg16b and cmpxchg8b instuction which are capable of switching two words instead of one during a cmpxchg.
Two words allow to swap more state in an atomic instruction.
this_cpu_cmpxchg_double() is used in the slub allocator to avoid interrupt disable/enable in both alloc and free fastpath. this_cpu_cmpxchg_double() works nicely with the per cpu data of the allocator. Using the new operation significantly speeds up the fastpaths.
V1->V2
- Change parameter convention for this_cpu_cmpxchg_double. Specify both percpu variables in same way as the two old and new values.
- Do not require a per cpu pointer but a variable to conform to the convention used in other this_cpu_ops.
PATCH v2 1/4 Generic support for this_cpu_cmpxchg_double
Introduce this_cpu_cmpxchg_double. this_cpu_cmpxchg_double() allows the comparision between two consecutive words and replaces them if there is a match.
bool this_cpu_cmpxchg_double(pcp1, pcp2, old_word1, old_word2, new_word1, new_word2)
this_cpu_cmpxchg_double does not return the old value (difficult since there are two words) but a boolean indicating if the operation was successful.
The first percpu variable must be double word aligned!
--- include/linux/percpu.h | 130 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+)
include/linux/percpu.h +130 -0
@@ -259,6 +259,29 @@ extern void __bad_size_call_parameter(vo pscr2_ret__; \ }) +/* +* Special handling for cmpxchg_double. cmpxchg_double is passed two +* percpu variables. The first has to be aligned to a double word +* boundary and the second has to follow directly thereafter. +*/ +#define __pcpu_double_call_return_int (stem, pcp1, pcp2, ...) \ +({ \ + int ret__; \ + __verify_pcpu_ptr(&pcp1); \ + VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \ + VM_BUG_ON((unsigned long)(&pcp2) != (unsigned long)(&pcp1) + sizeof(pcp1));\ + VM_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ + switch(sizeof(pcp1)) { \ + case 1: ret__ = stem##1(pcp1, pcp2, __VA_ARGS__);break; \ + case 2: ret__ = stem##2(pcp1, pcp2, __VA_ARGS__);break; \ + case 4: ret__ = stem##4(pcp1, pcp2, __VA_ARGS__);break; \ + case 8: ret__ = stem##8(pcp1, pcp2, __VA_ARGS__);break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ + ret__; \ +}) +#define __pcpu_size_call (stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \
@@ -422,6 +445,80 @@ do { \ __this_cpu_cmpxchg_, pcp, oval, nval)#endif +/* +* cmpxchg_double replaces two adjacent scalars at once. The first two +* parameters are per cpu variables which have to be of the same size. +* A truth value is returned to indicate success or +* failure (since a double register result is difficult to handle). +* There is very limited hardware support for these operations. So only certain +* sizes may work. +*/ +#define __this_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ +({ \ + int __ret = 0; \ + if (__this_cpu_read(pcp1) == (oval1) && \ + __this_cpu_read(pcp2) == (oval2)) { \ + __this_cpu_write(pcp1, (nval1)); \ + __this_cpu_write(pcp2, (nval2)); \ + __ret = 1; \ + } \ + (__ret); \ +}) + +#ifndef __this_cpu_cmpxchg_double +# ifndef __this_cpu_cmpxchg_double_1 +# define __this_cpu_cmpxchg_double_1 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef __this_cpu_cmpxchg_double_2 +# define __this_cpu_cmpxchg_double_2 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef __this_cpu_cmpxchg_double_4 +# define __this_cpu_cmpxchg_double_4 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef __this_cpu_cmpxchg_double_8 +# define __this_cpu_cmpxchg_double_8 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# define __this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __pcpu_double_call_return_int(__this_cpu_cmpxchg_double_, (pcp1), (pcp2) \ + oval1, oval2, nval1, nval2) +#endif + +#define _this_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ +({ \ + int ret__; \ + preempt_disable(); \ + ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \ + oval1, oval2, nval1, nval2); \ + preempt_enable(); \ + ret__; \ +}) + +#ifndef this_cpu_cmpxchg_double +# ifndef this_cpu_cmpxchg_double_1 +# define this_cpu_cmpxchg_double_1 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef this_cpu_cmpxchg_double_2 +# define this_cpu_cmpxchg_double_2 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef this_cpu_cmpxchg_double_4 +# define this_cpu_cmpxchg_double_4 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef this_cpu_cmpxchg_double_8 +# define this_cpu_cmpxchg_double_8 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# define this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __pcpu_double_call_return_int(this_cpu_cmpxchg_double_, (pcp1), (pcp2), \ + oval1, oval2, nval1, nval2) +#endif +#define _this_cpu_generic_to_op (pcp, val, op) \ do { \ preempt_disable(); \
@@ -825,4 +922,37 @@ do { \# define irqsafe_cpu_cmpxchg (pcp, oval, nval) __pcpu_size_call_return2(irqsafe_cpu_cmpxchg_, (pcp), oval, nval)#endif +#define irqsafe_generic_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ +({ \ + int ret__; \ + unsigned long flags; \ + local_irq_save(flags); \ + ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \ + oval1, oval2, nval1, nval2); \ + local_irq_restore(flags); \ + ret__; \ +}) + +#ifndef irqsafe_cpu_cmpxchg_double +# ifndef irqsafe_cpu_cmpxchg_double_1 +# define irqsafe_cpu_cmpxchg_double_1 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef irqsafe_cpu_cmpxchg_double_2 +# define irqsafe_cpu_cmpxchg_double_2 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef irqsafe_cpu_cmpxchg_double_4 +# define irqsafe_cpu_cmpxchg_double_4 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef irqsafe_cpu_cmpxchg_double_8 +# define irqsafe_cpu_cmpxchg_double_8 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# define irqsafe_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __pcpu_double_call_return_int(irqsafe_cpu_cmpxchg_double_, (pcp1), (pcp2), \ + oval1, oval2, nval1, nval2) +#endif +#endif /* __LINUX_PERCPU_H */
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
* Christoph Lameter ([email protected]) wrote:
[...]
Quoted 6 lines
+ *
cmpxchg_doublereplaces two adjacent scalars at once. The first two
+ * parameters are per cpu variables which have to be of the same size.
+ * A truth value is returned to indicate success or
+ * failure (since a double register result is difficult to handle).
+ * There is very limited hardware support for these operations. So only certain
+ * sizes may work.
What's the issue with returning the value read by cmpxchg_double in addition to the boolean ? "returning" it per se might be an issue, but you could add 2 more parameters to the macros that take the addresses of these outputs. Returning the values read by cmpxchg instead of just the boolean result helps removing the extra reads from cmpxchg loops, which is why I think it's a shame to just return the boolean result.
Thoughts ?
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 9 lines · Mathieu Desnoyers
* Christoph Lameter ([email protected]) wrote:
[...]Quoted 6 lines
+ *
cmpxchg_doublereplaces two adjacent scalars at once. The first two
+ * parameters are per cpu variables which have to be of the same size.
+ * A truth value is returned to indicate success or
+ * failure (since a double register result is difficult to handle).
+ * There is very limited hardware support for these operations. So only certain
+ * sizes may work.
Quoted 8 lines
What's the issue with returning the value read by
cmpxchg_doublein addition to the boolean ? "returning" it per se might be an issue, but you could add 2 more parameters to the macros that take the addresses of these outputs. Returning the values read by cmpxchg instead of just the boolean result helps removing the extra reads from cmpxchg loops, which is why I think it's a shame to just return the boolean result.Thoughts ?
In the use cases so far the cmpxchg_double is usually successful. If cmpxchg_double succeeds then we know what was in those memory locations before the operation.
Returning the data is only interesting for the case where the cmpxchg_double fails. We'd be optimizing for a very rare case.
cmpxchg_double is here not used for locking (where the mismatching value is of importance in the failure case and where we spin frequently) but for other types of serialization like the transactions id here. Note that this matches *two* types of information. Each mismatch on its own would require specific handling if we want to consider operating on the mismatched values.
this_cpu_cmpxchg also includes the relocation relative to the current per cpu area which is a third data item to keep in mind. A failure may be due to having been rescheduled to another cpu. Sorting all of that out is not worth it IMHO.
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
Introduce
this_cpu_cmpxchg_double.this_cpu_cmpxchg_double()allows the comparision between two consecutive words and replaces them if there is a match.
Quoted 8 lines
bool this_cpu_cmpxchg_double(pcp1, pcp2, old_word1, old_word2, new_word1, new_word2)
this_cpu_cmpxchg_doubledoes not return the old value (difficult since there are two words) but a boolean indicating if the operation was successful.The first percpu variable must be double word aligned!
I really truly hate this interface. The whole notion of passing two pointers where only one is really used, is just painful.
-hpa
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Christoph Lameter
Introduce
this_cpu_cmpxchg_double.this_cpu_cmpxchg_double()allows the comparision between two consecutive words and replaces them if there is a match.bool this_cpu_cmpxchg_double(pcp1, pcp2, old_word1, old_word2, new_word1, new_word2)
Quoted 8 lines
Quoted 5 lines
this_cpu_cmpxchg_doubledoes not return the old value (difficult since there are two words) but a boolean indicating if the operation was successful.The first percpu variable must be double word aligned!
I really truly hate this interface. The whole notion of passing two pointers where only one is really used, is just painful.
Well the second cpu variable is just there for correctness checking. That way we can verify that the types are compatible for the second word and that the second word was properly placed relative to the first one. It also helps the reader in the source because it shows explicitly which two words are modified by the operation.
If you look at the prior patch series at the use case you will see that s->cpu_slab->tid was not referred to in the cmpxchg operation but implicitly modified. That is bad and was the initial motivation to require both to be specified.
During debugging I had a couple of problems with the way the compiler placed those two fields that only became evident after painful debugging sessions. With the checks possible only because both are specifiec there is an explicit failure if either of the two cpu variables is specified in such a way that the cmpxchg32b cannot work.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 30 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Christoph Lameter
Introduce
this_cpu_cmpxchg_double.this_cpu_cmpxchg_double()allows the comparision between two consecutive words and replaces them if there is a match.bool this_cpu_cmpxchg_double(pcp1, pcp2, old_word1, old_word2, new_word1, new_word2)Quoted 8 lines
Quoted 5 lines
this_cpu_cmpxchg_doubledoes not return the old value (difficult since there are two words) but a boolean indicating if the operation was successful.The first percpu variable must be double word aligned!
I really truly hate this interface. The whole notion of passing two pointers where only one is really used, is just painful.
Well the second cpu variable is just there for correctness checking. That way we can verify that the types are compatible for the second word and that the second word was properly placed relative to the first one. It also helps the reader in the source because it shows explicitly which two words are modified by the operation.
If you look at the prior patch series at the use case you will see that
s->cpu_slab->tidwas not referred to in the cmpxchg operation but implicitly modified. That is bad and was the initial motivation to require both to be specified.
Quoted 5 lines
During debugging I had a couple of problems with the way the compiler placed those two fields that only became evident after painful debugging sessions. With the checks possible only because both are specifiec there is an explicit failure if either of the two cpu variables is specified in such a way that the cmpxchg32b cannot work.
I have to admit that I also hate the current interface for two reasons:
a) the way it splits the target address in two.
b) the loss of the value read (the fact that the only current user of this API does not need the value returned seems like a very weak argument to define an API).
I'm probably missing important compiler trickiness here, but why are we not rather relying on forced compiler alignment to declare the target address type ? e.g.:
typedef struct {
unsigned long v[2];
} __attribute__((aligned(2 * sizeof(long)))) cmpxchg_double_t;For the usage you are doing within the allocator code, you can use an union (the bitfield sizes are completely arbitrary).
union alloc_cd {
cmpxchg_double_t cd;
struct {
void *ptr;
unsigned long seq:48;
unsigned long cpu:16;
} v;
};So this_cpu_cmpxchg_double would expect the type "cmpxchg_double_t *" as its first argument.
One way to manage to return the value read would be to pass a pointer to "old_word1-2" as argument rather than the value itself, and return the value read in these locations. Anyway, we won't need the old words for comparison with the return value, because the returned "bool" takes care of telling us if they differ.
Adding the types just for clarity sake (should be handled differently in your macro scheme of course), this_cpu_cmpxchg_double would look like:
typedef struct {
unsigned long v[2];
} __attribute__((aligned(2 * sizeof(long)))) cmpxchg_double_t;bool this_cpu_cmpxchg_double(cmpxchg_double_t *pcp,
unsigned long *old_ret_word1, unsigned long *old_ret_word2, unsigned long new_word1, unsigned long new_word2)
Thoughts ?
Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 3 lines · Mathieu Desnoyers
I have to admit that I also hate the current interface for two reasons:
a) the way it splits the target address in two.
We are handling two different entities that are placed next to each other.
Quoted 3 lines
b) the loss of the value read (the fact that the only current user of this API does not need the value returned seems like a very weak argument to define an API).
The other user of cmpxchg_double that I have in my tree also does not have the need. Repeatability is not as simple to implement as with a single word cmpxchg.
Quoted 20 lines
I'm probably missing important compiler trickiness here, but why are we not rather relying on forced compiler alignment to declare the target address type ? e.g.:
typedef struct { unsigned long v[2]; } __attribute__((aligned(2 * sizeof(long)))) cmpxchg_double_t;For the usage you are doing within the allocator code, you can use an union (the bitfield sizes are completely arbitrary).
union alloc_cd { cmpxchg_double_t cd; struct { void *ptr; unsigned long seq:48; unsigned long cpu:16; } v; };
Quoted 2 lines
So
this_cpu_cmpxchg_doublewould expect the type "cmpxchg_double_t*" as its first argument.
That requires conversions back and forth between words and this type. We only need the bonding between the two per cpu variables for the single cmpxchg_double operation. There is no need to do this for the two old values or the two new values.
Quoted 5 lines
One way to manage to return the value read would be to pass a pointer to "old_word1-2" as argument rather than the value itself, and return the value read in these locations. Anyway, we won't need the old words for comparison with the return value, because the returned "bool" takes care of telling us if they differ.
That would require allocating storage for the two word variable required, passing an address etc. All not necessary right now. Would increase the latency of the inner loop. Saving the returned values also would increase the code uselessly.
Quoted 7 lines
Adding the types just for clarity sake (should be handled differently in your macro scheme of course),
this_cpu_cmpxchg_doublewould look like:typedef struct { unsigned long v[2]; } __attribute__((aligned(2 * sizeof(long)))) cmpxchg_double_t;
Quoted 7 lines
bool this_cpu_cmpxchg_double(cmpxchg_double_t *pcp,
unsigned long*old_ret_word1, unsigned long*old_ret_word2, unsigned longnew_word1, unsigned longnew_word2)Thoughts ?
Looks not symmetric. The proposed approach in the patch has a clear association of the first and second percpu variable with the first and second old and new values.
Also cmpxchg_double may be used with different types and different object sizes. Are we going to define cmpxchg_double_t for all types supported and maybe created by the user? this_cpu_cmpxchg_double also supports two 4 byte fields.
Plus you can have mixed types. In the current use case you have one integer and one pointer. So the cmpxchg_double_t would not fit in cleanly. We would have to define custom types. And with that we will then have difficulties codeing methods to check the compatibility of types.
I tried something like this before I came up with the current solution. It was significantly more code and the code was difficult to understand. Never was as clean and this one.
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Hello,
Quoted 3 lines · Christoph Lameter
Quoted 1 line · Mathieu Desnoyers
I have to admit that I also hate the current interface for two reasons:
Call me weird but I like this one than others. It sure is ugly but the operation itself isn't a particularly pretty so it kinda matches. Also, this one is the least error prone and more consistent with other cpu ops.
Quoted 7 lines
Quoted 3 lines
b) the loss of the value read (the fact that the only current user of this API does not need the value returned seems like a very weak argument to define an API).
The other user of
cmpxchg_doublethat I have in my tree also does not have the need. Repeatability is not as simple to implement as with a single word cmpxchg.
Yeah, even in regular cmpxchg, the read value on failure isn't of very high value. The cpu usually has to go retry anyway && likely to have the cacheline already, so it's not gonna cost much.
So, yeah, of the proposed ones, this is my favorite. Peter and Mathieu don't like it. What do others think? Pekka, Eric, Andrew, what do you guys think?
Thanks.
Signature · 1 line
tejun
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 20 lines · Tejun Heo
Hello,
Quoted 3 lines · Christoph Lameter
Quoted 1 line · Mathieu Desnoyers
I have to admit that I also hate the current interface for two reasons:
Call me weird but I like this one than others. It sure is ugly but the operation itself isn't a particularly pretty so it kinda matches. Also, this one is the least error prone and more consistent with other cpu ops.
Quoted 7 lines
Quoted 3 lines
b) the loss of the value read (the fact that the only current user of this API does not need the value returned seems like a very weak argument to define an API).
The other user of
cmpxchg_doublethat I have in my tree also does not have the need. Repeatability is not as simple to implement as with a single word cmpxchg.
Quoted 7 lines
Yeah, even in regular cmpxchg, the read value on failure isn't of very high value. The cpu usually has to go retry anyway && likely to have the cacheline already, so it's not gonna cost much.
So, yeah, of the proposed ones, this is my favorite. Peter and Mathieu don't like it. What do others think? Pekka, Eric, Andrew, what do you guys think?
I'm not a huge fan of the API either but like you, I like it best from the proposed ones.
Pekka
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Tejun HeoQuoted 11 lines · Tejun Heo
Hello,
Quoted 3 lines · Christoph Lameter
Quoted 1 line · Mathieu Desnoyers
I have to admit that I also hate the current interface for two reasons:
Call me weird but I like this one than others. It sure is ugly but the operation itself isn't a particularly pretty so it kinda matches. Also, this one is the least error prone and more consistent with other cpu ops.
So what are we going to do about this patch? I'd love to merge Christoph's SLUB patches for linux-next now that .38-rc1 is out.
Pekka
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Hello,
Quoted 8 lines · Pekka Enberg
Quoted 4 lines · Tejun Heo
Call me weird but I like this one than others. It sure is ugly but the operation itself isn't a particularly pretty so it kinda matches. Also, this one is the least error prone and more consistent with other cpu ops.
So what are we going to do about this patch? I'd love to merge Christoph's SLUB patches for linux-next now that .38-rc1 is out.
At least you like it, which is good.
I don't think the currently proposed one with two separate parameters is significantly better than other alternatives and vice-versa. They all have slightly different ugliness and error proneness issues.
That said, I still like the double parameter one best, and, unless there are distinctively good reasons to choose another one, I'm gonna commit it to percpu tree in a few days so that merge can proceed. So, if you have something to say, now would be a good time to assert it.
Thank you.
Signature · 1 line
tejun
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 14 lines · Tejun Heo
Hello,
Quoted 8 lines · Pekka Enberg
Quoted 4 lines · Tejun Heo
Call me weird but I like this one than others. It sure is ugly but the operation itself isn't a particularly pretty so it kinda matches. Also, this one is the least error prone and more consistent with other cpu ops.
So what are we going to do about this patch? I'd love to merge Christoph's SLUB patches for linux-next now that .38-rc1 is out.
At least you like it, which is good.
Quoted 9 lines
I don't think the currently proposed one with two separate parameters is significantly better than other alternatives and vice-versa. They all have slightly different ugliness and error proneness issues.
That said, I still like the double parameter one best, and, unless there are distinctively good reasons to choose another one, I'm gonna commit it to percpu tree in a few days so that merge can proceed. So, if you have something to say, now would be a good time to assert it.
I really object to passing two pointers where one of them has to be a fixed offset to the other. That really doesn't make any sense.
-hpa
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Hello, Peter.
Quoted 2 lines · H. Peter Anvin
I really object to passing two pointers where one of them has to be a fixed offset to the other. That really doesn't make any sense.
Yeah, I hear you, but it really comes down to which ugliness disgusts one the most. That, unfortunately, is inherently very subjective when there's no significantly better choice.
For me, the double parameter thing at least seems to have the advantages of being able to verify the two intended memory locations to be used actually are together and looking ugly reflecting its true nature.
The inherent ugliness stems from the fact that we don't have the built-in data type to properly deal with this. Array of length two might be better fit, but I can see as many downsides with that too.
So, if anyone can give something clearly better for technical reasons, I'll be more than happy to take it, but as it currently stands, it seems we'll have to choose one among uglies and not everyone would be happy about the choice. :-(
Thanks.
Signature · 1 line
tejun
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
There is a technical argument: any noninline version will have actual code overhead.
Quoted 28 lines · Tejun Heo
Hello, Peter.
Quoted 2 lines · H. Peter Anvin
I really object to passing two pointers where one of them has to be a fixed offset to the other. That really doesn't make any sense.
Yeah, I hear you, but it really comes down to which ugliness disgusts one the most. That, unfortunately, is inherently very subjective when there's no significantly better choice.
For me, the double parameter thing at least seems to have the advantages of being able to verify the two intended memory locations to be used actually are together and looking ugly reflecting its true nature.
The inherent ugliness stems from the fact that we don't have the built-in data type to properly deal with this. Array of length two might be better fit, but I can see as many downsides with that too.
So, if anyone can give something clearly better for technical reasons, I'll be more than happy to take it, but as it currently stands, it seems we'll have to choose one among uglies and not everyone would be happy about the choice. :-(
Thanks.
Signature · 1 line
tejun
Signature · 1 line
Sent from my mobile phone. Please pardon any lack of formatting.
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 2 lines · H. Peter Anvin
There is a technical argument: any noninline version will have actual code overhead.
But the type checking and all those stuff have to be macros anyway. If out-of-line implementation is necessary, it can take whatever parameter to optimize performance from the frontend macro.
Thanks.
Signature · 1 line
tejun
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Tejun HeoQuoted 15 lines · * Tejun Heo ([email protected]) wrote:
Hello, Peter.
Quoted 2 lines · H. Peter Anvin
I really object to passing two pointers where one of them has to be a fixed offset to the other. That really doesn't make any sense.
Yeah, I hear you, but it really comes down to which ugliness disgusts one the most. That, unfortunately, is inherently very subjective when there's no significantly better choice.
For me, the double parameter thing at least seems to have the advantages of being able to verify the two intended memory locations to be used actually are together and looking ugly reflecting its true nature.
Quoted 8 lines
The inherent ugliness stems from the fact that we don't have the built-in data type to properly deal with this. Array of length two might be better fit, but I can see as many downsides with that too.
So, if anyone can give something clearly better for technical reasons, I'll be more than happy to take it, but as it currently stands, it seems we'll have to choose one among uglies and not everyone would be happy about the choice. :-(
Quoting Christoph, from the previous exchange:
"The single large 128 bit scalar does not work. Having to define an additional structure it also a bit clumsy. I think its best to get another patchset out that also duplicates the first parameter and makes the percpu variable specs conform to the other this_cpu ops."
I'm again probably missing something, but what is "clumsy" about defining a structure like the following to ensure proper alignment of the target pointer (instead of adding a runtime test) ?
struct cmpxchg_double {
#if __BYTE_ORDER == __LITTLE_ENDIAN
unsigned long low, high;
#else
unsigned long high, low;#endif
} __attribute__((packed, aligned(2 * sizeof(unsigned long))));
(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
The prototype could then look like:
bool __this_cpu_generic_cmpxchg_double(pcp, oval_low, oval_high, nval_low, nval_high);
With:
struct cmpxchg_double *pcp
I think Christoph's point is that he wants to alias this with a pointer. Well, this can be done cleanly with:
union {
struct cmpxchg_double casdbl;
struct {
void *ptr;
unsigned long cpuid_tid;
} t;
}So by keeping distinct variables for the oval/nal arguments, we let the compiler use registers (instead of the mandatory stack use that would be required if we pass union or structures as oval/nval arguments), but we ensure proper alignment (and drop the unneeded second pointer, as well as the runtime pointer alignment checks) by passing one single pcp pointer of a fixed type with a known alignment.
Thoughts ?
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 17 lines · Mathieu Desnoyers
I'm again probably missing something, but what is "clumsy" about defining a structure like the following to ensure proper alignment of the target pointer (instead of adding a runtime test) ?
struct
cmpxchg_double{#if __BYTE_ORDER == __LITTLE_ENDIAN unsigned long low, high; #else unsigned long high, low;#endif
}__attribute__((packed, aligned(2 * sizeof(unsigned long))));(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
Quoted 6 lines
The prototype could then look like:
bool
__this_cpu_generic_cmpxchg_double(pcp, oval_low, oval_high, nval_low, nval_high);With:
structcmpxchg_double*pcp
That does not conform to the parameter conventions in other this_cpu_ops. The first parameter is a variable because the notion of a pointer is problematic given that percpu operations use a segment prefix to relocate pointers. You would be implicitly passing a 128 bit argument although the compiler may not need to generate code for that.
Quoted 10 lines
I think Christoph's point is that he wants to alias this with a pointer. Well, this can be done cleanly with:
union { struct cmpxchg_double casdbl; struct { void *ptr; unsigned long cpuid_tid; } t; }
There is no need for aliases with the existing implementation.
How will the macro check the parameters now?
Quoted 1 line
Thoughts ?
Could you actually try to write a patch instead running through points that we have discussed earlier?
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 23 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 21 lines · Mathieu Desnoyers
I'm again probably missing something, but what is "clumsy" about defining a structure like the following to ensure proper alignment of the target pointer (instead of adding a runtime test) ?
struct
cmpxchg_double{#if __BYTE_ORDER == __LITTLE_ENDIAN unsigned long low, high; #else unsigned long high, low;#endif
}__attribute__((packed, aligned(2 * sizeof(unsigned long))));(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
The prototype could then look like:
bool
__this_cpu_generic_cmpxchg_double(pcp, oval_low, oval_high, nval_low, nval_high);
Quoted 7 lines
Quoted 2 lines
With:
structcmpxchg_double*pcpThat does not conform to the parameter conventions in other
this_cpu_ops. The first parameter is a variable because the notion of a pointer is problematic given that percpu operations use a segment prefix to relocate pointers.
So the first argument could be along the lines of:
struct cmpxchg_double pcp
then.
Quoted 2 lines
You would be implicitly passing a 128 bit argument although the compiler may not need to generate code for that.
Sorry, I don't understand this last statement. Does it still apply if we pass pcp as I just proposed ? (without the pointer, with a __builtin_choose_expr check on __alignof__ of the pcp parameter)
Quoted 12 lines
Quoted 11 lines
I think Christoph's point is that he wants to alias this with a pointer. Well, this can be done cleanly with:
union { struct cmpxchg_double casdbl; struct { void *ptr; unsigned long cpuid_tid; } t; }
Quoted 3 lines
There is no need for aliases with the existing implementation.
How will the macro check the parameters now?
Well, my last proposal to check __alignof__ within a __builtin_choose_expr check wouldn't need this union actually, which would be much better I think.
Quoted 5 lines
Quoted 2 lines
Thoughts ?
Could you actually try to write a patch instead running through points that we have discussed earlier?
I'm going back to the points that have been previously dismissed with a rather large degree of handwaving. ;) I'm OK with looking into the API and providing code snippets to show the basic implementation behind the ideas, but I unfortunately don't have the bandwidth to stop everything I'm currently working on and start working on double-cas patches. Sorry, I wish I could do more.
Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 8 lines · Mathieu Desnoyers
Quoted 7 lines
Quoted 2 lines
With:
structcmpxchg_double*pcpThat does not conform to the parameter conventions in other
this_cpu_ops. The first parameter is a variable because the notion of a pointer is problematic given that percpu operations use a segment prefix to relocate pointers.
Quoted 5 lines
So the first argument could be along the lines of:
struct
cmpxchg_doublepcpthen.
Ok then you would pass a struct by value? Or use a non-scalar as a variable passed to a this_cpu_op? So far per cpu scalars have been the only variables allowed to be specified in this_cpu operations.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
struct cmpxchg_double casdbl; struct { void *ptr; unsigned long cpuid_tid; } t; }
Quoted 6 lines
Quoted 3 lines
There is no need for aliases with the existing implementation.
How will the macro check the parameters now?
Well, my last proposal to check
__alignof__within a__builtin_choose_exprcheck wouldn't need this union actually, which would be much better I think.
The existing implementation has a check for alignment. That is not the problem. The typechecking would need to be addressed. I.e. if I pass a pointer for old and an ulong for the new value then I'd like to see the compiler complain. Or if the first parameter is a long but the type of the first word is a pointer etc etc.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 12 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 7 lines
Quoted 2 lines
With:
structcmpxchg_double*pcpThat does not conform to the parameter conventions in other
this_cpu_ops. The first parameter is a variable because the notion of a pointer is problematic given that percpu operations use a segment prefix to relocate pointers.So the first argument could be along the lines of:
Quoted 7 lines
Quoted 3 lines
struct
cmpxchg_doublepcpthen.
Ok then you would pass a struct by value? Or use a non-scalar as a variable passed to a
this_cpu_op? So far per cpu scalars have been the only variables allowed to be specified inthis_cpuoperations.
What I have in mind is that the struct passed would be non-scalar for this specific operation. I'm not sure about the distinction between "pass a struct by value" and "use a non-scalar as a variable passed to a this_cpu_op" -- I feel I'm missing an important detail in what you say, because I see these as being the same thing.
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 7 lines
struct cmpxchg_double casdbl; struct { void *ptr; unsigned long cpuid_tid; } t; }There is no need for aliases with the existing implementation.
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
How will the macro check the parameters now?
Well, my last proposal to check
__alignof__within a__builtin_choose_exprcheck wouldn't need this union actually, which would be much better I think.The existing implementation has a check for alignment. That is not the problem.
It's a dynamic check right ? (based on VM_BUG_ON() if I remember well) It adds code and runtime conditions, which would go away if we let the alignment check be done at compile-time.
Quoted 4 lines
The typechecking would need to be addressed. I.e. if I pass a pointer for old and an ulong for the new value then I'd like to see the compiler complain. Or if the first parameter is a long but the type of the first word is a pointer etc etc.
Hrm. Then the only solution I see would be to require that the structure used as percpu_dd parameter have fixed field names (yeah, that's a bit odd, but could not come up with a more elegant solution at the moment):
struct mycustomdoublestruct {
sometype word1;
someothertype word2;
}So we can access percpu_dd.word1 and percpu_dd.word2 within this_cpu_cmpxchg_double for the type checking.
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Mathieu DesnoyersHello,
Quoted 14 lines · Mathieu Desnoyers
"The single large 128 bit scalar does not work. Having to define an additional structure it also a bit clumsy. I think its best to get another patchset out that also duplicates the first parameter and makes the percpu variable specs conform to the other
this_cpuops."I'm again probably missing something, but what is "clumsy" about defining a structure like the following to ensure proper alignment of the target pointer (instead of adding a runtime test) ?
struct
cmpxchg_double{#if __BYTE_ORDER == __LITTLE_ENDIAN unsigned long low, high; #else unsigned long high, low;
Quoted 8 lines
#endif
}__attribute__((packed, aligned(2 * sizeof(unsigned long))));(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
Yeah, good point. :-)
Quoted 18 lines
The prototype could then look like:
bool
__this_cpu_generic_cmpxchg_double(pcp, oval_low, oval_high, nval_low, nval_high);With:
structcmpxchg_double*pcpI think Christoph's point is that he wants to alias this with a pointer. Well, this can be done cleanly with:
union { struct cmpxchg_double casdbl; struct { void *ptr; unsigned long cpuid_tid; } t; }
Quoted 7 lines
So by keeping distinct variables for the oval/nal arguments, we let the compiler use registers (instead of the mandatory stack use that would be required if we pass union or structures as oval/nval arguments), but we ensure proper alignment (and drop the unneeded second pointer, as well as the runtime pointer alignment checks) by passing one single pcp pointer of a fixed type with a known alignment.
Actually, we might be able to work around the stack usage regardless of the passed type. We should be able to decompose the structure or ulonglong using macro and then pass the decomposed parameters to a function if necessary.
Hmm, if we do ulonglong, we don't even need a separate interface and can simply use cmpxchg(). If someone can come up with something which uses ulonglong that way, it would be great; then, we wouldn't need to worry about alignment either. Am I missing something?
Thanks.
Signature · 1 line
tejun
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with __int128 on 64 bits.
Quoted 70 lines · Tejun Heo
Hello,
Quoted 14 lines · Mathieu Desnoyers
"The single large 128 bit scalar does not work. Having to define an additional structure it also a bit clumsy. I think its best to get another patchset out that also duplicates the first parameter and makes the percpu variable specs conform to the other
this_cpuops."I'm again probably missing something, but what is "clumsy" about defining a structure like the following to ensure proper alignment of the target pointer (instead of adding a runtime test) ?
struct
cmpxchg_double{#if __BYTE_ORDER == __LITTLE_ENDIAN unsigned long low, high; #else unsigned long high, low;Quoted 8 lines
#endif
}__attribute__((packed, aligned(2 * sizeof(unsigned long))));(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
Yeah, good point. :-)
Quoted 3 lines
The prototype could then look like:
bool __this_cpu_generic_cmpxchg_double(pcp,
oval_low,oval_high,
nval_low,nval_high);Quoted 5 lines
With:
structcmpxchg_double*pcpI think Christoph's point is that he wants to alias this with a
pointer. Well,
Quoted 10 lines
this can be done cleanly with:
union { struct cmpxchg_double casdbl; struct { void *ptr; unsigned long cpuid_tid; } t; }Quoted 7 lines
So by keeping distinct variables for the oval/nal arguments, we let the compiler use registers (instead of the mandatory stack use that would be required if we pass union or structures as oval/nval arguments), but we ensure proper alignment (and drop the unneeded second pointer, as well as the runtime pointer alignment checks) by passing one single pcp pointer of a fixed type with a known alignment.
Actually, we might be able to work around the stack usage regardless of the passed type. We should be able to decompose the structure or ulonglong using macro and then pass the decomposed parameters to a function if necessary.
Hmm, if we do ulonglong, we don't even need a separate interface and can simply use
cmpxchg(). If someone can come up with something which uses ulonglong that way, it would be great; then, we wouldn't need to worry about alignment either. Am I missing something?Thanks.
Signature · 1 line
tejun
Signature · 1 line
Sent from my mobile phone. Please pardon any lack of formatting.
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 2 lines · H. Peter Anvin
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with
__int128on 64 bits.
But, IIRC, the problem with int128 was with passing it as parameter and return value. We don't have to do that. We'll be just using it as a data storage / container type. Or even that is broken?
Thanks.
Signature · 1 line
tejun
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 7 lines · Tejun Heo
Quoted 2 lines · H. Peter Anvin
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with
__int128on 64 bits.But, IIRC, the problem with int128 was with passing it as parameter and return value. We don't have to do that. We'll be just using it as a data storage / container type. Or even that is broken?
Well, part of the point was to pass in registers.
No idea on the data storage type.
-hpa
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 2 lines · H. Peter Anvin
Quoted 2 lines
Quoted 2 lines
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with
__int128on 64 bits.
Quoted 3 lines · Tejun Heo
Quoted 3 lines
But, IIRC, the problem with int128 was with passing it as parameter and return value. We don't have to do that. We'll be just using it as a data storage / container type. Or even that is broken?
Quoted 3 lines · H. Peter Anvin
Well, part of the point was to pass in registers.
No idea on the data storage type.
Ping? The current situation is that we're unable to merge a perfectly good SLUB performance optimization because we can't seem to agree on the this_cpu_cmpxchg_double() API.
Pekka
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 9 lines · Pekka Enberg
Quoted 2 lines · H. Peter Anvin
Quoted 2 lines
Quoted 2 lines
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with
__int128on 64 bits.Quoted 3 lines · Tejun Heo
Quoted 3 lines
But, IIRC, the problem with int128 was with passing it as parameter and return value. We don't have to do that. We'll be just using it as a data storage / container type. Or even that is broken?
Quoted 8 lines
Quoted 3 lines · H. Peter Anvin
Well, part of the point was to pass in registers.
No idea on the data storage type.
Ping? The current situation is that we're unable to merge a perfectly good SLUB performance optimization because we can't seem to agree on the
this_cpu_cmpxchg_double()API.
I thought cl was preparing new version of the patchset. Christoph?
Signature · 1 line
tejun
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 5 lines · Tejun Heo
Quoted 3 lines
Ping? The current situation is that we're unable to merge a perfectly good SLUB performance optimization because we can't seem to agree on the
this_cpu_cmpxchg_double()API.I thought cl was preparing new version of the patchset. Christoph?
In order to do that I would have to see a way to improve the patches. None of the suggestions so far seem to be realizable.
Tejun Heo
maintainerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 10 lines · Christoph Lameter
Quoted 5 lines · Tejun Heo
Quoted 3 lines
Ping? The current situation is that we're unable to merge a perfectly good SLUB performance optimization because we can't seem to agree on the
this_cpu_cmpxchg_double()API.I thought cl was preparing new version of the patchset. Christoph?
In order to do that I would have to see a way to improve the patches. None of the suggestions so far seem to be realizable.
I see. For some reason, I thought you and hpa agreed on something and you were writing up the patches. I'll read the thread again. Let's just pick one. It's not like this is an API which will be used througout the tree.
Thanks.
Signature · 1 line
tejun
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 9 lines · Tejun Heo
Quoted 8 lines · Christoph Lameter
Quoted 5 lines · Tejun Heo
Quoted 3 lines
Ping? The current situation is that we're unable to merge a perfectly good SLUB performance optimization because we can't seem to agree on the
this_cpu_cmpxchg_double()API.I thought cl was preparing new version of the patchset. Christoph?
Quoted 7 lines
Quoted 2 lines
In order to do that I would have to see a way to improve the patches. None of the suggestions so far seem to be realizable.
I see. For some reason, I thought you and hpa agreed on something and you were writing up the patches. I'll read the thread again. Let's just pick one. It's not like this is an API which will be used througout the tree.
We were discussing some hypothetical changes but as far as I can see all those scenarios lead to more issues.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Christoph LameterQuoted 12 lines · Christoph Lameter
Quoted 5 lines · Tejun Heo
Quoted 3 lines
Ping? The current situation is that we're unable to merge a perfectly good SLUB performance optimization because we can't seem to agree on the
this_cpu_cmpxchg_double()API.I thought cl was preparing new version of the patchset. Christoph?
In order to do that I would have to see a way to improve the patches. None of the suggestions so far seem to be realizable.
Just respin your patches then.
I suspect we'll have so few users of this API, that we can easily change it later if really someone brings something better.
"Le mieux est l'ennemi du bien." (The perfect is the enemy of the good).
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 1 line · Eric Dumazet
Just respin your patches then.
Ok. Will do that now.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to H. Peter AnvinQuoted 1 line · H. Peter Anvin
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with
__int128on 64 bits.
We also would need to be pass the structure by value (well its really a variable but its like passing by value) in order to be similar to the other this_cpu_ops
You'd want either
DEFINE_PERCPU(struct mycustomdoublestruct, percpu_dd)
this_cpu_cmpxchg_double(percpu_dd, oldword1, oldword2, newword1, newword2)
with the problem of type checking
or
this_cpu_cmpxchg_double(percpu_dd, old_dd, new_dd)
with the problem of 128 bit constants/structs passed by value.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 8 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 1 line · H. Peter Anvin
We could do cmpxchg with a structure... the problem with a lon int type is that Cristoph ran into bugs with
__int128on 64 bits.We also would need to be pass the structure by value (well its really a variable but its like passing by value) in order to be similar to the other
this_cpu_ops
Quoted 7 lines
You'd want either
DEFINE_PERCPU(struct mycustomdoublestruct, percpu_dd)
this_cpu_cmpxchg_double(percpu_dd, oldword1, oldword2, newword1, newword2)with the problem of type checking
What is the problem with type checking here ?
We could use a gcc builtin like the following to check if the alignment of "percpu_dd" meets the double-cas requirements:
#define testmacro(a, b) \
__builtin_choose_expr(__alignof__(a) >= 2 * sizeof(unsigned long), \
((a).low) = (b), \ /* success */
((a).low) = (void) 0) /* compile-error */Quoted 5 lines
or
this_cpu_cmpxchg_double(percpu_dd, old_dd, new_dd)with the problem of 128 bit constants/structs passed by value.
Yeah, I guess trying to deal with 128-bit value might be a bit tricky. But having something sane and with compile-time-checked alignment for the percpu_dd type is not to be looked over.
Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 5 lines · Mathieu Desnoyers
Quoted 3 lines
this_cpu_cmpxchg_double(percpu_dd, oldword1, oldword2, newword1, newword2)with the problem of type checking
What is the problem with type checking here ?
You need to know the fields in the struct to do the type checking with each of the other parameters.
Quoted 10 lines
We could use a gcc builtin like the following to check if the alignment of "
percpu_dd" meets the double-cas requirements:#define testmacro(a, b) \ __builtin_choose_expr(__alignof__(a) >= 2 * sizeof(unsigned long), \ ((a).low) = (b), \ /* success */ ((a).low) = (void) 0) /* compile-error */Quoted 2 lines
or
Quoted 7 lines
Quoted 3 lines
this_cpu_cmpxchg_double(percpu_dd, old_dd, new_dd)with the problem of 128 bit constants/structs passed by value.
Yeah, I guess trying to deal with 128-bit value might be a bit tricky. But having something sane and with compile-time-checked alignment for the
percpu_ddtype is not to be looked over.
The existing implementation could be equipped to do a compile time check for the proper alignment if the pointer passed is constant.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 10 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 5 lines · Mathieu Desnoyers
Quoted 3 lines
this_cpu_cmpxchg_double(percpu_dd, oldword1, oldword2, newword1, newword2)with the problem of type checking
What is the problem with type checking here ?
You need to know the fields in the struct to do the type checking with each of the other parameters.
Isn't that a bit much to try to match the type of each oldword/newword parameter to the structure fields ? Having separated word 1-2 parameter is just an artefact caused by the inability of some gcc to deal with int128; were we to use int128, we would have none of this type-checking whatsoever.
We could simply check that the first parameter alignment is >= 2 * sizeof(long) and that its size == 2 * sizeof(long), so that the layout in memory fits the cmpxchg_double requirements. This should work both for structure and array parameters.
Now if the user needs to map "oldword1, oldword2" to the actual percpu_dd fields, we could ensure that the order of these two parameters actually match the structure field or array index order. This would, of course, be documented above this_cpu_cmpxchg_double().
Quoted 13 lines
Quoted 13 lines
We could use a gcc builtin like the following to check if the alignment of "
percpu_dd" meets the double-cas requirements:#define testmacro(a, b) \ __builtin_choose_expr(__alignof__(a) >= 2 * sizeof(unsigned long), \ ((a).low) = (b), \ /* success */ ((a).low) = (void) 0) /* compile-error */Quoted 4 lines
or
this_cpu_cmpxchg_double(percpu_dd, old_dd, new_dd)
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
with the problem of 128 bit constants/structs passed by value.
Yeah, I guess trying to deal with 128-bit value might be a bit tricky. But having something sane and with compile-time-checked alignment for the
percpu_ddtype is not to be looked over.The existing implementation could be equipped to do a compile time check for the proper alignment if the pointer passed is constant.
"if the pointer passed is constant" -> if you use the actual type of percpu_dd to check the alignment, then you can do an alignment check at compile-time even for a non-const parameter. The requirement imposed on typing will take care to make sure that even a non-const pointer will have the proper alignment.
Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 4 lines · Mathieu Desnoyers
"if the pointer passed is constant" -> if you use the actual type of
percpu_ddto check the alignment, then you can do an alignment check at compile-time even for a non-const parameter. The requirement imposed on typing will take care to make sure that even a non-const pointer will have the proper alignment.
True that would be a benefit if we used something like percpu_dd. If somehow we can marry that to a parameter list that only includes single word entities then we would have the best of both.
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Mathieu DesnoyersWhat is the problem with passing 128-bit structures by value? I'm quite sure *that* is okay.
Quoted 52 lines · Mathieu Desnoyers
Quoted 3 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 1 line · H. Peter Anvin
We could do cmpxchg with a structure... the problem with a lon int
type is that Cristoph ran into bugs with
__int128on 64 bits.Quoted 2 lines
We also would need to be pass the structure by value (well its really
a
Quoted 8 lines
variable but its like passing by value) in order to be similar to the other
this_cpu_opsYou'd want either
DEFINE_PERCPU(struct mycustomdoublestruct, percpu_dd)this_cpu_cmpxchg_double(percpu_dd, oldword1, oldword2, newword1,
newword2)
Quoted 2 lines
with the problem of type checking
What is the problem with type checking here ?
We could use a gcc builtin like the following to check if the alignment of
"percpu_dd" meets the double-cas requirements:#define
testmacro(a, b)\__builtin_choose_expr(__alignof__(a) >= 2 * sizeof(unsigned long), \ ((a).low) = (b), \ /* success */ ((a).low) = (void) 0) /* compile-error */Quoted 5 lines
or
this_cpu_cmpxchg_double(percpu_dd, old_dd, new_dd)with the problem of 128 bit constants/structs passed by value.
Yeah, I guess trying to deal with 128-bit value might be a bit tricky. But
having something sane and with compile-time-checked alignment for thepercpu_dd
type is not to be looked over.Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Signature · 1 line
Sent from my mobile phone. Please pardon any lack of formatting.
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Quoted 1 line · H. Peter Anvin
What is the problem with passing 128-bit structures by value? I'm quite sure *that* is okay.
You mentioned that earlier and I tried to pass 128 bit structures. My compiler did not do any of that (gcc 4.4.5). Could come up with some example code to make yourself certain?
Re: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
In reply to Tejun Heo* Tejun Heo ([email protected]) wrote:
[...]
Quoted 7 lines
Quoted 5 lines
(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
Yeah, good point. :-)
For the records, I just noticed that "packed, aligned(8)" can generate unaligned accesses on sparc64 by removing the padding between a "int" and a following pointer. So we should not use it.
Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
H. Peter Anvin
reviewerRe: [cpuops cmpxchg double V2 1/4] Generic support for this_cpu_cmpxchg_double
Don't apply "packed" to a structure which needs padding! That's just user error.
Quoted 25 lines · Mathieu Desnoyers
* Tejun Heo ([email protected]) wrote:
[...]Quoted 7 lines
Quoted 5 lines
(note: packed here along with "aligned" does _not_ generate ugly bytewise read/write memory ops like "packed" alone. The use of "packed" is to let the compiler down-align the structure to the value requested, instead of uselessly aligning it on 32-byte if it chooses to.)
Yeah, good point. :-)
For the records, I just noticed that "packed, aligned(8)" can generate unaligned
accesses on sparc64 by removing the padding between a "int" and a following
pointer. So we should not use it.Thanks,
Mathieu
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Signature · 1 line
Sent from my mobile phone. Please pardon any lack of formatting.
PATCH v2 2/4 x86: this_cpu_cmpxchg_double() support
In reply to Christoph LameterSupport this_cpu_cmpxchg_double using the cmpxchg16b and cmpxchg8b instructions.
--- arch/x86/include/asm/percpu.h | 47 +++++++++++++++++++++++++++++++ arch/x86/lib/Makefile | 1 arch/x86/lib/cmpxchg16b_emu.S | 62 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+)
arch/x86/include/asm/percpu.h +47 -0
@@ -442,6 +442,26 @@ do { \#define irqsafe_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)#endif /* !CONFIG_M386 */ +#ifdef CONFIG_X86_CMPXCHG64 +#define percpu_cmpxchg8b_double (pcp1, o1, o2, n1, n2) \ +({ \ + char __ret; \ + typeof(o1) __o1 = o1; \ + typeof(o1) __n1 = n1; \ + typeof(o2) __o2 = o2; \ + typeof(o2) __n2 = n2; \ + typeof(o2) __dummy = n2; \ + asm volatile("cmpxchg8b "__percpu_arg(1)"\n\tsetz %0\n\t" \ + : "=a"(__ret), "=m" (pcp1), "=d"(__dummy) \ + : "b"(__n1), "c"(__n2), "a"(__o1), "d"(__o2)); \ + __ret; \ +}) + +#define __this_cpu_cmpxchg_double_4 (pcp1, pcp2, o1, o2, n1, n2) percpu_cmpxchg8b_double(pcp1, o1, o2, n1, n2) +#define this_cpu_cmpxchg_double_4 (pcp1, pcp2, o1, o2, n1, n2) percpu_cmpxchg8b_double(pcp1, o1, o2, n1, n2) +#define irqsafe_cpu_cmpxchg_double_4 (pcp1, pcp2, o1, o2, n1, n2) percpu_cmpxchg8b_double(pcp1, o1, o2, n1, n2) +#endif /* CONFIG_X86_CMPXCHG64 */ +/* * Per cpu atomic 64 bit operations are only available under 64 bit. * 32 bit must fall back to generic operations.
@@ -477,6 +497,33 @@ do { \#define this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)#define irqsafe_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) +/* +* Pretty complex macro to generate cmpxchg16 instruction. The instruction +* is not supported on early AMD64 processors so we must be able to emulate +* it in software. The address used in the cmpxchg16 instruction must be +* aligned to a 16 byte boundary. +*/ +#define percpu_cmpxchg16b (pcp1, o1, o2, n1, n2) \ +({ \ + char __ret; \ + typeof(o1) __o1 = o1; \ + typeof(o1) __n1 = n1; \ + typeof(o2) __o2 = o2; \ + typeof(o2) __n2 = n2; \ + typeof(o2) __dummy; \ + alternative_io("call this_cpu_cmpxchg16b_emu\n\t" P6_NOP4, \ + "cmpxchg16b %%gs:(%%rsi)\n\tsetz %0\n\t", \ + X86_FEATURE_CX16, \ + ASM_OUTPUT2("=a"(__ret), "=d"(__dummy)), \ + "S" (&pcp1), "b"(__n1), "c"(__n2), \ + "a"(__o1), "d"(__o2)); \ + __ret; \ +}) + +#define __this_cpu_cmpxchg_double_8 (pcp1, pcp2, o1, o2, n1, n2) percpu_cmpxchg16b(pcp1, o1, o2, n1, n2) +#define this_cpu_cmpxchg_double_8 (pcp1, pcp2, o1, o2, n1, n2) percpu_cmpxchg16b(pcp1, o1, o2, n1, n2) +#define irqsafe_cpu_cmpxchg_double_8 (pcp1, pcp2, o1, o2, n1, n2) percpu_cmpxchg16b(pcp1, o1, o2, n1, n2) +#endif /* This is not atomic against other CPUs -- CPU preemption needs to be off */
arch/x86/lib/Makefile +1 -0
@@ -42,4 +42,5 @@ else lib-y += memmove_64.o memset_64.o lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o + lib-y += cmpxchg16b_emu.o endif
arch/x86/lib/cmpxchg16b_emu.S new file +62 -0
62 lines not on this page · raw message
PATCH v2 3/4 slub: Get rid of slab_free_hook_irq()
In reply to Christoph LameterThe following patch will make the fastpaths lockless and will no longer require interrupts to be disabled. Calling the free hook with irq disabled will no longer be possible.
Move the slab_free_hook_irq() logic into slab_free_hook. Only disable interrupts if the features are selected that require callbacks with interrupts off and reenable after calls have been made.
--- mm/slub.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-)
mm/slub.c +17 -12
@@ -805,14 +805,24 @@ static inline void slab_post_alloc_hook(static inline void slab_free_hook (struct kmem_cache *s,void *x) {kmemleak_free_recursive (x, s->flags); -} -static inline void slab_free_hook_irq (struct kmem_cache *s,void *object) -{ -kmemcheck_slab_free (s, object, s->objsize); -debug_check_no_locks_freed (object, s->objsize); -if (!(s->flags &SLAB_DEBUG_OBJECTS )) -debug_check_no_obj_freed (object, s->objsize); +/* +* Trouble is that we may no longer disable interupts in the fast path +* So in order to make the debug calls that expect irqs to be +* disabled we need to disable interrupts temporarily. +*/ +#if defined(CONFIG_KMEMCHECK ) || defined(CONFIG_LOCKDEP ) + { +unsigned long flags; + +local_irq_save (flags); +kmemcheck_slab_free (s, x, s->objsize); +debug_check_no_locks_freed (x, s->objsize); +if (!(s->flags &SLAB_DEBUG_OBJECTS )) +debug_check_no_obj_freed (x, s->objsize); +local_irq_restore (flags); + } +#endif }/*
@@ -1099,9 +1109,6 @@ static inline void slab_post_alloc_hook(static inline void slab_free_hook (struct kmem_cache *s,void *x) {} -static inline void slab_free_hook_irq (struct kmem_cache *s, -void *object) {} -#endif /* CONFIG_SLUB_DEBUG */ /*
@@ -1893,8 +1900,6 @@ static __always_inline void slab_free(stlocal_irq_save (flags); c =__this_cpu_ptr (s->cpu_slab); -slab_free_hook_irq (s, x); -if (likely (page == c->page && c->node !=NUMA_NO_NODE )) {set_freepointer (s, object, c->freelist); c->freelist = object;
PATCH v2 4/4 Lockless (and preemptless) fastpaths for slub
In reply to Christoph LameterUse the this_cpu_cmpxchg_double functionality to implement a lockless allocation algorithm on arches that support fast this_cpu_ops.
Each of the per cpu pointers is paired with a transaction id that ensures that updates of the per cpu information can only occur in sequence on a certain cpu.
A transaction id is a "long" integer that is comprised of an event number and the cpu number. The event number is incremented for every change to the per cpu state. The cmpxchg instruction can therefore verify for an update that nothing else has interfered and that we are updating the percpu' structure for the processor where we initially picked up the information and that we are also currently on that processor.
So there is no need even to disable preemption.
Test results show that the fastpath cycle count is reduced by up to ~ 40% (alloc/free test goes from ~140 cycles down to ~80). The slowpath for kfree adds a few cycles.
Sadly this does nothing for the slowpath which is where the main issues with performance in slub are but the best case performance rises significantly. (For that see the more complex slub patches that require cmpxchg_double)
Kmalloc: alloc/free test
Before:
10000 times kmalloc(8)/kfree -> 134 cycles
10000 times kmalloc(16)/kfree -> 152 cycles
10000 times kmalloc(32)/kfree -> 144 cycles
10000 times kmalloc(64)/kfree -> 142 cycles
10000 times kmalloc(128)/kfree -> 142 cycles
10000 times kmalloc(256)/kfree -> 132 cycles
10000 times kmalloc(512)/kfree -> 132 cycles
10000 times kmalloc(1024)/kfree -> 135 cycles
10000 times kmalloc(2048)/kfree -> 135 cycles
10000 times kmalloc(4096)/kfree -> 135 cycles
10000 times kmalloc(8192)/kfree -> 144 cycles
10000 times kmalloc(16384)/kfree -> 754 cycles
After:
10000 times kmalloc(8)/kfree -> 78 cycles
10000 times kmalloc(16)/kfree -> 78 cycles
10000 times kmalloc(32)/kfree -> 82 cycles
10000 times kmalloc(64)/kfree -> 88 cycles
10000 times kmalloc(128)/kfree -> 79 cycles
10000 times kmalloc(256)/kfree -> 79 cycles
10000 times kmalloc(512)/kfree -> 85 cycles
10000 times kmalloc(1024)/kfree -> 82 cycles
10000 times kmalloc(2048)/kfree -> 82 cycles
10000 times kmalloc(4096)/kfree -> 85 cycles
10000 times kmalloc(8192)/kfree -> 82 cycles
10000 times kmalloc(16384)/kfree -> 706 cycles
Kmalloc: Repeatedly allocate then free test
Before:
10000 times kmalloc(8) -> 211 cycles kfree -> 113 cycles 10000 times kmalloc(16) -> 174 cycles kfree -> 115 cycles 10000 times kmalloc(32) -> 235 cycles kfree -> 129 cycles 10000 times kmalloc(64) -> 222 cycles kfree -> 120 cycles 10000 times kmalloc(128) -> 343 cycles kfree -> 139 cycles 10000 times kmalloc(256) -> 827 cycles kfree -> 147 cycles 10000 times kmalloc(512) -> 1048 cycles kfree -> 272 cycles 10000 times kmalloc(1024) -> 2043 cycles kfree -> 528 cycles 10000 times kmalloc(2048) -> 4002 cycles kfree -> 571 cycles 10000 times kmalloc(4096) -> 7740 cycles kfree -> 628 cycles 10000 times kmalloc(8192) -> 8062 cycles kfree -> 850 cycles 10000 times kmalloc(16384) -> 8895 cycles kfree -> 1249 cycles
After:
10000 times kmalloc(8) -> 190 cycles kfree -> 129 cycles 10000 times kmalloc(16) -> 76 cycles kfree -> 123 cycles 10000 times kmalloc(32) -> 126 cycles kfree -> 124 cycles 10000 times kmalloc(64) -> 181 cycles kfree -> 128 cycles 10000 times kmalloc(128) -> 310 cycles kfree -> 140 cycles 10000 times kmalloc(256) -> 809 cycles kfree -> 165 cycles 10000 times kmalloc(512) -> 1005 cycles kfree -> 269 cycles 10000 times kmalloc(1024) -> 1999 cycles kfree -> 527 cycles 10000 times kmalloc(2048) -> 3967 cycles kfree -> 570 cycles 10000 times kmalloc(4096) -> 7658 cycles kfree -> 637 cycles 10000 times kmalloc(8192) -> 8111 cycles kfree -> 859 cycles 10000 times kmalloc(16384) -> 8791 cycles kfree -> 1173 cycles
--- include/linux/slub_def.h | 5 - mm/slub.c | 205 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 207 insertions(+), 3 deletions(-)
include/linux/slub_def.h +4 -1
@@ -36,7 +36,10 @@ enum stat_item { NR_SLUB_STAT_ITEMS };struct kmem_cache_cpu { -void **freelist;/* Pointer to first free per cpu object */ +void **freelist;/* Pointer to next available object */ +#ifdef CONFIG_CMPXCHG_LOCAL +unsigned long tid;/* Globally unique transaction id */ +#endif struct page *page;/* The slab from which we are allocating */ int node;/* The node of the page (or -1 for debug) */ #ifdef CONFIG_SLUB_STATS
mm/slub.c +203 -2
301 lines not on this page · raw message