diff --git a/hypervisor/include/hv_lib.h b/hypervisor/include/hv_lib.h index 2ccd5fd7a..a206d7814 100644 --- a/hypervisor/include/hv_lib.h +++ b/hypervisor/include/hv_lib.h @@ -40,6 +40,7 @@ #include #include #include +#include #include #include diff --git a/hypervisor/include/lib/atomic.h b/hypervisor/include/lib/atomic.h new file mode 100644 index 000000000..394d0c94b --- /dev/null +++ b/hypervisor/include/lib/atomic.h @@ -0,0 +1,355 @@ +/*- + * Copyright (c) 1998 Doug Rabson + * Copyright (c) 2018 Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ATOMIC_H +#define ATOMIC_H + +#define BUS_LOCK "lock ; " +/* + * #define atomic_set_char(P, V) (*(unsigned char *)(P) |= (V)) + */ +static inline void atomic_set_char(unsigned char *p, unsigned char v) +{ + __asm __volatile(BUS_LOCK "orb %b1,%0" + : "+m" (*p) + : "q" (v) + : "cc", "memory"); +} + +/* + * #define atomic_clear_char(P, V) (*(unsigned char *)(P) &= ~(V)) + */ +static inline void atomic_clear_char(unsigned char *p, unsigned char v) +{ + __asm __volatile(BUS_LOCK "andb %b1,%0" + : "+m" (*p) + : "q" (~v) + : "cc", "memory"); +} + +/* + * #define atomic_add_char(P, V) (*(unsigned char *)(P) += (V)) + */ +static inline void atomic_add_char(unsigned char *p, unsigned char v) +{ + __asm __volatile(BUS_LOCK "addb %b1,%0" + : "+m" (*p) + : "q" (v) + : "cc", "memory"); +} + +/* + * #define atomic_subtract_char(P, V) (*(unsigned char *)(P) -= (V)) + */ +static inline void atomic_subtract_char(unsigned char *p, unsigned char v) +{ + __asm __volatile(BUS_LOCK "subb %b1,%0" + : "+m" (*p) + : "q" (v) + : "cc", "memory"); +} + +/* + * #define atomic_set_short(P, V) (*(unsigned short *)(P) |= (V)) + */ +static inline void atomic_set_short(unsigned short *p, unsigned short v) +{ + __asm __volatile(BUS_LOCK "orw %w1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_clear_short(P, V) (*(unsigned short *)(P) &= ~(V)) + */ +static inline void atomic_clear_short(unsigned short *p, unsigned short v) +{ + __asm __volatile(BUS_LOCK "andw %w1,%0" + : "+m" (*p) + : "r" (~v) + : "cc", "memory"); +} + +/* + * #define atomic_add_short(P, V) (*(unsigned short *)(P) += (V)) + */ +static inline void atomic_add_short(unsigned short *p, unsigned short v) +{ + __asm __volatile(BUS_LOCK "addw %w1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_subtract_short(P, V) (*(unsigned short *)(P) -= (V)) + */ +static inline void atomic_subtract_short(unsigned short *p, unsigned short v) +{ + __asm __volatile(BUS_LOCK "subw %w1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_set_int(P, V) (*(unsigned int *)(P) |= (V)) + */ +static inline void atomic_set_int(unsigned int *p, unsigned int v) +{ + __asm __volatile(BUS_LOCK "orl %1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_clear_int(P, V) (*(unsigned int *)(P) &= ~(V)) + */ +static inline void atomic_clear_int(unsigned int *p, unsigned int v) +{ + __asm __volatile(BUS_LOCK "andl %1,%0" + : "+m" (*p) + : "r" (~v) + : "cc", "memory"); +} + +/* + * #define atomic_add_int(P, V) (*(unsigned int *)(P) += (V)) + */ +static inline void atomic_add_int(unsigned int *p, unsigned int v) +{ + __asm __volatile(BUS_LOCK "addl %1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_subtract_int(P, V) (*(unsigned int *)(P) -= (V)) + */ +static inline void atomic_subtract_int(unsigned int *p, unsigned int v) +{ + __asm __volatile(BUS_LOCK "subl %1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_swap_int(P, V) \ + * (return (*(unsigned int *)(P)); *(unsigned int *)(P) = (V);) + */ +static inline int atomic_swap_int(unsigned int *p, unsigned int v) +{ + __asm __volatile(BUS_LOCK "xchgl %1,%0" + : "+m" (*p), "+r" (v) + : + : "cc", "memory"); + return v; +} + +/* + * #define atomic_readandclear_int(P) \ + * (return (*(unsigned int *)(P)); *(unsigned int *)(P) = 0;) + */ +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) + +/* + * #define atomic_set_long(P, V) (*(unsigned long *)(P) |= (V)) + */ +static inline void atomic_set_long(unsigned long *p, unsigned long v) +{ + __asm __volatile(BUS_LOCK "orq %1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) + */ +static inline void atomic_clear_long(unsigned long *p, unsigned long v) +{ + __asm __volatile(BUS_LOCK "andq %1,%0" + : "+m" (*p) + : "r" (~v) + : "cc", "memory"); +} + +/* + * #define atomic_add_long(P, V) (*(unsigned long *)(P) += (V)) + */ +static inline void atomic_add_long(unsigned long *p, unsigned long v) +{ + __asm __volatile(BUS_LOCK "addq %1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_subtract_long(P, V) (*(unsigned long *)(P) -= (V)) + */ +static inline void atomic_subtract_long(unsigned long *p, unsigned long v) +{ + __asm __volatile(BUS_LOCK "subq %1,%0" + : "+m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_swap_long(P, V) \ + * (return (*(unsigned long *)(P)); *(unsigned long *)(P) = (V);) + */ +static inline long atomic_swap_long(unsigned long *p, unsigned long v) +{ + __asm __volatile(BUS_LOCK "xchgq %1,%0" + : "+m" (*p), "+r" (v) + : + : "cc", "memory"); + return v; +} + +/* + * #define atomic_readandclear_long(P) \ + * (return (*(unsigned long *)(P)); *(unsigned long *)(P) = 0;) + */ +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + +/* + * #define atomic_load_acq_int(P) (*(unsigned int*)(P)) + */ +static inline int atomic_load_acq_int(unsigned int *p) +{ + int ret; + + __asm __volatile("movl %1,%0" + : "=r"(ret) + : "m" (*p) + : "cc", "memory"); + return ret; +} + +/* + * #define atomic_store_rel_int(P, V) (*(unsigned int *)(P) = (V)) + */ +static inline void atomic_store_rel_int(unsigned int *p, unsigned int v) +{ + __asm __volatile("movl %1,%0" + : "=m" (*p) + : "r" (v) + : "cc", "memory"); +} + +/* + * #define atomic_load_acq_long(P) (*(unsigned long*)(P)) + */ +static inline long atomic_load_acq_long(unsigned long *p) +{ + long ret; + + __asm __volatile("movq %1,%0" + : "=r"(ret) + : "m" (*p) + : "cc", "memory"); + return ret; +} + +/* + * #define atomic_store_rel_long(P, V) (*(unsigned long *)(P) = (V)) + */ +static inline void atomic_store_rel_long(unsigned long *p, unsigned long v) +{ + __asm __volatile("movq %1,%0" + : "=m" (*p) + : "r" (v) + : "cc", "memory"); +} + +static inline int atomic_cmpxchg_int(unsigned int *p, + int old, int new) +{ + int ret; + + __asm __volatile(BUS_LOCK "cmpxchgl %2,%1" + : "=a" (ret), "+m" (*p) + : "r" (new), "0" (old) + : "memory"); + + return ret; +} + +#define atomic_load_acq_32 atomic_load_acq_int +#define atomic_store_rel_32 atomic_store_rel_int +#define atomic_load_acq_64 atomic_load_acq_long +#define atomic_store_rel_64 atomic_store_rel_long + +/* + * #define atomic_xadd_int(P, V) \ + * (return (*(unsigned long *)(P)); *(unsigned long *)(P) += (V);) + */ +static inline int atomic_xadd_int(unsigned int *p, unsigned int v) +{ + __asm __volatile(BUS_LOCK "xaddl %0,%1" + : "+r" (v), "+m" (*p) + : + : "cc", "memory"); + return v; +} + +static inline int atomic_add_return(int v, unsigned int *p) +{ + return v + atomic_xadd_int(p, v); +} + +static inline int atomic_sub_return(int v, unsigned int *p) +{ + return atomic_xadd_int(p, -v) - v; +} + +#define atomic_inc_return(v) atomic_add_return(1, (v)) +#define atomic_dec_return(v) atomic_sub_return(1, (v)) + +static inline int +atomic_cmpset_long(unsigned long *dst, unsigned long expect, unsigned long src) +{ + unsigned char res; + + __asm __volatile(BUS_LOCK "cmpxchg %3,%1\n\tsete %0" + : "=q" (res), "+m" (*dst), "+a" (expect) + : "r" (src) + : "memory", "cc"); + + return res; +} + +#endif /* ATOMIC_H*/ diff --git a/hypervisor/include/lib/bits.h b/hypervisor/include/lib/bits.h index 9d6538a3b..f3bb75491 100644 --- a/hypervisor/include/lib/bits.h +++ b/hypervisor/include/lib/bits.h @@ -31,313 +31,6 @@ #define BITS_H #define BUS_LOCK "lock ; " -/* - * #define atomic_set_char(P, V) (*(unsigned char *)(P) |= (V)) - */ -static inline void atomic_set_char(unsigned char *p, unsigned char v) -{ - __asm __volatile(BUS_LOCK "orb %b1,%0" - : "+m" (*p) - : "q" (v) - : "cc", "memory"); -} - -/* - * #define atomic_clear_char(P, V) (*(unsigned char *)(P) &= ~(V)) - */ -static inline void atomic_clear_char(unsigned char *p, unsigned char v) -{ - __asm __volatile(BUS_LOCK "andb %b1,%0" - : "+m" (*p) - : "q" (~v) - : "cc", "memory"); -} - -/* - * #define atomic_add_char(P, V) (*(unsigned char *)(P) += (V)) - */ -static inline void atomic_add_char(unsigned char *p, unsigned char v) -{ - __asm __volatile(BUS_LOCK "addb %b1,%0" - : "+m" (*p) - : "q" (v) - : "cc", "memory"); -} - -/* - * #define atomic_subtract_char(P, V) (*(unsigned char *)(P) -= (V)) - */ -static inline void atomic_subtract_char(unsigned char *p, unsigned char v) -{ - __asm __volatile(BUS_LOCK "subb %b1,%0" - : "+m" (*p) - : "q" (v) - : "cc", "memory"); -} - -/* - * #define atomic_set_short(P, V) (*(unsigned short *)(P) |= (V)) - */ -static inline void atomic_set_short(unsigned short *p, unsigned short v) -{ - __asm __volatile(BUS_LOCK "orw %w1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_clear_short(P, V) (*(unsigned short *)(P) &= ~(V)) - */ -static inline void atomic_clear_short(unsigned short *p, unsigned short v) -{ - __asm __volatile(BUS_LOCK "andw %w1,%0" - : "+m" (*p) - : "r" (~v) - : "cc", "memory"); -} - -/* - * #define atomic_add_short(P, V) (*(unsigned short *)(P) += (V)) - */ -static inline void atomic_add_short(unsigned short *p, unsigned short v) -{ - __asm __volatile(BUS_LOCK "addw %w1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_subtract_short(P, V) (*(unsigned short *)(P) -= (V)) - */ -static inline void atomic_subtract_short(unsigned short *p, unsigned short v) -{ - __asm __volatile(BUS_LOCK "subw %w1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_set_int(P, V) (*(unsigned int *)(P) |= (V)) - */ -static inline void atomic_set_int(unsigned int *p, unsigned int v) -{ - __asm __volatile(BUS_LOCK "orl %1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_clear_int(P, V) (*(unsigned int *)(P) &= ~(V)) - */ -static inline void atomic_clear_int(unsigned int *p, unsigned int v) -{ - __asm __volatile(BUS_LOCK "andl %1,%0" - : "+m" (*p) - : "r" (~v) - : "cc", "memory"); -} - -/* - * #define atomic_add_int(P, V) (*(unsigned int *)(P) += (V)) - */ -static inline void atomic_add_int(unsigned int *p, unsigned int v) -{ - __asm __volatile(BUS_LOCK "addl %1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_subtract_int(P, V) (*(unsigned int *)(P) -= (V)) - */ -static inline void atomic_subtract_int(unsigned int *p, unsigned int v) -{ - __asm __volatile(BUS_LOCK "subl %1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_swap_int(P, V) \ - * (return (*(unsigned int *)(P)); *(unsigned int *)(P) = (V);) - */ -static inline int atomic_swap_int(unsigned int *p, unsigned int v) -{ - __asm __volatile(BUS_LOCK "xchgl %1,%0" - : "+m" (*p), "+r" (v) - : - : "cc", "memory"); - return v; -} - -/* - * #define atomic_readandclear_int(P) \ - * (return (*(unsigned int *)(P)); *(unsigned int *)(P) = 0;) - */ -#define atomic_readandclear_int(p) atomic_swap_int(p, 0) - -/* - * #define atomic_set_long(P, V) (*(unsigned long *)(P) |= (V)) - */ -static inline void atomic_set_long(unsigned long *p, unsigned long v) -{ - __asm __volatile(BUS_LOCK "orq %1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) - */ -static inline void atomic_clear_long(unsigned long *p, unsigned long v) -{ - __asm __volatile(BUS_LOCK "andq %1,%0" - : "+m" (*p) - : "r" (~v) - : "cc", "memory"); -} - -/* - * #define atomic_add_long(P, V) (*(unsigned long *)(P) += (V)) - */ -static inline void atomic_add_long(unsigned long *p, unsigned long v) -{ - __asm __volatile(BUS_LOCK "addq %1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_subtract_long(P, V) (*(unsigned long *)(P) -= (V)) - */ -static inline void atomic_subtract_long(unsigned long *p, unsigned long v) -{ - __asm __volatile(BUS_LOCK "subq %1,%0" - : "+m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_swap_long(P, V) \ - * (return (*(unsigned long *)(P)); *(unsigned long *)(P) = (V);) - */ -static inline long atomic_swap_long(unsigned long *p, unsigned long v) -{ - __asm __volatile(BUS_LOCK "xchgq %1,%0" - : "+m" (*p), "+r" (v) - : - : "cc", "memory"); - return v; -} - -/* - * #define atomic_readandclear_long(P) \ - * (return (*(unsigned long *)(P)); *(unsigned long *)(P) = 0;) - */ -#define atomic_readandclear_long(p) atomic_swap_long(p, 0) - -/* - * #define atomic_load_acq_int(P) (*(unsigned int*)(P)) - */ -static inline int atomic_load_acq_int(unsigned int *p) -{ - int ret; - - __asm __volatile("movl %1,%0" - : "=r"(ret) - : "m" (*p) - : "cc", "memory"); - return ret; -} - -/* - * #define atomic_store_rel_int(P, V) (*(unsigned int *)(P) = (V)) - */ -static inline void atomic_store_rel_int(unsigned int *p, unsigned int v) -{ - __asm __volatile("movl %1,%0" - : "=m" (*p) - : "r" (v) - : "cc", "memory"); -} - -/* - * #define atomic_load_acq_long(P) (*(unsigned long*)(P)) - */ -static inline long atomic_load_acq_long(unsigned long *p) -{ - long ret; - - __asm __volatile("movq %1,%0" - : "=r"(ret) - : "m" (*p) - : "cc", "memory"); - return ret; -} - -/* - * #define atomic_store_rel_long(P, V) (*(unsigned long *)(P) = (V)) - */ -static inline void atomic_store_rel_long(unsigned long *p, unsigned long v) -{ - __asm __volatile("movq %1,%0" - : "=m" (*p) - : "r" (v) - : "cc", "memory"); -} - -static inline int atomic_cmpxchg_int(unsigned int *p, - int old, int new) -{ - int ret; - - __asm __volatile(BUS_LOCK "cmpxchgl %2,%1" - : "=a" (ret), "+m" (*p) - : "r" (new), "0" (old) - : "memory"); - - return ret; -} - -#define atomic_load_acq_32 atomic_load_acq_int -#define atomic_store_rel_32 atomic_store_rel_int -#define atomic_load_acq_64 atomic_load_acq_long -#define atomic_store_rel_64 atomic_store_rel_long - -/* - * #define atomic_xadd_int(P, V) \ - * (return (*(unsigned long *)(P)); *(unsigned long *)(P) += (V);) - */ -static inline int atomic_xadd_int(unsigned int *p, unsigned int v) -{ - __asm __volatile(BUS_LOCK "xaddl %0,%1" - : "+r" (v), "+m" (*p) - : - : "cc", "memory"); - return v; -} - -static inline int atomic_add_return(int v, unsigned int *p) -{ - return v + atomic_xadd_int(p, v); -} - -static inline int atomic_sub_return(int v, unsigned int *p) -{ - return atomic_xadd_int(p, -v) - v; -} - -#define atomic_inc_return(v) atomic_add_return(1, (v)) -#define atomic_dec_return(v) atomic_sub_return(1, (v)) static inline unsigned int bsrl(unsigned int mask) @@ -518,19 +211,6 @@ bitmap_ffs(unsigned long *bits) return ffsl(*bits); } -static inline int -atomic_cmpset_long(unsigned long *dst, unsigned long expect, unsigned long src) -{ - unsigned char res; - - __asm __volatile(BUS_LOCK "cmpxchg %3,%1\n\tsete %0" - : "=q" (res), "+m" (*dst), "+a" (expect) - : "r" (src) - : "memory", "cc"); - - return res; -} - /*bit scan forward for the least significant bit '0'*/ static inline int get_first_zero_bit(unsigned long value)