/*
 * CoreFreq
 * Copyright (C) 2015-2026 CYRIL COURTIAT
 * Licenses: GPL2
 */

#define FEAT_MESSAGE(_msg)		_Pragma(#_msg)
#define FEAT_MSG(_msg)			FEAT_MESSAGE(message(#_msg))

enum {
	SYNC	=  0,
	SYNC0	= 10,
	SYNC1	= 11,
	BURN	= 31,
	COMP0	= 40,
	COMP1	= 41,
	NTFY0	= 50,
	NTFY1	= 51,
	NTFY	= 63
};

#define BIT_MASK_SYNC							\
	0b0000000000000000000000000000000000000000000000000000110000000000LLU

#define BIT_MASK_COMP							\
	0b0000000000000000000000110000000000000000000000000000000000000000LLU

#define BIT_MASK_NTFY							\
	0b0000000000001100000000000000000000000000000000000000000000000000LLU

#define MAXCOUNTER(M, m)	((M) > (m) ? (M) : (m))
#define MINCOUNTER(m, M)	((m) < (M) ? (m) : (M))

#define CORE_COUNT_MASK(_cc)	(_cc - 1)
#define CORE_WORD_TOP(_cc)	(CORE_COUNT_MASK(_cc) >> 6)
#define CORE_WORD_MOD(_cc_,_offset_) ((_offset_ & CORE_COUNT_MASK(_cc_)) & 0x3f)
#define CORE_WORD_POS(_cc_,_offset_) ((_offset_ & CORE_COUNT_MASK(_cc_)) >> 6)

typedef unsigned long long int	Bit1024[16];
typedef unsigned long long int	Bit512[8];
typedef unsigned long long int	Bit256[4];
typedef unsigned long long int	Bit128[2];
typedef unsigned long long int	Bit64;
typedef unsigned int		Bit32;

#define BitT2(_cc)		Bit##_cc
#define BitT1(_cc)		BitT2(_cc)
#define BitCC			BitT1(CORE_COUNT)

#if (CORE_COUNT == 64)
#define InitCC(_val)		(_val)
#else
#define InitCC(_val)		{[0 ... CORE_WORD_TOP(CORE_COUNT) - 1] = _val}
#endif

#define ATOMIC_SEED 0x436f726546726571LLU

#define LOCKLESS LOCK_LESS
#define BUS_LOCK FULL_LOCK

#define BARRIER(_option)						\
__asm__ volatile							\
(									\
	"dsb	" #_option			"\n\t"			\
	"isb"								\
	:								\
	:								\
	: "memory"							\
)

#define WBINVD()							\
__asm__ volatile							\
(									\
	"ic iallu"							\
	:								\
	:								\
	: "memory"							\
)

#define SERIALIZE()							\
__asm__ volatile							\
(									\
	"isb" 								\
	:								\
	:								\
	: "memory"							\
)

#define RDTSC64(_mem64) 						\
__asm__ volatile							\
(									\
	"mrs	%0	,	cntvct_el0"				\
	: "=r" (_mem64) 						\
	:								\
	: "cc", "memory"						\
)

#define ASM_RDTSC(_reg) 						\
	"# Read variant TSC."			"\n\t"			\
	"mrs	" #_reg ",	cntvct_el0"	"\n\t"

#define ASM_CODE_RDPMC(_ctr, _reg)					\
	"# Read PMU counter."			"\n\t"			\
	"mrs	" #_reg ", " #_ctr		"\n\t"

#define ASM_RDPMC(_ctr, _reg) ASM_CODE_RDPMC(_ctr, _reg)

#define RDPMC(_ctr, _reg, _mem) 					\
__asm__ volatile							\
(									\
	ASM_CODE_RDPMC(_ctr, _reg)					\
	"# Store values into memory."		"\n\t"			\
	"str	" #_reg ",	%0"					\
	: "=m" (_mem)							\
	:								\
	: "%" #_reg"", 							\
	  "cc", "memory"						\
)

#define ASM_RDTSC_PMCx1(_reg0, _reg1,					\
			_tsc_inst, mem_tsc,				\
			_ctr1, _mem1)					\
__asm__ volatile							\
(									\
	_tsc_inst(_reg0)						\
	ASM_RDPMC(_ctr1, _reg1) 					\
	"# Store values into memory."		"\n\t"			\
	"str	" #_reg0 ",	%0"		"\n\t"			\
	"str	" #_reg1 ",	%1"					\
	: "=m" (mem_tsc),	"=m" (_mem1)				\
	:								\
	: "%" #_reg0""	,	"%" #_reg1"",				\
	  "cc", "memory"						\
)

#define RDTSC_PMCx1(mem_tsc, ...)					\
ASM_RDTSC_PMCx1(x14, x15, ASM_RDTSC, mem_tsc, __VA_ARGS__)

#define	_BITSET_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define	_BITSET_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define	_BITSET_COMMON_INST						\
		"tst	x11, x12"		"\n\t"			\
		"cset	w10, ne"		"\n\t"			\
		"strb	w10, %[ret]"		"\n\t"			\
		"orr	x11, x11, x12"		"\n\t"

#define	_BITSET_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define	_BITSET_POST_INST_LOCK_LESS					\
		"str	x11, [%[addr]]"

#define	_BITSET_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%w10", "%x11", "%x12"

#define	_BITSET_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%w10", "%x11", "%x12"

#define _BITSET_GPR(_lock, _base, _offset)				\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		"mov	x12, #1"		"\n\t"			\
		"lsl	x12, x12, %[offset]"	"\n\t"			\
		_BITSET_PRE_INST_##_lock				\
		_BITSET_COMMON_INST					\
		_BITSET_POST_INST_##_lock				\
		: [ret] "=m" (_ret)					\
		: [addr] "r" (&_base),					\
		  [offset] "r" ((Bit64)(_offset))			\
		_BITSET_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define _BITSET_IMM(_lock, _base, _imm6)				\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		"mov	x12, #1"		"\n\t"			\
		"lsl	x12, x12, %[imm6]"	"\n\t"			\
		_BITSET_PRE_INST_##_lock				\
		_BITSET_COMMON_INST					\
		_BITSET_POST_INST_##_lock				\
		: [ret] "=m" (_ret)					\
		: [addr] "r" (&_base),					\
		  [imm6] "i" (_imm6)					\
		_BITSET_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define _BITCLR_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define _BITCLR_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define _BITCLR_COMMON_INST						\
		"tst	x11, x12"		"\n\t"			\
		"cset	w10, ne"		"\n\t"			\
		"strb	w10, %[ret]"		"\n\t"			\
		"bic	x11, x11, x12"		"\n\t"

#define _BITCLR_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITCLR_POST_INST_LOCK_LESS					\
		"str	x11, [%[addr]]"

#define _BITCLR_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%w10", "%x11", "%x12"

#define _BITCLR_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%w10", "%x11", "%x12"

#define _BITCLR_GPR(_lock, _base, _offset)				\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		"mov	x12, #1"		"\n\t"			\
		"lsl	x12, x12, %[offset]"	"\n\t"			\
		_BITCLR_PRE_INST_##_lock				\
		_BITCLR_COMMON_INST					\
		_BITCLR_POST_INST_##_lock				\
		: [ret] "=m" (_ret)					\
		: [addr] "r" (&_base),					\
		  [offset] "r" ((Bit64)(_offset))			\
		_BITCLR_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define _BITCLR_IMM(_lock, _base, _imm6)				\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		"mov	x12, #1"		"\n\t"			\
		"lsl	x12, x12, %[imm6]"	"\n\t"			\
		_BITCLR_PRE_INST_##_lock				\
		_BITCLR_COMMON_INST					\
		_BITCLR_POST_INST_##_lock				\
		: [ret] "=m" (_ret)					\
		: [addr] "r" (&_base),					\
		  [imm6] "i" (_imm6)					\
		_BITCLR_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define _BIT_TEST_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define _BIT_TEST_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define _BIT_TEST_COMMON_INST						\
		"tst	x11, x12"		"\n\t"			\
		"cset	w10, ne"		"\n\t"			\
		"strb	w10, %[ret]"		"\n\t"

#define _BIT_TEST_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BIT_TEST_POST_INST_LOCK_LESS					\
		"#	NOP"

#define _BIT_TEST_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%w10", "%x11", "%x12"

#define _BIT_TEST_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%w10", "%x11", "%x12"

#define _BIT_TEST_GPR(_lock, _base, _offset)				\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		"mov	x12, #1"		"\n\t"			\
		"lsl	x12, x12, %[offset]"	"\n\t"			\
		_BIT_TEST_PRE_INST_##_lock				\
		_BIT_TEST_COMMON_INST					\
		_BIT_TEST_POST_INST_##_lock				\
		: [ret] "=m" (_ret)					\
		: [addr] "r" (&_base),					\
		  [offset] "r" ((Bit64)(_offset))			\
		_BIT_TEST_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define _BIT_TEST_IMM(_lock, _base, _imm6)				\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		"mov	x12, #1"		"\n\t"			\
		"lsl	x12, x12, %[imm6]"	"\n\t"			\
		_BIT_TEST_PRE_INST_##_lock				\
		_BIT_TEST_COMMON_INST					\
		_BIT_TEST_POST_INST_##_lock				\
		: [ret] "=m" (_ret)					\
		: [addr] "r" (&_base),					\
		  [imm6] "i" (_imm6)					\
		_BIT_TEST_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define _BITWISEAND_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define _BITWISEAND_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define _BITWISEAND_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITWISEAND_POST_INST_LOCK_LESS 				\
		"#	NOP"

#define _BITWISEAND_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%x10", "%x11"

#define _BITWISEAND_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%x10", "%x11"

#define _BITWISEAND(_lock, _opl, _opr)					\
({									\
	volatile Bit64 _dest __attribute__ ((aligned (8)));		\
									\
	__asm__ volatile						\
	(								\
		_BITWISEAND_PRE_INST_##_lock				\
		"and	x10, x11, %[opr]"	"\n\t"			\
		"str	x10, %[dest]"		"\n\t"			\
		 _BITWISEAND_POST_INST_##_lock				\
		: [dest] "=m" (_dest)					\
		: [addr] "r" (&_opl),					\
		  [opr]  "Lr" (_opr)					\
		_BITWISEAND_CLOBBERS_##_lock				\
	);								\
	_dest;								\
})

#define _BITWISEOR_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define _BITWISEOR_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define _BITWISEOR_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITWISEOR_POST_INST_LOCK_LESS					\
		"#	NOP"

#define _BITWISEOR_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%x10", "%x11"

#define _BITWISEOR_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%x10", "%x11"

#define _BITWISEOR(_lock, _opl, _opr)					\
({									\
	volatile Bit64 _dest __attribute__ ((aligned (8)));		\
									\
	__asm__ volatile						\
	(								\
		_BITWISEOR_PRE_INST_##_lock				\
		"orr	x10, x11, %[opr]"	"\n\t"			\
		"str	x10, %[dest]"		"\n\t"			\
		_BITWISEOR_POST_INST_##_lock				\
		: [dest] "=m" (_dest)					\
		: [addr] "r" (&_opl),					\
		  [opr]  "Lr" (_opr)					\
		_BITWISEOR_CLOBBERS_##_lock				\
	);								\
	_dest;								\
})

#define _BITWISEXOR_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define _BITWISEXOR_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define _BITWISEXOR_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITWISEXOR_POST_INST_LOCK_LESS					\
		"#	NOP"

#define _BITWISEXOR_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%x10", "%x11"

#define _BITWISEXOR_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%x10", "%x11"

#define _BITWISEXOR(_lock, _opl, _opr)					\
({									\
	volatile Bit64 _dest __attribute__ ((aligned (8)));		\
									\
	__asm__ volatile						\
	(								\
		_BITWISEXOR_PRE_INST_##_lock				\
		"eor	x10, x11, %[opr]"	"\n\t"			\
		"str	x10, %[dest]"		"\n\t"			\
		_BITWISEXOR_POST_INST_##_lock				\
		: [dest] "=m" (_dest)					\
		: [addr] "r" (&_opl),					\
		  [opr]  "Lr" (_opr)					\
		_BITWISEXOR_CLOBBERS_##_lock				\
	);								\
	_dest;								\
})

#define BITSET(_lock, _base, _offset)					\
(									\
	__builtin_constant_p(_offset) ? 				\
		_BITSET_IMM(_lock, _base, _offset)			\
	:	_BITSET_GPR(_lock, _base, _offset)			\
)

#define BITCLR(_lock, _base, _offset)					\
(									\
	__builtin_constant_p(_offset) ? 				\
		_BITCLR_IMM(_lock, _base, _offset)			\
	:	_BITCLR_GPR(_lock, _base, _offset)			\
)

#define BITVAL_3xPARAM(_lock, _base, _offset)				\
(									\
	__builtin_constant_p(_offset) ? 				\
		_BIT_TEST_IMM(_lock, _base, _offset)			\
	:	_BIT_TEST_GPR(_lock, _base, _offset)			\
)

#define BITVAL_2xPARAM(_base, _offset)					\
(									\
	BITVAL_3xPARAM(LOCKLESS, _base, _offset)			\
)

#define BITVAL_DISPATCH( _1, _2, _3, BITVAL_CURSOR, ... )		\
	BITVAL_CURSOR

#define BITVAL(...)							\
	BITVAL_DISPATCH(__VA_ARGS__,	BITVAL_3xPARAM, /*3*/		\
					BITVAL_2xPARAM, /*2*/		\
					NULL)		/*1*/		\
							( __VA_ARGS__ )

#define BITCPL(_src)							\
({									\
	unsigned long long _dest;					\
									\
	__asm__ volatile						\
	(								\
		"neg	%x[dest], %x[src]"				\
		: [dest] "=rQ" (_dest)					\
		: [src] "Lr" (_src)					\
		: "cc", "memory"					\
	);								\
	_dest;								\
})

#define BITWISEAND(_lock, _opl, _opr)	_BITWISEAND(_lock, _opl, _opr)
#define BITWISEOR(_lock, _opl, _opr)	_BITWISEOR(_lock, _opl, _opr)
#define BITWISEXOR(_lock, _opl, _opr)	_BITWISEXOR(_lock, _opl, _opr)

#define _BITSTOR_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	xzr, [%[addr]]" 	"\n\t"

#define _BITSTOR_PRE_INST_LOCK_LESS					\
		"#	NOP"			"\n\t"

#define _BITSTOR_POST_INST_FULL_LOCK					\
		"stxr	w9, %[src], [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITSTOR_POST_INST_LOCK_LESS					\
		"str	%[src], [%[addr]]"

#define _BITSTOR_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9"

#define _BITSTOR_CLOBBERS_LOCK_LESS					\
		: "cc", "memory"

#define _BITSTOR(_lock, _dest, _src)					\
({									\
	__asm__ volatile						\
	(								\
		_BITSTOR_PRE_INST_##_lock				\
		_BITSTOR_POST_INST_##_lock				\
		:							\
		: [addr] "r" (&_dest),					\
		  [src] "r" (_src)					\
		_BITSTOR_CLOBBERS_##_lock				\
	);								\
})

#define BITSTOR(_lock, _dest, _src)					\
	_BITSTOR(_lock, _dest, _src)

#define BITBSF(_base, _index)						\
({									\
	register unsigned char _ret;					\
	if (_base) {							\
		_index = 0;						\
		while (BITVAL(_base, _index) == 0) {			\
			if (_index < (8 * sizeof(_base)) - 1) {		\
				_index = _index + 1;			\
			} else {					\
				break;					\
			}						\
		}							\
		_ret = 0;						\
	} else {							\
		_index = 0;						\
		_ret = 1;						\
	}								\
	_ret;								\
})

#define BITBSR(_base, _index)						\
({									\
	register unsigned char _ret;					\
	if (_base) {							\
		_index = (8 * sizeof(_base)) - 1;			\
		while (BITVAL(_base, _index) == 0) {			\
			if (_index > 0) {				\
				_index = _index - 1;			\
			} else {					\
				break;					\
			}						\
		}							\
		_ret = 0;						\
	} else {							\
		_index = 0;						\
		_ret = 1;						\
	}								\
	_ret;								\
})

#define BITEXTRZ(_src, _offset, _length)				\
({									\
	unsigned long long _dest = ((1 << _length) - 1) 		\
				 & (_src >> _offset);			\
	_dest;								\
})

#define BIT_ATOM_INIT(atom, seed)	atomic64_set(&(atom), (seed))

#define BIT_ATOM_TRYLOCK(_lock, atom, seed)				\
({									\
	atomic64_cmpxchg(&(atom), (seed), 0) == (seed); 		\
})

#define BIT_ATOM_UNLOCK(_lock, atom, seed)				\
({									\
	atomic64_set(&(atom), (seed));					\
	0ULL;								\
})

#define _BITWISESET_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	x11, [%[addr]]" 	"\n\t"

#define _BITWISESET_PRE_INST_LOCK_LESS					\
		"ldr	x11, [%[addr]]" 	"\n\t"

#define _BITWISESET_POST_INST_FULL_LOCK					\
		"stxr	w9, x11, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITWISESET_POST_INST_LOCK_LESS					\
		"str	x11, [%[addr]]"

#define _BITWISESET_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%x11"

#define _BITWISESET_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%x11"

#define _BITWISESET(_lock, _opl, _opr)					\
({									\
	__asm__ volatile						\
	(								\
		_BITWISESET_PRE_INST_##_lock				\
		"orr	x11, x11, %[opr]"	"\n\t"			\
		_BITWISESET_POST_INST_##_lock				\
		:							\
		: [addr] "r" (&_opl),					\
		  [opr]  "Lr" (_opr)					\
		_BITWISESET_CLOBBERS_##_lock				\
	);								\
})

#define BITWISESET(_lock, _opl, _opr)					\
	_BITWISESET(_lock, _opl, _opr)

#define _BITWISECLR_PRE_INST_FULL_LOCK					\
	"1:"					"\n\t"			\
		"ldxr	xzr, [%[addr]]" 	"\n\t"

#define _BITWISECLR_PRE_INST_LOCK_LESS					\
		"#	NOP"			"\n\t"

#define _BITWISECLR_POST_INST_FULL_LOCK					\
		"stxr	w9, xzr, [%[addr]]"	"\n\t"			\
		"cbnz	w9, 1b" 		"\n\t"			\
		"dmb	ish"

#define _BITWISECLR_POST_INST_LOCK_LESS					\
		"str	xzr, [%[addr]]"

#define _BITWISECLR_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9"

#define _BITWISECLR_CLOBBERS_LOCK_LESS					\
		: "cc", "memory"

#define _BITWISECLR(_lock, _dest)					\
({									\
	__asm__ volatile						\
	(								\
		_BITWISECLR_PRE_INST_##_lock				\
		_BITWISECLR_POST_INST_##_lock				\
		:							\
		: [addr] "r" (&_dest)					\
		_BITWISECLR_CLOBBERS_##_lock				\
	);								\
})

#define BITWISECLR(_lock, _dest)					\
	_BITWISECLR(_lock, _dest)

#if (CORE_COUNT == 64)
#define BITSET_CC(_lock, _base, _offset) BITSET(_lock, _base, _offset)
#else
#define BITSET_CC(_lock, _base, _offset)				\
(									\
	BITSET(_lock,	_base[ CORE_WORD_TOP(CORE_COUNT)		\
				- CORE_WORD_POS(CORE_COUNT, _offset) ] ,\
			CORE_WORD_MOD(CORE_COUNT, _offset) )		\
)
#endif

#if (CORE_COUNT == 64)
#define BITCLR_CC(_lock, _base, _offset) BITCLR(_lock, _base, _offset)
#else
#define BITCLR_CC(_lock, _base, _offset)				\
(									\
	BITCLR(_lock,	_base[ CORE_WORD_TOP(CORE_COUNT)		\
				- CORE_WORD_POS(CORE_COUNT, _offset) ] ,\
			CORE_WORD_MOD(CORE_COUNT, _offset) )		\
)
#endif

#if (CORE_COUNT == 64)
#define BITVAL_CC(_base, _offset) BITVAL(_base, _offset)
#else
#define BITVAL_CC(_base, _offset)					\
(									\
	BITVAL(_base[CORE_WORD_TOP(CORE_COUNT)				\
		- CORE_WORD_POS(CORE_COUNT, _offset)],			\
		CORE_WORD_MOD(CORE_COUNT, _offset) )			\
)
#endif

#if (CORE_COUNT == 64)
#define BITWISEAND_CC(_lock, _opl, _opr) _BITWISEAND(_lock, _opl, _opr)
#else
#define BITWISEAND_CC(_lock, _opl, _opr)				\
({									\
	volatile Bit64 _ret __attribute__ ((aligned (8))) = 0;		\
	unsigned int cw = 0;						\
	do {								\
		_ret |= _BITWISEAND(_lock, _opl[cw], _opr[cw]) ;	\
	} while (++cw <= CORE_WORD_TOP(CORE_COUNT));			\
	_ret;								\
})
#endif

#if (CORE_COUNT == 64)
#define BITSTOR_CC(_lock, _dest, _src) _BITSTOR(_lock, _dest, _src)
#else
#define BITSTOR_CC(_lock, _dest, _src)					\
({									\
	unsigned int cw = 0;						\
	do {								\
		_BITSTOR(_lock, _dest[cw], _src[cw]);			\
	} while (++cw <= CORE_WORD_TOP(CORE_COUNT));			\
})
#endif

#define _BITCMP_PRE_INST_FULL_LOCK					\
	"1:"						"\n\t"		\
		"ldxr	x11,	[%[addr]]"		"\n\t"

#define _BITCMP_PRE_INST_LOCK_LESS					\
		"ldr	x11,	[%[addr]]"		"\n\t"

#define _BITCMP_POST_INST_FULL_LOCK					\
		"stxr	w9,	x11, [%[addr]]" 	"\n\t"		\
		"cbnz	w9,	1b"			"\n\t"		\
		"dmb	ish"

#define _BITCMP_POST_INST_LOCK_LESS					\
		"#	NOP"

#define _BITCMP_CLOBBERS_FULL_LOCK					\
		: "cc", "memory", "%w9", "%w10", "%x11"

#define _BITCMP_CLOBBERS_LOCK_LESS					\
		: "cc", "memory", "%w10", "%x11"

#define _BITCMP(_lock, _opl, _opr)					\
({									\
	volatile unsigned char _ret;					\
									\
	__asm__ volatile						\
	(								\
		_BITCMP_PRE_INST_##_lock				\
		"cmp	x11,	%[opr]" 		"\n\t"		\
		"cset	w10,	eq"			"\n\t"		\
		"strb	w10,	%[ret]" 		"\n\t"		\
		_BITCMP_POST_INST_##_lock				\
		: [ret] "=m"	(_ret)					\
		: [addr] "r"	(&_opl), 				\
		  [opr]	"Lr"	(_opr)					\
		_BITCMP_CLOBBERS_##_lock				\
	);								\
	_ret;								\
})

#define BITZERO(_lock, _src)						\
	_BITCMP(_lock, _src, 0)

#if (CORE_COUNT == 64)
#define BITCMP_CC(_lock, _opl, _opr) _BITCMP(_lock, _opl, _opr)
#else
#define BITCMP_CC(_lock, _opl, _opr)					\
({									\
	unsigned char ret = 1;						\
	unsigned int cw = 0;						\
	do {								\
		ret &= _BITCMP(_lock, _opl[cw], _opr[cw]);		\
	} while (++cw <= CORE_WORD_TOP(CORE_COUNT));			\
	ret;								\
})
#endif

/* Micro-benchmark. Prerequisites: CPU affinity, RDTSC[P] optionnaly RDPMC */
#if defined(UBENCH) && UBENCH == 1

#define UBENCH_DECLARE()						\
static unsigned long long uBenchCounter[2][4] __attribute__((aligned(8)))=\
{									\
/* TSC*/{0,		0,		0,		0},		\
/*INST*/{0,		0,		0,		0}		\
/*	[0]DELTA,	[1]PREV,	[2]LAST,	[3]OVERHEAD  */ \
};									\
									\
inline static void UBENCH_RDCOUNTER_VOID(unsigned int idx) {}		\
									\
inline static void UBENCH_With_RDTSC_No_RDPMC(unsigned int idx) 	\
{									\
	RDTSC64(uBenchCounter[0][idx]) ;	SERIALIZE();		\
}									\
									\
inline static void UBENCH_With_RDTSCP_RDPMC(unsigned int idx)		\
{									\
	RDTSC_PMCx1(	uBenchCounter[0][idx],				\
			cntvct_el0,					\
			uBenchCounter[1][idx]) ;			\
}									\
									\
									\
inline static void UBENCH_With_RDTSC_RDPMC(unsigned int idx)		\
{									\
	RDTSC_PMCx1(	uBenchCounter[0][idx],				\
			cntvct_el0,					\
			uBenchCounter[1][idx]) ;			\
}									\
									\
static void (*UBENCH_RDCOUNTER)(unsigned int) = UBENCH_RDCOUNTER_VOID;

#define UBENCH_COMPUTE()						\
({									\
	uBenchCounter[0][0] = uBenchCounter[0][2] - uBenchCounter[0][1];\
	uBenchCounter[1][0] = uBenchCounter[1][2] - uBenchCounter[1][1];\
	uBenchCounter[0][0] -= uBenchCounter[0][3];			\
	uBenchCounter[1][0] -= uBenchCounter[1][3];			\
})

#define UBENCH_METRIC(metric) (uBenchCounter[metric][0])

#define UBENCH_SETUP(withRDTSCP , withRDPMC)				\
({									\
	void (*MatrixCallFunc[2][2])(unsigned int) = {			\
		{UBENCH_With_RDTSC_No_RDPMC, UBENCH_With_RDTSC_RDPMC},	\
		{UBENCH_With_RDTSC_No_RDPMC, UBENCH_With_RDTSCP_RDPMC}	\
	};								\
	UBENCH_RDCOUNTER = MatrixCallFunc[withRDTSCP][withRDPMC];	\
									\
	UBENCH_RDCOUNTER(0);						\
	UBENCH_RDCOUNTER(3);						\
									\
	uBenchCounter[0][0] = uBenchCounter[0][3]- uBenchCounter[0][0]; \
	uBenchCounter[1][0] = uBenchCounter[1][3]- uBenchCounter[1][0]; \
	uBenchCounter[0][3] = uBenchCounter[0][0];			\
	uBenchCounter[1][3] = uBenchCounter[1][0];			\
})

#else /* UBENCH == 1 */

#define UBENCH_DECLARE()			/* UBENCH_DECLARE()	*/
#define UBENCH_RDCOUNTER(idx)			/* UBENCH_RDCOUNTER()	*/
#define UBENCH_COMPUTE()			/* UBENCH_COMPUTE()	*/
#define UBENCH_METRIC(metric)			/* UBENCH_METRIC()	*/
#define UBENCH_SETUP(withTSCP , withPMC)	/* UBENCH_SETUP()	*/

#endif /* UBENCH == 1 */
