Skip to content

Commit 26e30f3

Browse files
Hou TaoNobody
Hou Tao
authored and
Nobody
committed
arm64: insn: add encoders for atomic operations
It is a preparation patch for eBPF atomic supports under arm64. eBPF needs support atomic[64]_fetch_add, atomic[64]_[fetch_]{and,or,xor} and atomic[64]_{xchg|cmpxchg}. The ordering semantics of eBPF atomics are the same with the implementations in linux kernel. Add three helpers to support LDCLR/LDEOR/LDSET/SWP, CAS and DMB instructions. STADD/STCLR/STEOR/STSET are simply encoded as aliases for LDADD/LDCLR/LDEOR/LDSET with XZR as the destination register, so no extra helper is added. atomic_fetch_add() and other atomic ops needs support for STLXR instruction, so extend enum aarch64_insn_ldst_type to do that. LDADD/LDEOR/LDSET/SWP and CAS instructions are only available when LSE atomics is enabled, so just return AARCH64_BREAK_FAULT directly in these newly-added helpers if CONFIG_ARM64_LSE_ATOMICS is disabled. Signed-off-by: Hou Tao <[email protected]>
1 parent 8b53ae5 commit 26e30f3

File tree

3 files changed

+253
-23
lines changed

3 files changed

+253
-23
lines changed

‎arch/arm64/include/asm/insn.h‎

Lines changed: 73 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,9 @@ enum aarch64_insn_ldst_type {
205205
AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX,
206206
AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX,
207207
AARCH64_INSN_LDST_LOAD_EX,
208+
AARCH64_INSN_LDST_LOAD_ACQ_EX,
208209
AARCH64_INSN_LDST_STORE_EX,
210+
AARCH64_INSN_LDST_STORE_REL_EX,
209211
};
210212

211213
enum aarch64_insn_adsb_type {
@@ -280,6 +282,36 @@ enum aarch64_insn_adr_type {
280282
AARCH64_INSN_ADR_TYPE_ADR,
281283
};
282284

285+
enum aarch64_insn_mem_atomic_op {
286+
AARCH64_INSN_MEM_ATOMIC_ADD,
287+
AARCH64_INSN_MEM_ATOMIC_CLR,
288+
AARCH64_INSN_MEM_ATOMIC_EOR,
289+
AARCH64_INSN_MEM_ATOMIC_SET,
290+
AARCH64_INSN_MEM_ATOMIC_SWP,
291+
};
292+
293+
enum aarch64_insn_mem_order_type {
294+
AARCH64_INSN_MEM_ORDER_NONE,
295+
AARCH64_INSN_MEM_ORDER_ACQ,
296+
AARCH64_INSN_MEM_ORDER_REL,
297+
AARCH64_INSN_MEM_ORDER_ACQREL,
298+
};
299+
300+
enum aarch64_insn_mb_type {
301+
AARCH64_INSN_MB_SY,
302+
AARCH64_INSN_MB_ST,
303+
AARCH64_INSN_MB_LD,
304+
AARCH64_INSN_MB_ISH,
305+
AARCH64_INSN_MB_ISHST,
306+
AARCH64_INSN_MB_ISHLD,
307+
AARCH64_INSN_MB_NSH,
308+
AARCH64_INSN_MB_NSHST,
309+
AARCH64_INSN_MB_NSHLD,
310+
AARCH64_INSN_MB_OSH,
311+
AARCH64_INSN_MB_OSHST,
312+
AARCH64_INSN_MB_OSHLD,
313+
};
314+
283315
#define __AARCH64_INSN_FUNCS(abbr, mask, val) \
284316
static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
285317
{ \
@@ -303,6 +335,11 @@ __AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400)
303335
__AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400)
304336
__AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800)
305337
__AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000)
338+
__AARCH64_INSN_FUNCS(ldclr, 0x3F20FC00, 0x38201000)
339+
__AARCH64_INSN_FUNCS(ldeor, 0x3F20FC00, 0x38202000)
340+
__AARCH64_INSN_FUNCS(ldset, 0x3F20FC00, 0x38203000)
341+
__AARCH64_INSN_FUNCS(swp, 0x3F20FC00, 0x38208000)
342+
__AARCH64_INSN_FUNCS(cas, 0x3FA07C00, 0x08A07C00)
306343
__AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800)
307344
__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000)
308345
__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000)
@@ -474,13 +511,6 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
474511
enum aarch64_insn_register state,
475512
enum aarch64_insn_size_type size,
476513
enum aarch64_insn_ldst_type type);
477-
u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
478-
enum aarch64_insn_register address,
479-
enum aarch64_insn_register value,
480-
enum aarch64_insn_size_type size);
481-
u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address,
482-
enum aarch64_insn_register value,
483-
enum aarch64_insn_size_type size);
484514
u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
485515
enum aarch64_insn_register src,
486516
int imm, enum aarch64_insn_variant variant,
@@ -541,6 +571,42 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
541571
enum aarch64_insn_prfm_type type,
542572
enum aarch64_insn_prfm_target target,
543573
enum aarch64_insn_prfm_policy policy);
574+
#ifdef CONFIG_ARM64_LSE_ATOMICS
575+
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
576+
enum aarch64_insn_register address,
577+
enum aarch64_insn_register value,
578+
enum aarch64_insn_size_type size,
579+
enum aarch64_insn_mem_atomic_op op,
580+
enum aarch64_insn_mem_order_type order);
581+
u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
582+
enum aarch64_insn_register address,
583+
enum aarch64_insn_register value,
584+
enum aarch64_insn_size_type size,
585+
enum aarch64_insn_mem_order_type order);
586+
#else
587+
static inline
588+
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
589+
enum aarch64_insn_register address,
590+
enum aarch64_insn_register value,
591+
enum aarch64_insn_size_type size,
592+
enum aarch64_insn_mem_atomic_op op,
593+
enum aarch64_insn_mem_order_type order)
594+
{
595+
return AARCH64_BREAK_FAULT;
596+
}
597+
598+
static inline
599+
u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
600+
enum aarch64_insn_register address,
601+
enum aarch64_insn_register value,
602+
enum aarch64_insn_size_type size,
603+
enum aarch64_insn_mem_order_type order)
604+
{
605+
return AARCH64_BREAK_FAULT;
606+
}
607+
#endif
608+
u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type);
609+
544610
s32 aarch64_get_branch_offset(u32 insn);
545611
u32 aarch64_set_branch_offset(u32 insn, s32 offset);
546612

‎arch/arm64/lib/insn.c‎

Lines changed: 171 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -578,10 +578,16 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
578578

579579
switch (type) {
580580
case AARCH64_INSN_LDST_LOAD_EX:
581+
case AARCH64_INSN_LDST_LOAD_ACQ_EX:
581582
insn = aarch64_insn_get_load_ex_value();
583+
if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX)
584+
insn |= BIT(15);
582585
break;
583586
case AARCH64_INSN_LDST_STORE_EX:
587+
case AARCH64_INSN_LDST_STORE_REL_EX:
584588
insn = aarch64_insn_get_store_ex_value();
589+
if (type == AARCH64_INSN_LDST_STORE_REL_EX)
590+
insn |= BIT(15);
585591
break;
586592
default:
587593
pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
@@ -603,12 +609,65 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
603609
state);
604610
}
605611

606-
u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
607-
enum aarch64_insn_register address,
608-
enum aarch64_insn_register value,
609-
enum aarch64_insn_size_type size)
612+
#ifdef CONFIG_ARM64_LSE_ATOMICS
613+
static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,
614+
u32 insn)
610615
{
611-
u32 insn = aarch64_insn_get_ldadd_value();
616+
u32 order;
617+
618+
switch (type) {
619+
case AARCH64_INSN_MEM_ORDER_NONE:
620+
order = 0;
621+
break;
622+
case AARCH64_INSN_MEM_ORDER_ACQ:
623+
order = 2;
624+
break;
625+
case AARCH64_INSN_MEM_ORDER_REL:
626+
order = 1;
627+
break;
628+
case AARCH64_INSN_MEM_ORDER_ACQREL:
629+
order = 3;
630+
break;
631+
default:
632+
pr_err("%s: unknown mem order %d\n", __func__, type);
633+
return AARCH64_BREAK_FAULT;
634+
}
635+
636+
insn &= ~GENMASK(23, 22);
637+
insn |= order << 22;
638+
639+
return insn;
640+
}
641+
642+
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
643+
enum aarch64_insn_register address,
644+
enum aarch64_insn_register value,
645+
enum aarch64_insn_size_type size,
646+
enum aarch64_insn_mem_atomic_op op,
647+
enum aarch64_insn_mem_order_type order)
648+
{
649+
u32 insn;
650+
651+
switch (op) {
652+
case AARCH64_INSN_MEM_ATOMIC_ADD:
653+
insn = aarch64_insn_get_ldadd_value();
654+
break;
655+
case AARCH64_INSN_MEM_ATOMIC_CLR:
656+
insn = aarch64_insn_get_ldclr_value();
657+
break;
658+
case AARCH64_INSN_MEM_ATOMIC_EOR:
659+
insn = aarch64_insn_get_ldeor_value();
660+
break;
661+
case AARCH64_INSN_MEM_ATOMIC_SET:
662+
insn = aarch64_insn_get_ldset_value();
663+
break;
664+
case AARCH64_INSN_MEM_ATOMIC_SWP:
665+
insn = aarch64_insn_get_swp_value();
666+
break;
667+
default:
668+
pr_err("%s: unimplemented mem atomic op %d\n", __func__, op);
669+
return AARCH64_BREAK_FAULT;
670+
}
612671

613672
switch (size) {
614673
case AARCH64_INSN_SIZE_32:
@@ -621,6 +680,8 @@ u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
621680

622681
insn = aarch64_insn_encode_ldst_size(size, insn);
623682

683+
insn = aarch64_insn_encode_ldst_order(order, insn);
684+
624685
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
625686
result);
626687

@@ -631,17 +692,68 @@ u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
631692
value);
632693
}
633694

634-
u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address,
635-
enum aarch64_insn_register value,
636-
enum aarch64_insn_size_type size)
695+
static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,
696+
u32 insn)
637697
{
638-
/*
639-
* STADD is simply encoded as an alias for LDADD with XZR as
640-
* the destination register.
641-
*/
642-
return aarch64_insn_gen_ldadd(AARCH64_INSN_REG_ZR, address,
643-
value, size);
698+
u32 order;
699+
700+
switch (type) {
701+
case AARCH64_INSN_MEM_ORDER_NONE:
702+
order = 0;
703+
break;
704+
case AARCH64_INSN_MEM_ORDER_ACQ:
705+
order = BIT(22);
706+
break;
707+
case AARCH64_INSN_MEM_ORDER_REL:
708+
order = BIT(15);
709+
break;
710+
case AARCH64_INSN_MEM_ORDER_ACQREL:
711+
order = BIT(15) | BIT(22);
712+
break;
713+
default:
714+
pr_err("%s: unknown mem order %d\n", __func__, type);
715+
return AARCH64_BREAK_FAULT;
716+
}
717+
718+
insn &= ~(BIT(15) | BIT(22));
719+
insn |= order;
720+
721+
return insn;
722+
}
723+
724+
u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
725+
enum aarch64_insn_register address,
726+
enum aarch64_insn_register value,
727+
enum aarch64_insn_size_type size,
728+
enum aarch64_insn_mem_order_type order)
729+
{
730+
u32 insn;
731+
732+
switch (size) {
733+
case AARCH64_INSN_SIZE_32:
734+
case AARCH64_INSN_SIZE_64:
735+
break;
736+
default:
737+
pr_err("%s: unimplemented size encoding %d\n", __func__, size);
738+
return AARCH64_BREAK_FAULT;
739+
}
740+
741+
insn = aarch64_insn_get_cas_value();
742+
743+
insn = aarch64_insn_encode_ldst_size(size, insn);
744+
745+
insn = aarch64_insn_encode_cas_order(order, insn);
746+
747+
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
748+
result);
749+
750+
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
751+
address);
752+
753+
return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
754+
value);
644755
}
756+
#endif
645757

646758
static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
647759
enum aarch64_insn_prfm_target target,
@@ -1456,3 +1568,48 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
14561568
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
14571569
return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
14581570
}
1571+
1572+
u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
1573+
{
1574+
u32 opt;
1575+
u32 insn;
1576+
1577+
switch (type) {
1578+
case AARCH64_INSN_MB_SY:
1579+
opt = 0xf;
1580+
break;
1581+
case AARCH64_INSN_MB_ST:
1582+
opt = 0xe;
1583+
break;
1584+
case AARCH64_INSN_MB_LD:
1585+
opt = 0xd;
1586+
break;
1587+
case AARCH64_INSN_MB_ISH:
1588+
opt = 0xb;
1589+
break;
1590+
case AARCH64_INSN_MB_ISHST:
1591+
opt = 0xa;
1592+
break;
1593+
case AARCH64_INSN_MB_ISHLD:
1594+
opt = 0x9;
1595+
break;
1596+
case AARCH64_INSN_MB_NSH:
1597+
opt = 0x7;
1598+
break;
1599+
case AARCH64_INSN_MB_NSHST:
1600+
opt = 0x6;
1601+
break;
1602+
case AARCH64_INSN_MB_NSHLD:
1603+
opt = 0x5;
1604+
break;
1605+
default:
1606+
pr_err("%s: unknown dmb type %d\n", __func__, type);
1607+
return AARCH64_BREAK_FAULT;
1608+
}
1609+
1610+
insn = aarch64_insn_get_dmb_value();
1611+
insn &= ~GENMASK(11, 8);
1612+
insn |= (opt << 8);
1613+
1614+
return insn;
1615+
}

‎arch/arm64/net/bpf_jit.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,16 @@
8989
#define A64_STXR(sf, Rt, Rn, Rs) \
9090
A64_LSX(sf, Rt, Rn, Rs, STORE_EX)
9191

92-
/* LSE atomics */
92+
/*
93+
* LSE atomics
94+
*
95+
* STADD is simply encoded as an alias for LDADD with XZR as
96+
* the destination register.
97+
*/
9398
#define A64_STADD(sf, Rn, Rs) \
94-
aarch64_insn_gen_stadd(Rn, Rs, A64_SIZE(sf))
99+
aarch64_insn_gen_atomic_ld_op(A64_ZR, Rn, Rs, \
100+
A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_ADD, \
101+
AARCH64_INSN_MEM_ORDER_NONE)
95102

96103
/* Add/subtract (immediate) */
97104
#define A64_ADDSUB_IMM(sf, Rd, Rn, imm12, type) \

0 commit comments

Comments
 (0)