2 * Copyright (C) 2013 Huawei Ltd.
3 * Author: Jiang Liu <liuj97@gmail.com>
5 * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #include <linux/bitops.h>
20 #include <linux/bug.h>
21 #include <linux/compiler.h>
22 #include <linux/kernel.h>
24 #include <linux/smp.h>
25 #include <linux/spinlock.h>
26 #include <linux/stop_machine.h>
27 #include <linux/types.h>
28 #include <linux/uaccess.h>
30 #include <asm/cacheflush.h>
31 #include <asm/debug-monitors.h>
32 #include <asm/fixmap.h>
35 #define AARCH64_INSN_SF_BIT BIT(31)
36 #define AARCH64_INSN_N_BIT BIT(22)
38 static int aarch64_insn_encoding_class[] = {
39 AARCH64_INSN_CLS_UNKNOWN,
40 AARCH64_INSN_CLS_UNKNOWN,
41 AARCH64_INSN_CLS_UNKNOWN,
42 AARCH64_INSN_CLS_UNKNOWN,
43 AARCH64_INSN_CLS_LDST,
44 AARCH64_INSN_CLS_DP_REG,
45 AARCH64_INSN_CLS_LDST,
46 AARCH64_INSN_CLS_DP_FPSIMD,
47 AARCH64_INSN_CLS_DP_IMM,
48 AARCH64_INSN_CLS_DP_IMM,
49 AARCH64_INSN_CLS_BR_SYS,
50 AARCH64_INSN_CLS_BR_SYS,
51 AARCH64_INSN_CLS_LDST,
52 AARCH64_INSN_CLS_DP_REG,
53 AARCH64_INSN_CLS_LDST,
54 AARCH64_INSN_CLS_DP_FPSIMD,
57 enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
59 return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
62 /* NOP is an alias of HINT */
63 bool __kprobes aarch64_insn_is_nop(u32 insn)
65 if (!aarch64_insn_is_hint(insn))
68 switch (insn & 0xFE0) {
69 case AARCH64_INSN_HINT_YIELD:
70 case AARCH64_INSN_HINT_WFE:
71 case AARCH64_INSN_HINT_WFI:
72 case AARCH64_INSN_HINT_SEV:
73 case AARCH64_INSN_HINT_SEVL:
80 bool aarch64_insn_is_branch_imm(u32 insn)
82 return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) ||
83 aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) ||
84 aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
85 aarch64_insn_is_bcond(insn));
88 static DEFINE_RAW_SPINLOCK(patch_lock);
90 static void __kprobes *patch_map(void *addr, int fixmap)
92 unsigned long uintaddr = (uintptr_t) addr;
93 bool module = !core_kernel_text(uintaddr);
96 if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
97 page = vmalloc_to_page(addr);
98 else if (!module && IS_ENABLED(CONFIG_DEBUG_RODATA))
99 page = virt_to_page(addr);
104 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
105 (uintaddr & ~PAGE_MASK));
108 static void __kprobes patch_unmap(int fixmap)
110 clear_fixmap(fixmap);
113 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
116 int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
121 ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE);
123 *insnp = le32_to_cpu(val);
128 static int __kprobes __aarch64_insn_write(void *addr, u32 insn)
131 unsigned long flags = 0;
134 raw_spin_lock_irqsave(&patch_lock, flags);
135 waddr = patch_map(addr, FIX_TEXT_POKE0);
137 ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE);
139 patch_unmap(FIX_TEXT_POKE0);
140 raw_spin_unlock_irqrestore(&patch_lock, flags);
145 int __kprobes aarch64_insn_write(void *addr, u32 insn)
147 insn = cpu_to_le32(insn);
148 return __aarch64_insn_write(addr, insn);
151 static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
153 if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
156 return aarch64_insn_is_b(insn) ||
157 aarch64_insn_is_bl(insn) ||
158 aarch64_insn_is_svc(insn) ||
159 aarch64_insn_is_hvc(insn) ||
160 aarch64_insn_is_smc(insn) ||
161 aarch64_insn_is_brk(insn) ||
162 aarch64_insn_is_nop(insn);
166 * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
167 * Section B2.6.5 "Concurrent modification and execution of instructions":
168 * Concurrent modification and execution of instructions can lead to the
169 * resulting instruction performing any behavior that can be achieved by
170 * executing any sequence of instructions that can be executed from the
171 * same Exception level, except where the instruction before modification
172 * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
173 * or SMC instruction.
175 bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
177 return __aarch64_insn_hotpatch_safe(old_insn) &&
178 __aarch64_insn_hotpatch_safe(new_insn);
181 int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
186 /* A64 instructions must be word aligned */
187 if ((uintptr_t)tp & 0x3)
190 ret = aarch64_insn_write(tp, insn);
192 flush_icache_range((uintptr_t)tp,
193 (uintptr_t)tp + AARCH64_INSN_SIZE);
198 struct aarch64_insn_patch {
205 static int __kprobes aarch64_insn_patch_text_cb(void *arg)
208 struct aarch64_insn_patch *pp = arg;
210 /* The first CPU becomes master */
211 if (atomic_inc_return(&pp->cpu_count) == 1) {
212 for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
213 ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
216 * aarch64_insn_patch_text_nosync() calls flush_icache_range(),
217 * which ends with "dsb; isb" pair guaranteeing global
220 /* Notify other processors with an additional increment. */
221 atomic_inc(&pp->cpu_count);
223 while (atomic_read(&pp->cpu_count) <= num_online_cpus())
231 int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
233 struct aarch64_insn_patch patch = {
237 .cpu_count = ATOMIC_INIT(0),
243 return stop_machine(aarch64_insn_patch_text_cb, &patch,
247 int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
252 /* Unsafe to patch multiple instructions without synchronizaiton */
254 ret = aarch64_insn_read(addrs[0], &insn);
258 if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
260 * ARMv8 architecture doesn't guarantee all CPUs see
261 * the new instruction after returning from function
262 * aarch64_insn_patch_text_nosync(). So send IPIs to
263 * all other CPUs to achieve instruction
266 ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
267 kick_all_cpus_sync();
272 return aarch64_insn_patch_text_sync(addrs, insns, cnt);
275 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
276 u32 *maskp, int *shiftp)
282 case AARCH64_INSN_IMM_26:
286 case AARCH64_INSN_IMM_19:
290 case AARCH64_INSN_IMM_16:
294 case AARCH64_INSN_IMM_14:
298 case AARCH64_INSN_IMM_12:
302 case AARCH64_INSN_IMM_9:
306 case AARCH64_INSN_IMM_7:
310 case AARCH64_INSN_IMM_6:
311 case AARCH64_INSN_IMM_S:
315 case AARCH64_INSN_IMM_R:
329 #define ADR_IMM_HILOSPLIT 2
330 #define ADR_IMM_SIZE SZ_2M
331 #define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1)
332 #define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
333 #define ADR_IMM_LOSHIFT 29
334 #define ADR_IMM_HISHIFT 5
336 u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
338 u32 immlo, immhi, mask;
342 case AARCH64_INSN_IMM_ADR:
344 immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
345 immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
346 insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
347 mask = ADR_IMM_SIZE - 1;
350 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
351 pr_err("aarch64_insn_decode_immediate: unknown immediate encoding %d\n",
357 return (insn >> shift) & mask;
360 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
363 u32 immlo, immhi, mask;
366 if (insn == AARCH64_BREAK_FAULT)
367 return AARCH64_BREAK_FAULT;
370 case AARCH64_INSN_IMM_ADR:
372 immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
373 imm >>= ADR_IMM_HILOSPLIT;
374 immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
376 mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
377 (ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
380 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
381 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
383 return AARCH64_BREAK_FAULT;
387 /* Update the immediate field. */
388 insn &= ~(mask << shift);
389 insn |= (imm & mask) << shift;
394 static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
396 enum aarch64_insn_register reg)
400 if (insn == AARCH64_BREAK_FAULT)
401 return AARCH64_BREAK_FAULT;
403 if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
404 pr_err("%s: unknown register encoding %d\n", __func__, reg);
405 return AARCH64_BREAK_FAULT;
409 case AARCH64_INSN_REGTYPE_RT:
410 case AARCH64_INSN_REGTYPE_RD:
413 case AARCH64_INSN_REGTYPE_RN:
416 case AARCH64_INSN_REGTYPE_RT2:
417 case AARCH64_INSN_REGTYPE_RA:
420 case AARCH64_INSN_REGTYPE_RM:
424 pr_err("%s: unknown register type encoding %d\n", __func__,
426 return AARCH64_BREAK_FAULT;
429 insn &= ~(GENMASK(4, 0) << shift);
430 insn |= reg << shift;
435 static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
441 case AARCH64_INSN_SIZE_8:
444 case AARCH64_INSN_SIZE_16:
447 case AARCH64_INSN_SIZE_32:
450 case AARCH64_INSN_SIZE_64:
454 pr_err("%s: unknown size encoding %d\n", __func__, type);
455 return AARCH64_BREAK_FAULT;
458 insn &= ~GENMASK(31, 30);
464 static inline long branch_imm_common(unsigned long pc, unsigned long addr,
469 if ((pc & 0x3) || (addr & 0x3)) {
470 pr_err("%s: A64 instructions must be word aligned\n", __func__);
474 offset = ((long)addr - (long)pc);
476 if (offset < -range || offset >= range) {
477 pr_err("%s: offset out of range\n", __func__);
484 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
485 enum aarch64_insn_branch_type type)
491 * B/BL support [-128M, 128M) offset
492 * ARM64 virtual address arrangement guarantees all kernel and module
493 * texts are within +/-128M.
495 offset = branch_imm_common(pc, addr, SZ_128M);
496 if (offset >= SZ_128M)
497 return AARCH64_BREAK_FAULT;
500 case AARCH64_INSN_BRANCH_LINK:
501 insn = aarch64_insn_get_bl_value();
503 case AARCH64_INSN_BRANCH_NOLINK:
504 insn = aarch64_insn_get_b_value();
507 pr_err("%s: unknown branch encoding %d\n", __func__, type);
508 return AARCH64_BREAK_FAULT;
511 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
515 u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
516 enum aarch64_insn_register reg,
517 enum aarch64_insn_variant variant,
518 enum aarch64_insn_branch_type type)
523 offset = branch_imm_common(pc, addr, SZ_1M);
525 return AARCH64_BREAK_FAULT;
528 case AARCH64_INSN_BRANCH_COMP_ZERO:
529 insn = aarch64_insn_get_cbz_value();
531 case AARCH64_INSN_BRANCH_COMP_NONZERO:
532 insn = aarch64_insn_get_cbnz_value();
535 pr_err("%s: unknown branch encoding %d\n", __func__, type);
536 return AARCH64_BREAK_FAULT;
540 case AARCH64_INSN_VARIANT_32BIT:
542 case AARCH64_INSN_VARIANT_64BIT:
543 insn |= AARCH64_INSN_SF_BIT;
546 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
547 return AARCH64_BREAK_FAULT;
550 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
552 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
556 u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
557 enum aarch64_insn_condition cond)
562 offset = branch_imm_common(pc, addr, SZ_1M);
564 insn = aarch64_insn_get_bcond_value();
566 if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) {
567 pr_err("%s: unknown condition encoding %d\n", __func__, cond);
568 return AARCH64_BREAK_FAULT;
572 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
576 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
578 return aarch64_insn_get_hint_value() | op;
581 u32 __kprobes aarch64_insn_gen_nop(void)
583 return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
586 u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
587 enum aarch64_insn_branch_type type)
592 case AARCH64_INSN_BRANCH_NOLINK:
593 insn = aarch64_insn_get_br_value();
595 case AARCH64_INSN_BRANCH_LINK:
596 insn = aarch64_insn_get_blr_value();
598 case AARCH64_INSN_BRANCH_RETURN:
599 insn = aarch64_insn_get_ret_value();
602 pr_err("%s: unknown branch encoding %d\n", __func__, type);
603 return AARCH64_BREAK_FAULT;
606 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
609 u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
610 enum aarch64_insn_register base,
611 enum aarch64_insn_register offset,
612 enum aarch64_insn_size_type size,
613 enum aarch64_insn_ldst_type type)
618 case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
619 insn = aarch64_insn_get_ldr_reg_value();
621 case AARCH64_INSN_LDST_STORE_REG_OFFSET:
622 insn = aarch64_insn_get_str_reg_value();
625 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
626 return AARCH64_BREAK_FAULT;
629 insn = aarch64_insn_encode_ldst_size(size, insn);
631 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
633 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
636 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
640 u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
641 enum aarch64_insn_register reg2,
642 enum aarch64_insn_register base,
644 enum aarch64_insn_variant variant,
645 enum aarch64_insn_ldst_type type)
651 case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
652 insn = aarch64_insn_get_ldp_pre_value();
654 case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
655 insn = aarch64_insn_get_stp_pre_value();
657 case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
658 insn = aarch64_insn_get_ldp_post_value();
660 case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
661 insn = aarch64_insn_get_stp_post_value();
664 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
665 return AARCH64_BREAK_FAULT;
669 case AARCH64_INSN_VARIANT_32BIT:
670 if ((offset & 0x3) || (offset < -256) || (offset > 252)) {
671 pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n",
673 return AARCH64_BREAK_FAULT;
677 case AARCH64_INSN_VARIANT_64BIT:
678 if ((offset & 0x7) || (offset < -512) || (offset > 504)) {
679 pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n",
681 return AARCH64_BREAK_FAULT;
684 insn |= AARCH64_INSN_SF_BIT;
687 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
688 return AARCH64_BREAK_FAULT;
691 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
694 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
697 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
700 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
704 u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
705 enum aarch64_insn_register src,
706 int imm, enum aarch64_insn_variant variant,
707 enum aarch64_insn_adsb_type type)
712 case AARCH64_INSN_ADSB_ADD:
713 insn = aarch64_insn_get_add_imm_value();
715 case AARCH64_INSN_ADSB_SUB:
716 insn = aarch64_insn_get_sub_imm_value();
718 case AARCH64_INSN_ADSB_ADD_SETFLAGS:
719 insn = aarch64_insn_get_adds_imm_value();
721 case AARCH64_INSN_ADSB_SUB_SETFLAGS:
722 insn = aarch64_insn_get_subs_imm_value();
725 pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
726 return AARCH64_BREAK_FAULT;
730 case AARCH64_INSN_VARIANT_32BIT:
732 case AARCH64_INSN_VARIANT_64BIT:
733 insn |= AARCH64_INSN_SF_BIT;
736 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
737 return AARCH64_BREAK_FAULT;
740 if (imm & ~(SZ_4K - 1)) {
741 pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
742 return AARCH64_BREAK_FAULT;
745 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
747 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
749 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
752 u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
753 enum aarch64_insn_register src,
755 enum aarch64_insn_variant variant,
756 enum aarch64_insn_bitfield_type type)
762 case AARCH64_INSN_BITFIELD_MOVE:
763 insn = aarch64_insn_get_bfm_value();
765 case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
766 insn = aarch64_insn_get_ubfm_value();
768 case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
769 insn = aarch64_insn_get_sbfm_value();
772 pr_err("%s: unknown bitfield encoding %d\n", __func__, type);
773 return AARCH64_BREAK_FAULT;
777 case AARCH64_INSN_VARIANT_32BIT:
778 mask = GENMASK(4, 0);
780 case AARCH64_INSN_VARIANT_64BIT:
781 insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
782 mask = GENMASK(5, 0);
785 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
786 return AARCH64_BREAK_FAULT;
790 pr_err("%s: invalid immr encoding %d\n", __func__, immr);
791 return AARCH64_BREAK_FAULT;
794 pr_err("%s: invalid imms encoding %d\n", __func__, imms);
795 return AARCH64_BREAK_FAULT;
798 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
800 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
802 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
804 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
807 u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
809 enum aarch64_insn_variant variant,
810 enum aarch64_insn_movewide_type type)
815 case AARCH64_INSN_MOVEWIDE_ZERO:
816 insn = aarch64_insn_get_movz_value();
818 case AARCH64_INSN_MOVEWIDE_KEEP:
819 insn = aarch64_insn_get_movk_value();
821 case AARCH64_INSN_MOVEWIDE_INVERSE:
822 insn = aarch64_insn_get_movn_value();
825 pr_err("%s: unknown movewide encoding %d\n", __func__, type);
826 return AARCH64_BREAK_FAULT;
829 if (imm & ~(SZ_64K - 1)) {
830 pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
831 return AARCH64_BREAK_FAULT;
835 case AARCH64_INSN_VARIANT_32BIT:
836 if (shift != 0 && shift != 16) {
837 pr_err("%s: invalid shift encoding %d\n", __func__,
839 return AARCH64_BREAK_FAULT;
842 case AARCH64_INSN_VARIANT_64BIT:
843 insn |= AARCH64_INSN_SF_BIT;
844 if (shift != 0 && shift != 16 && shift != 32 && shift != 48) {
845 pr_err("%s: invalid shift encoding %d\n", __func__,
847 return AARCH64_BREAK_FAULT;
851 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
852 return AARCH64_BREAK_FAULT;
855 insn |= (shift >> 4) << 21;
857 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
859 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
862 u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
863 enum aarch64_insn_register src,
864 enum aarch64_insn_register reg,
866 enum aarch64_insn_variant variant,
867 enum aarch64_insn_adsb_type type)
872 case AARCH64_INSN_ADSB_ADD:
873 insn = aarch64_insn_get_add_value();
875 case AARCH64_INSN_ADSB_SUB:
876 insn = aarch64_insn_get_sub_value();
878 case AARCH64_INSN_ADSB_ADD_SETFLAGS:
879 insn = aarch64_insn_get_adds_value();
881 case AARCH64_INSN_ADSB_SUB_SETFLAGS:
882 insn = aarch64_insn_get_subs_value();
885 pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
886 return AARCH64_BREAK_FAULT;
890 case AARCH64_INSN_VARIANT_32BIT:
891 if (shift & ~(SZ_32 - 1)) {
892 pr_err("%s: invalid shift encoding %d\n", __func__,
894 return AARCH64_BREAK_FAULT;
897 case AARCH64_INSN_VARIANT_64BIT:
898 insn |= AARCH64_INSN_SF_BIT;
899 if (shift & ~(SZ_64 - 1)) {
900 pr_err("%s: invalid shift encoding %d\n", __func__,
902 return AARCH64_BREAK_FAULT;
906 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
907 return AARCH64_BREAK_FAULT;
911 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
913 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
915 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
917 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
920 u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
921 enum aarch64_insn_register src,
922 enum aarch64_insn_variant variant,
923 enum aarch64_insn_data1_type type)
928 case AARCH64_INSN_DATA1_REVERSE_16:
929 insn = aarch64_insn_get_rev16_value();
931 case AARCH64_INSN_DATA1_REVERSE_32:
932 insn = aarch64_insn_get_rev32_value();
934 case AARCH64_INSN_DATA1_REVERSE_64:
935 if (variant != AARCH64_INSN_VARIANT_64BIT) {
936 pr_err("%s: invalid variant for reverse64 %d\n",
938 return AARCH64_BREAK_FAULT;
940 insn = aarch64_insn_get_rev64_value();
943 pr_err("%s: unknown data1 encoding %d\n", __func__, type);
944 return AARCH64_BREAK_FAULT;
948 case AARCH64_INSN_VARIANT_32BIT:
950 case AARCH64_INSN_VARIANT_64BIT:
951 insn |= AARCH64_INSN_SF_BIT;
954 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
955 return AARCH64_BREAK_FAULT;
958 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
960 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
963 u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
964 enum aarch64_insn_register src,
965 enum aarch64_insn_register reg,
966 enum aarch64_insn_variant variant,
967 enum aarch64_insn_data2_type type)
972 case AARCH64_INSN_DATA2_UDIV:
973 insn = aarch64_insn_get_udiv_value();
975 case AARCH64_INSN_DATA2_SDIV:
976 insn = aarch64_insn_get_sdiv_value();
978 case AARCH64_INSN_DATA2_LSLV:
979 insn = aarch64_insn_get_lslv_value();
981 case AARCH64_INSN_DATA2_LSRV:
982 insn = aarch64_insn_get_lsrv_value();
984 case AARCH64_INSN_DATA2_ASRV:
985 insn = aarch64_insn_get_asrv_value();
987 case AARCH64_INSN_DATA2_RORV:
988 insn = aarch64_insn_get_rorv_value();
991 pr_err("%s: unknown data2 encoding %d\n", __func__, type);
992 return AARCH64_BREAK_FAULT;
996 case AARCH64_INSN_VARIANT_32BIT:
998 case AARCH64_INSN_VARIANT_64BIT:
999 insn |= AARCH64_INSN_SF_BIT;
1002 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1003 return AARCH64_BREAK_FAULT;
1006 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1008 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1010 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1013 u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
1014 enum aarch64_insn_register src,
1015 enum aarch64_insn_register reg1,
1016 enum aarch64_insn_register reg2,
1017 enum aarch64_insn_variant variant,
1018 enum aarch64_insn_data3_type type)
1023 case AARCH64_INSN_DATA3_MADD:
1024 insn = aarch64_insn_get_madd_value();
1026 case AARCH64_INSN_DATA3_MSUB:
1027 insn = aarch64_insn_get_msub_value();
1030 pr_err("%s: unknown data3 encoding %d\n", __func__, type);
1031 return AARCH64_BREAK_FAULT;
1035 case AARCH64_INSN_VARIANT_32BIT:
1037 case AARCH64_INSN_VARIANT_64BIT:
1038 insn |= AARCH64_INSN_SF_BIT;
1041 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1042 return AARCH64_BREAK_FAULT;
1045 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1047 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
1049 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
1052 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
1056 u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
1057 enum aarch64_insn_register src,
1058 enum aarch64_insn_register reg,
1060 enum aarch64_insn_variant variant,
1061 enum aarch64_insn_logic_type type)
1066 case AARCH64_INSN_LOGIC_AND:
1067 insn = aarch64_insn_get_and_value();
1069 case AARCH64_INSN_LOGIC_BIC:
1070 insn = aarch64_insn_get_bic_value();
1072 case AARCH64_INSN_LOGIC_ORR:
1073 insn = aarch64_insn_get_orr_value();
1075 case AARCH64_INSN_LOGIC_ORN:
1076 insn = aarch64_insn_get_orn_value();
1078 case AARCH64_INSN_LOGIC_EOR:
1079 insn = aarch64_insn_get_eor_value();
1081 case AARCH64_INSN_LOGIC_EON:
1082 insn = aarch64_insn_get_eon_value();
1084 case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1085 insn = aarch64_insn_get_ands_value();
1087 case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
1088 insn = aarch64_insn_get_bics_value();
1091 pr_err("%s: unknown logical encoding %d\n", __func__, type);
1092 return AARCH64_BREAK_FAULT;
1096 case AARCH64_INSN_VARIANT_32BIT:
1097 if (shift & ~(SZ_32 - 1)) {
1098 pr_err("%s: invalid shift encoding %d\n", __func__,
1100 return AARCH64_BREAK_FAULT;
1103 case AARCH64_INSN_VARIANT_64BIT:
1104 insn |= AARCH64_INSN_SF_BIT;
1105 if (shift & ~(SZ_64 - 1)) {
1106 pr_err("%s: invalid shift encoding %d\n", __func__,
1108 return AARCH64_BREAK_FAULT;
1112 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1113 return AARCH64_BREAK_FAULT;
1117 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1119 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1121 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1123 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1127 * Decode the imm field of a branch, and return the byte offset as a
1128 * signed value (so it can be used when computing a new branch
1131 s32 aarch64_get_branch_offset(u32 insn)
1135 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
1136 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
1137 return (imm << 6) >> 4;
1140 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1141 aarch64_insn_is_bcond(insn)) {
1142 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn);
1143 return (imm << 13) >> 11;
1146 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) {
1147 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn);
1148 return (imm << 18) >> 16;
1151 /* Unhandled instruction */
1156 * Encode the displacement of a branch in the imm field and return the
1157 * updated instruction.
1159 u32 aarch64_set_branch_offset(u32 insn, s32 offset)
1161 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn))
1162 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
1165 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1166 aarch64_insn_is_bcond(insn))
1167 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
1170 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn))
1171 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn,
1174 /* Unhandled instruction */
1178 bool aarch32_insn_is_wide(u32 insn)
1180 return insn >= 0xe800;
1184 * Macros/defines for extracting register numbers from instruction.
1186 u32 aarch32_insn_extract_reg_num(u32 insn, int offset)
1188 return (insn & (0xf << offset)) >> offset;
1191 #define OPC2_MASK 0x7
1192 #define OPC2_OFFSET 5
1193 u32 aarch32_insn_mcr_extract_opc2(u32 insn)
1195 return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET;
1198 #define CRM_MASK 0xf
1199 u32 aarch32_insn_mcr_extract_crm(u32 insn)
1201 return insn & CRM_MASK;