/* * libbinjit: a just-in-time recompiler for machine code * Copyright (c) 2016 Andrew Church * * This software may be copied and redistributed under certain conditions; * see the file "COPYING" in the source code distribution for details. * NO WARRANTY is provided with this software. */ #include "src/common.h" #include "src/rtl.h" #include "src/rtl-internal.h" /* * This source file defines functions for encoding RTL instructions into * the RTLInsn structure based on the type of instruction (2-operand ALU, * memory load, and so on). These functions are not exported directly, but * are instead called through a lookup table by rtl_insn_make(), an inline * function defined in rtl-internal.h. * * Both the inlining of rtl_insn_make() and the requirement of preloading * the opcode into insn->opcode (and then taking the RTLInsn pointer in * place of the opcode parameter to rtl_add_insn()) are to help * optimization of the fast path; for example, on MIPS architectures, * rtl_add_insn() can directly call the encoding function in this file * without having to pass through a call to rtl_insn_make() or reload the * parameter registers ($a0-$t1). */ /*************************************************************************/ /************************** Local declarations ***************************/ /*************************************************************************/ /** * OPERAND_ASSERT: Validate operand constraints, returning false from the * current function on constraint violation. */ #define OPERAND_ASSERT(constraint) do { \ if (UNLIKELY(!(constraint))) { \ log_ice(unit->handle, "Operand constraint violated: %s", #constraint);\ return false; \ } \ } while (0) /*-----------------------------------------------------------------------*/ /** * rtl_mark_live: Mark the given register live, also updating the birth * and death fields as appropriate. * * [Parameters] * unit: RTLUnit into which instruction is being inserted. * insn_index: Index of instruction in unit->insns[]. * reg: Pointer to RTLRegister structure for register. * index: Register index. */ static ALWAYS_INLINE void rtl_mark_live( RTLUnit * const unit, const int insn_index, RTLRegister * const reg, const int index) { if (!reg->live) { reg->live = true; reg->birth = insn_index; if (!unit->first_live_reg) { unit->first_live_reg = index; } unit->last_live_reg = index; } reg->death = insn_index; unit->blocks[unit->cur_block].max_live_reg = max(unit->blocks[unit->cur_block].max_live_reg, index); } /*************************************************************************/ /*************************** Encoding routines ***************************/ /*************************************************************************/ /** * make_0op: Encode an instruction which takes no operands. */ static bool make_0op(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { return true; // Nothing to encode. } /*-----------------------------------------------------------------------*/ /** * make_nop: Encode a NOP instruction. */ static bool make_nop(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS if (dest != 0) { OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); } if (src1 != 0) { OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); } if (src2 != 0) { OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); } OPERAND_ASSERT(!(src1 == 0 && src2 != 0)); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->src_imm = other; const int insn_index = unit->num_insns; if (dest) { RTLRegister * const destreg = &unit->regs[dest]; unit->regs[dest].source = RTLREG_RESULT_NOFOLD; unit->regs[dest].result.opcode = RTLOP_NOP; rtl_mark_live(unit, insn_index, destreg, dest); } if (src1) { RTLRegister * const src1reg = &unit->regs[src1]; rtl_mark_live(unit, insn_index, src1reg, src1); } if (src2) { RTLRegister * const src2reg = &unit->regs[src2]; rtl_mark_live(unit, insn_index, src2reg, src2); } return true; } /*-----------------------------------------------------------------------*/ /** * make_set_alias: Encode a SET_ALIAS instruction. */ static bool make_set_alias(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(other < unit->next_alias); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == unit->aliases[other].type); #endif insn->src1 = src1; insn->alias = (uint16_t)other; RTLRegister * const src1reg = &unit->regs[src1]; RTLAlias * const alias = &unit->aliases[other]; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, src1reg, src1); if (alias->base) { rtl_mark_live(unit, insn_index, &unit->regs[alias->base], alias->base); } return true; } /*-----------------------------------------------------------------------*/ /** * make_get_alias: Encode a GET_ALIAS instruction. */ static bool make_get_alias(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(other < unit->next_alias); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == unit->aliases[other].type); #endif insn->dest = dest; insn->alias = (uint16_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLAlias * const alias = &unit->aliases[other]; const int insn_index = unit->num_insns; destreg->source = RTLREG_ALIAS; destreg->alias.src = (uint16_t)other; rtl_mark_live(unit, insn_index, destreg, dest); if (alias->base) { rtl_mark_live(unit, insn_index, &unit->regs[alias->base], alias->base); } return true; } /*-----------------------------------------------------------------------*/ /** * make_move: Encode a MOVE instruction. */ static bool make_move(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_select: Encode a SELECT instruction. */ static bool make_select(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); ASSERT(other < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[other].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[dest].type); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[other])); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->src3 = (uint16_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; RTLRegister * const src3reg = &unit->regs[other]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.src3 = (uint16_t)other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); rtl_mark_live(unit, insn_index, src3reg, other); return true; } /*-----------------------------------------------------------------------*/ /** * make_intcast: Encode an integer size-cast instruction. */ static bool make_intcast(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_alu_1op: Encode a 1-operand ALU-type instruction. */ static bool make_alu_1op(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_alu_2op: Encode a 2-operand ALU-type instruction. */ static bool make_alu_2op(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_shift: Encode a shift/rotate instruction. */ static bool make_shift(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src2])); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_alu_imm: Encode a register-immediate ALU-type instruction. */ static bool make_alu_imm(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_INT32 || other + UINT64_C(0x80000000) < UINT64_C(0x100000000)); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; insn->src_imm = (int32_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 1; destreg->result.src1 = src1; destreg->result.src_imm = (int32_t)other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_clz: Encode a CLZ instruction. */ static bool make_clz(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_cmp: Encode a comparison instruction. */ static bool make_cmp(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[src1].type); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_cmp_imm: Encode a register-immediate comparison instruction. */ static bool make_cmp_imm(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_INT32 || other + UINT64_C(0x80000000) < UINT64_C(0x100000000)); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); #endif insn->dest = dest; insn->src1 = src1; insn->src_imm = (int32_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 1; destreg->result.src1 = src1; destreg->result.src_imm = (int32_t)other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_bitfield: Encode a bitfield instruction. */ static bool make_bitfield(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); const int start = other & 0xFF; const int count = (other >> 8) & 0xFF; #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); if (insn->opcode == RTLOP_BFINS) { OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[dest].type); } #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->bitfield.start = start; insn->bitfield.count = count; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.start = start; destreg->result.count = count; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); if (src2) { rtl_mark_live(unit, insn_index, src2reg, src2); } return true; } /*-----------------------------------------------------------------------*/ /** * make_bitcast: Encode a BITCAST instruction. */ static bool make_bitcast(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT((unit->regs[dest].type == RTLTYPE_INT32 && unit->regs[src1].type == RTLTYPE_FLOAT32) || (unit->regs[dest].type == RTLTYPE_INT64 && unit->regs[src1].type == RTLTYPE_FLOAT64) || (unit->regs[dest].type == RTLTYPE_FLOAT32 && unit->regs[src1].type == RTLTYPE_INT32) || (unit->regs[dest].type == RTLTYPE_FLOAT64 && unit->regs[src1].type == RTLTYPE_INT64)); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_fcvt: Encode an FCVT instruction. */ static bool make_fcvt(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[src1])); OPERAND_ASSERT(unit->regs[src1].type != unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_ficast: Encode an integer-to-float cast instruction (FZCAST or * FSCAST). */ static bool make_ficast(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_froundi: Encode a float-to-integer rounding instruction (FROUNDI * or FTRUNCI). */ static bool make_froundi(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[src1])); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_fpu_1op: Encode a 1-operand floating-point instruction. */ static bool make_fpu_1op(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); /* We currently don't have integer vector types, so we don't need to * explicitly check that vectors have floating-point elements. */ OPERAND_ASSERT(rtl_register_is_float(&unit->regs[dest]) || rtl_register_is_vector(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_fpu_2op: Encode a 2-operand floating-point instruction. */ static bool make_fpu_2op(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[dest]) || rtl_register_is_vector(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_fpu_3op: Encode a 3-operand floating-point instruction. */ static bool make_fpu_3op(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); ASSERT(other < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[other].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[dest]) || rtl_register_is_vector(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[other].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->src3 = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; RTLRegister * const src3reg = &unit->regs[other]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.src3 = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); rtl_mark_live(unit, insn_index, src3reg, other); return true; } /*-----------------------------------------------------------------------*/ /** * make_fcmp: Encode an FCMP instruction. */ static bool make_fcmp(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(rtl_register_is_float(&unit->regs[src1])); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[src1].type); OPERAND_ASSERT(other <= 31); OPERAND_ASSERT((other & 7) <= RTLFCMP_UN); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->fcmp = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.fcmp = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_fgetstate: Encode an FGETSTATE instruction. */ static bool make_fgetstate(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_FPSTATE); #endif insn->dest = dest; RTLRegister * const destreg = &unit->regs[dest]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; rtl_mark_live(unit, insn_index, destreg, dest); return true; } /*-----------------------------------------------------------------------*/ /** * make_fsetstate: Encode an FSETSTATE instruction. */ static bool make_fsetstate(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_FPSTATE); #endif insn->src1 = src1; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_ftestexc: Encode an FTESTEXC instruction. */ static bool make_ftestexc(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_FPSTATE); OPERAND_ASSERT(other <= RTLFEXC_ZERO_DIVIDE); #endif insn->dest = dest; insn->src1 = src1; insn->src_imm = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 1; destreg->result.src1 = src1; destreg->result.src_imm = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_fclearexc: Encode an FCLEAREXC instruction. */ static bool make_fclearexc(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_FPSTATE); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_FPSTATE); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_fsetround: Encode an FSETROUND instruction. */ static bool make_fsetround(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_FPSTATE); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_FPSTATE); OPERAND_ASSERT(other <= RTLFROUND_CEIL); #endif insn->dest = dest; insn->src1 = src1; insn->src_imm = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 1; destreg->result.src1 = src1; destreg->result.src_imm = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_fcopyround: Encode an FCOPYROUND instruction. */ static bool make_fcopyround(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_FPSTATE); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_FPSTATE); OPERAND_ASSERT(unit->regs[src2].type == RTLTYPE_FPSTATE); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_vbuild2: Encode a VBUILD2 instruction. */ static bool make_vbuild2(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[dest])); /* Currently we only have 2-element vector types, so this test will * always be true. */ ASSERT(rtl_vector_length(unit->regs[dest].type) == 2); OPERAND_ASSERT(unit->regs[src1].type == rtl_vector_element_type(unit->regs[dest].type)); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[src1].type); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_vbroadcast: Encode a VBROADCAST instruction. */ static bool make_vbroadcast(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == rtl_vector_element_type(unit->regs[dest].type)); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_vextract: Encode a VEXTRACT instruction. */ static bool make_vextract(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[src1])); OPERAND_ASSERT(unit->regs[dest].type == rtl_vector_element_type(unit->regs[src1].type)); OPERAND_ASSERT(other < (unsigned int)rtl_vector_length(unit->regs[src1].type)); #endif insn->dest = dest; insn->src1 = src1; insn->elem = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.elem = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_vinsert: Encode a VINSERT instruction. */ static bool make_vinsert(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[src2].type == rtl_vector_element_type(unit->regs[dest].type)); OPERAND_ASSERT(other < (unsigned int)rtl_vector_length(unit->regs[dest].type)); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->elem = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.elem = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_vfcvt: Encode a VFCVT instruction. */ static bool make_vfcvt(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[dest])); /* Currently we don't have any integer vector types, so this test will * always be true. */ ASSERT(rtl_type_is_float(rtl_vector_element_type(unit->regs[dest].type))); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[src1])); ASSERT(rtl_type_is_float(rtl_vector_element_type(unit->regs[src1].type))); OPERAND_ASSERT(unit->regs[src1].type != unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_vfcmp: Encode a VFCMP instruction. */ static bool make_vfcmp(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_INT64); OPERAND_ASSERT(rtl_register_is_vector(&unit->regs[src1])); /* Currently we don't have any integer vector types, so this test will * always be true. */ ASSERT(rtl_type_is_float(rtl_vector_element_type(unit->regs[src1].type))); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[src1].type); OPERAND_ASSERT(other <= 31); OPERAND_ASSERT((other & 7) <= RTLFCMP_UN); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->fcmp = other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.fcmp = other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_load_imm: Encode a LOAD_IMM instruction. */ static bool make_load_imm(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_scalar(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_INT64 || unit->regs[dest].type == RTLTYPE_ADDRESS || unit->regs[dest].type == RTLTYPE_FLOAT64 || (unit->regs[dest].type == RTLTYPE_INT32 && other >= UINT64_C(-0x80000000)) || other <= UINT64_C(0xFFFFFFFF)); #endif insn->dest = dest; if (unit->regs[dest].type == RTLTYPE_INT32 || unit->regs[dest].type == RTLTYPE_FLOAT32) { insn->src_imm = (uint32_t)other; } else { insn->src_imm = other; } RTLRegister * const destreg = &unit->regs[dest]; const int insn_index = unit->num_insns; destreg->source = RTLREG_CONSTANT; if (unit->regs[dest].type == RTLTYPE_INT32 || unit->regs[dest].type == RTLTYPE_FLOAT32) { destreg->value.i64 = (uint32_t)other; } else { destreg->value.i64 = other; } rtl_mark_live(unit, insn_index, destreg, dest); return true; } /*-----------------------------------------------------------------------*/ /** * make_load_arg: Encode a LOAD_ARG instruction. */ static bool make_load_arg(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); #endif insn->dest = dest; insn->arg_index = (uint8_t)other; RTLRegister * const destreg = &unit->regs[dest]; const int insn_index = unit->num_insns; destreg->source = RTLREG_FUNC_ARG; destreg->arg_index = (uint8_t)other; rtl_mark_live(unit, insn_index, destreg, dest); return true; } /*-----------------------------------------------------------------------*/ /** * make_load: Encode a memory load instruction. */ static bool make_load(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(other <= 0x7FFF || other >= UINT64_C(-0x8000)); #endif insn->dest = dest; insn->src1 = src1; insn->offset = (int16_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_MEMORY; destreg->memory.base = src1; destreg->memory.offset = (int16_t)other; destreg->memory.byterev = (insn->opcode == RTLOP_LOAD_BR); rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_load_narrow: Encode a memory load instruction for a narrow integer. */ static bool make_load_narrow(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); /* Lookup tables for each instruction. */ static const struct { uint8_t size; // destreg->memory.size for this insn bool is_signed; // destreg->memory.is_signed for this insn } insn_info[] = { [RTLOP_LOAD_U8 - RTLOP_LOAD_U8] = {.size = 1, .is_signed = false}, [RTLOP_LOAD_S8 - RTLOP_LOAD_U8] = {.size = 1, .is_signed = true}, [RTLOP_LOAD_U16 - RTLOP_LOAD_U8] = {.size = 2, .is_signed = false}, [RTLOP_LOAD_S16 - RTLOP_LOAD_U8] = {.size = 2, .is_signed = true}, [RTLOP_LOAD_U16_BR - RTLOP_LOAD_U8] = {.size = 2, .is_signed = false}, [RTLOP_LOAD_S16_BR - RTLOP_LOAD_U8] = {.size = 2, .is_signed = true}, }; const int lookup_index = insn->opcode - RTLOP_LOAD_U8; ASSERT(lookup_index >= 0 && lookup_index < lenof(insn_info)); ASSERT(insn_info[lookup_index].size > 0); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[dest].type == RTLTYPE_INT32); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(other <= 0x7FFF || other >= UINT64_C(-0x8000)); #endif insn->dest = dest; insn->src1 = src1; insn->offset = (int16_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_MEMORY; destreg->memory.base = src1; destreg->memory.offset = (int16_t)other; destreg->memory.byterev = (insn->opcode >= RTLOP_LOAD_U16_BR); destreg->memory.size = insn_info[lookup_index].size; destreg->memory.is_signed = insn_info[lookup_index].is_signed; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_store: Encode a memory store instruction. */ static bool make_store(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(other <= 0x7FFF || other >= UINT64_C(-0x8000)); #endif insn->src1 = src1; insn->src2 = src2; insn->offset = (int16_t)other; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_store_narrow: Encode a memory store instruction. */ static bool make_store_narrow(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].type == RTLTYPE_INT32); OPERAND_ASSERT(other <= 0x7FFF || other >= UINT64_C(-0x8000)); #endif insn->src1 = src1; insn->src2 = src2; insn->offset = (int16_t)other; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); return true; } /*-----------------------------------------------------------------------*/ /** * make_atomic_inc: Encode an ATOMIC_INC instruction. */ static bool make_atomic_inc(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); #endif insn->dest = dest; insn->src1 = src1; insn->offset = (int16_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); return true; } /*-----------------------------------------------------------------------*/ /** * make_cmpxchg: Encode a CMPXCHG instruction. */ static bool make_cmpxchg(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); ASSERT(other < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(dest != 0); OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(src2 != 0); OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src2].type == unit->regs[dest].type); OPERAND_ASSERT(unit->regs[other].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[other].type == unit->regs[dest].type); #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->src3 = (uint16_t)other; RTLRegister * const destreg = &unit->regs[dest]; RTLRegister * const src1reg = &unit->regs[src1]; RTLRegister * const src2reg = &unit->regs[src2]; RTLRegister * const src3reg = &unit->regs[other]; const int insn_index = unit->num_insns; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.src3 = (uint16_t)other; rtl_mark_live(unit, insn_index, destreg, dest); rtl_mark_live(unit, insn_index, src1reg, src1); rtl_mark_live(unit, insn_index, src2reg, src2); rtl_mark_live(unit, insn_index, src3reg, other); return true; } /*-----------------------------------------------------------------------*/ /** * make_label: Encode a LABEL instruction. */ static bool make_label(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->blocks != NULL); ASSERT(unit->cur_block < unit->num_blocks); ASSERT(unit->label_blockmap != NULL); ASSERT(insn != NULL); ASSERT(other < unit->next_label); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->label_blockmap[other] < 0); #endif insn->label = (uint16_t)other; /* If this is _not_ the first instruction in the current basic block, * end it and create a new basic block starting here, since there could * potentially be a branch to this location. */ if ((uint32_t)unit->blocks[unit->cur_block].first_insn != unit->num_insns) { unit->blocks[unit->cur_block].last_insn = unit->num_insns - 1; if (UNLIKELY(!rtl_block_add(unit))) { log_ice(unit->handle, "Failed to start a new basic block at %u", unit->num_insns); return false; } const uint32_t new_block = unit->num_blocks - 1; /* The only ways rtl_block_add_edge() can fail is if the block * already has two outgoing edges or the new block is full of * incoming edges. We never add outgoing edges except when * terminating a block, and the new block will have no incoming * edges, so this call will always succeed. */ ASSERT(rtl_block_add_edge(unit, unit->cur_block, new_block)); unit->cur_block = new_block; unit->blocks[unit->cur_block].first_insn = unit->num_insns; } /* Save the label's unit number in the label-to-unit map. */ unit->label_blockmap[insn->label] = unit->cur_block; return true; } /*-----------------------------------------------------------------------*/ /** * make_goto: Encode a GOTO instruction. */ static bool make_goto(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(other < unit->next_label); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(other != 0); #endif insn->label = (uint16_t)other; /* Terminate the current basic block after this instruction. */ unit->blocks[unit->cur_block].last_insn = unit->num_insns; unit->have_block = false; return true; } /*-----------------------------------------------------------------------*/ /** * make_goto_cond: Encode a GOTO_IF_Z or GOTO_IF_NZ instruction. */ static bool make_goto_cond(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(other < unit->next_label); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(other != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); #endif insn->src1 = src1; insn->label = (uint16_t)other; RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, src1reg, src1); /* Terminate the current basic block after this instruction, and * start a new basic block with an edge connecting from this one. */ unit->blocks[unit->cur_block].last_insn = unit->num_insns; if (UNLIKELY(!rtl_block_add(unit))) { log_ice(unit->handle, "Failed to start a new basic block at %u", unit->num_insns); return false; } const unsigned int new_block = unit->num_blocks - 1; /* See note in make_label() for why this can never fail. */ ASSERT(rtl_block_add_edge(unit, unit->cur_block, new_block)); unit->cur_block = new_block; unit->blocks[unit->cur_block].first_insn = unit->num_insns + 1; return true; } /*-----------------------------------------------------------------------*/ /** * make_call: Encode a CALL instruction. */ static bool make_call(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(dest >= 0 && dest < unit->next_reg); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); ASSERT(other < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS if (dest != 0) { OPERAND_ASSERT(unit->regs[dest].source == RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[dest])); } OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(!(src2 == 0 && other != 0)); if (src2 != 0) { OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src2])); if (other != 0) { OPERAND_ASSERT(unit->regs[other].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[other])); } } #endif insn->dest = dest; insn->src1 = src1; insn->src2 = src2; insn->src3 = other; const int insn_index = unit->num_insns; if (dest) { RTLRegister * const destreg = &unit->regs[dest]; destreg->source = RTLREG_RESULT_NOFOLD; destreg->result.opcode = insn->opcode; destreg->result.is_imm = 0; destreg->result.src1 = src1; destreg->result.src2 = src2; destreg->result.src3 = (uint16_t)other; rtl_mark_live(unit, insn_index, destreg, dest); } rtl_mark_live(unit, insn_index, &unit->regs[src1], src1); if (src2) { rtl_mark_live(unit, insn_index, &unit->regs[src2], src2); if (other) { rtl_mark_live(unit, insn_index, &unit->regs[other], other); } } return true; } /*-----------------------------------------------------------------------*/ /** * make_return: Encode a RETURN instruction. */ static bool make_return(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS if (src1 != 0) { OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); } #endif insn->src1 = src1; if (src1) { RTLRegister * const src1reg = &unit->regs[src1]; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, src1reg, src1); } /* Terminate the current basic block, like GOTO. */ unit->blocks[unit->cur_block].last_insn = unit->num_insns; unit->have_block = false; return true; } /*-----------------------------------------------------------------------*/ /** * make_chain: Encode a CHAIN instruction. */ static bool make_chain(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(src2 >= 0 && src2 < unit->next_reg); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(!(src1 == 0 && src2 != 0)); if (src1 != 0) { OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src1])); if (src2 != 0) { OPERAND_ASSERT(unit->regs[src2].source != RTLREG_UNDEFINED); OPERAND_ASSERT(rtl_register_is_int(&unit->regs[src2])); } } #endif insn->src1 = src1; insn->src2 = src2; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, &unit->regs[src1], src1); rtl_mark_live(unit, insn_index, &unit->regs[src2], src2); /* Properly speaking, a CHAIN should terminate its basic block since * it's effectively a conditional branch out of the unit, but we only * use it in very specific circumstances which don't suffer from not * terminating the block, so we don't bother. */ return true; } /*-----------------------------------------------------------------------*/ /** * make_chain_resolve: Encode a CHAIN_RESOLVE instruction. */ static bool make_chain_resolve(RTLUnit *unit, RTLInsn *insn, int dest, int src1, int src2, uint64_t other) { ASSERT(unit != NULL); ASSERT(unit->regs != NULL); ASSERT(insn != NULL); ASSERT(src1 >= 0 && src1 < unit->next_reg); ASSERT(other < unit->num_insns); #ifdef ENABLE_OPERAND_SANITY_CHECKS OPERAND_ASSERT(src1 != 0); OPERAND_ASSERT(unit->regs[src1].source != RTLREG_UNDEFINED); OPERAND_ASSERT(unit->regs[src1].type == RTLTYPE_ADDRESS); OPERAND_ASSERT(unit->insns[other].opcode == RTLOP_CHAIN); #endif insn->src1 = src1; insn->src_imm = other; const int insn_index = unit->num_insns; rtl_mark_live(unit, insn_index, &unit->regs[src1], src1); return true; } /*************************************************************************/ /************************ Encoding function table ************************/ /*************************************************************************/ bool (* const makefunc_table[])(RTLUnit *, RTLInsn *, int, int, int, uint64_t) = { [RTLOP_NOP ] = make_nop, [RTLOP_SET_ALIAS ] = make_set_alias, [RTLOP_GET_ALIAS ] = make_get_alias, [RTLOP_MOVE ] = make_move, [RTLOP_SELECT ] = make_select, [RTLOP_SCAST ] = make_intcast, [RTLOP_ZCAST ] = make_intcast, [RTLOP_SEXT8 ] = make_intcast, [RTLOP_SEXT16 ] = make_intcast, [RTLOP_ADD ] = make_alu_2op, [RTLOP_SUB ] = make_alu_2op, [RTLOP_NEG ] = make_alu_1op, [RTLOP_MUL ] = make_alu_2op, [RTLOP_MULHU ] = make_alu_2op, [RTLOP_MULHS ] = make_alu_2op, [RTLOP_DIVU ] = make_alu_2op, [RTLOP_DIVS ] = make_alu_2op, [RTLOP_MODU ] = make_alu_2op, [RTLOP_MODS ] = make_alu_2op, [RTLOP_AND ] = make_alu_2op, [RTLOP_OR ] = make_alu_2op, [RTLOP_XOR ] = make_alu_2op, [RTLOP_NOT ] = make_alu_1op, [RTLOP_SLL ] = make_shift, [RTLOP_SRL ] = make_shift, [RTLOP_SRA ] = make_shift, [RTLOP_ROL ] = make_shift, [RTLOP_ROR ] = make_shift, [RTLOP_CLZ ] = make_clz, [RTLOP_BSWAP ] = make_alu_1op, [RTLOP_SEQ ] = make_cmp, [RTLOP_SLTU ] = make_cmp, [RTLOP_SLTS ] = make_cmp, [RTLOP_SGTU ] = make_cmp, [RTLOP_SGTS ] = make_cmp, [RTLOP_BFEXT ] = make_bitfield, [RTLOP_BFINS ] = make_bitfield, [RTLOP_ADDI ] = make_alu_imm, [RTLOP_MULI ] = make_alu_imm, [RTLOP_ANDI ] = make_alu_imm, [RTLOP_ORI ] = make_alu_imm, [RTLOP_XORI ] = make_alu_imm, [RTLOP_SLLI ] = make_alu_imm, [RTLOP_SRLI ] = make_alu_imm, [RTLOP_SRAI ] = make_alu_imm, [RTLOP_RORI ] = make_alu_imm, [RTLOP_SEQI ] = make_cmp_imm, [RTLOP_SLTUI ] = make_cmp_imm, [RTLOP_SLTSI ] = make_cmp_imm, [RTLOP_SGTUI ] = make_cmp_imm, [RTLOP_SGTSI ] = make_cmp_imm, [RTLOP_BITCAST ] = make_bitcast, [RTLOP_FCVT ] = make_fcvt, [RTLOP_FZCAST ] = make_ficast, [RTLOP_FSCAST ] = make_ficast, [RTLOP_FROUNDI ] = make_froundi, [RTLOP_FTRUNCI ] = make_froundi, [RTLOP_FNEG ] = make_fpu_1op, [RTLOP_FABS ] = make_fpu_1op, [RTLOP_FNABS ] = make_fpu_1op, [RTLOP_FADD ] = make_fpu_2op, [RTLOP_FSUB ] = make_fpu_2op, [RTLOP_FMUL ] = make_fpu_2op, [RTLOP_FDIV ] = make_fpu_2op, [RTLOP_FSQRT ] = make_fpu_1op, [RTLOP_FCMP ] = make_fcmp, [RTLOP_FMADD ] = make_fpu_3op, [RTLOP_FMSUB ] = make_fpu_3op, [RTLOP_FNMADD ] = make_fpu_3op, [RTLOP_FNMSUB ] = make_fpu_3op, [RTLOP_FGETSTATE ] = make_fgetstate, [RTLOP_FSETSTATE ] = make_fsetstate, [RTLOP_FTESTEXC ] = make_ftestexc, [RTLOP_FCLEAREXC ] = make_fclearexc, [RTLOP_FSETROUND ] = make_fsetround, [RTLOP_FCOPYROUND] = make_fcopyround, [RTLOP_VBUILD2 ] = make_vbuild2, [RTLOP_VBROADCAST] = make_vbroadcast, [RTLOP_VEXTRACT ] = make_vextract, [RTLOP_VINSERT ] = make_vinsert, [RTLOP_VFCVT ] = make_vfcvt, [RTLOP_VFCMP ] = make_vfcmp, [RTLOP_LOAD_IMM ] = make_load_imm, [RTLOP_LOAD_ARG ] = make_load_arg, [RTLOP_LOAD ] = make_load, [RTLOP_LOAD_U8 ] = make_load_narrow, [RTLOP_LOAD_S8 ] = make_load_narrow, [RTLOP_LOAD_U16 ] = make_load_narrow, [RTLOP_LOAD_S16 ] = make_load_narrow, [RTLOP_STORE ] = make_store, [RTLOP_STORE_I8 ] = make_store_narrow, [RTLOP_STORE_I16 ] = make_store_narrow, [RTLOP_LOAD_BR ] = make_load, [RTLOP_LOAD_U16_BR] = make_load_narrow, [RTLOP_LOAD_S16_BR] = make_load_narrow, [RTLOP_STORE_BR ] = make_store, [RTLOP_STORE_I16_BR] = make_store_narrow, [RTLOP_ATOMIC_INC] = make_atomic_inc, [RTLOP_CMPXCHG ] = make_cmpxchg, [RTLOP_LABEL ] = make_label, [RTLOP_GOTO ] = make_goto, [RTLOP_GOTO_IF_Z ] = make_goto_cond, [RTLOP_GOTO_IF_NZ] = make_goto_cond, [RTLOP_CALL ] = make_call, [RTLOP_CALL_TRANSPARENT] = make_call, [RTLOP_RETURN ] = make_return, [RTLOP_CHAIN ] = make_chain, [RTLOP_CHAIN_RESOLVE] = make_chain_resolve, [RTLOP_ILLEGAL ] = make_0op, }; /*************************************************************************/ /*********************** Other interface routines ************************/ /*************************************************************************/ bool rtl_add_insn_copy(RTLUnit *unit, const RTLInsn *insn) { ASSERT(unit != NULL); ASSERT(!unit->finalized); ASSERT(unit->insns != NULL); ASSERT(unit->blocks != NULL); ASSERT(unit->regs != NULL); ASSERT(unit->label_blockmap != NULL); ASSERT(insn); if (!rtl_add_insn(unit, RTLOP_NOP, 0, 0, 0, 0)) { return false; } const int insn_index = unit->num_insns - 1; unit->insns[insn_index] = *insn; if (insn->dest) { RTLRegister * const destreg = &unit->regs[insn->dest]; rtl_mark_live(unit, insn_index, destreg, insn->dest); } if (insn->src1) { RTLRegister * const src1reg = &unit->regs[insn->src1]; rtl_mark_live(unit, insn_index, src1reg, insn->src1); } if (insn->src2) { RTLRegister * const src2reg = &unit->regs[insn->src2]; rtl_mark_live(unit, insn_index, src2reg, insn->src2); } if (rtl_opcode_has_src3(insn->opcode) && insn->src3) { RTLRegister * const src3reg = &unit->regs[insn->src3]; rtl_mark_live(unit, insn_index, src3reg, insn->src3); } return true; } /*************************************************************************/ /*************************************************************************/