luna - luajit/src/lj_asm

   1/*
   2** ARM64 IR assembler (SSA IR -> machine code).
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4**
   5** Contributed by Djordje Kovacevic and Stefan Pejic from RT-RK.com.
   6** Sponsored by Cisco Systems, Inc.
   7*/
   8
   9/* -- Register allocator extensions --------------------------------------- */
  10
  11/* Allocate a register with a hint. */
  12static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
  13{
  14  Reg r = IR(ref)->r;
  15  if (ra_noreg(r)) {
  16    if (!ra_hashint(r) && !iscrossref(as, ref))
  17      ra_sethint(IR(ref)->r, hint);  /* Propagate register hint. */
  18    r = ra_allocref(as, ref, allow);
  19  }
  20  ra_noweak(as, r);
  21  return r;
  22}
  23
  24/* Allocate two source registers for three-operand instructions. */
  25static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
  26{
  27  IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
  28  Reg left = irl->r, right = irr->r;
  29  if (ra_hasreg(left)) {
  30    ra_noweak(as, left);
  31    if (ra_noreg(right))
  32      right = ra_allocref(as, ir->op2, rset_exclude(allow, left));
  33    else
  34      ra_noweak(as, right);
  35  } else if (ra_hasreg(right)) {
  36    ra_noweak(as, right);
  37    left = ra_allocref(as, ir->op1, rset_exclude(allow, right));
  38  } else if (ra_hashint(right)) {
  39    right = ra_allocref(as, ir->op2, allow);
  40    left = ra_alloc1(as, ir->op1, rset_exclude(allow, right));
  41  } else {
  42    left = ra_allocref(as, ir->op1, allow);
  43    right = ra_alloc1(as, ir->op2, rset_exclude(allow, left));
  44  }
  45  return left | (right << 8);
  46}
  47
  48/* -- Guard handling ------------------------------------------------------ */
  49
  50/* Setup all needed exit stubs. */
  51static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
  52{
  53  ExitNo i;
  54  int ind;
  55  MCode *target = (MCode *)(void *)lj_vm_exit_handler;
  56  MCode *mxp = as->mctop;
  57  if (mxp - (nexits + 4 + MCLIM_REDZONE) < as->mclim)
  58    asm_mclimit(as);
  59  ind = !A64F_S_OK(target - (mxp - nexits - 2), 26);
  60  /* !ind: 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno;
  61  **  ind: 1: str lr,[sp]; ldr lr, [gl, K64_VXH]; blr lr; movz w0,traceno;
  62  **          bl <1; bl <1; ...
  63  */
  64  for (i = nexits-1; (int32_t)i >= 0; i--)
  65    *--mxp = A64I_LE(A64I_BL | A64F_S26(-3-ind-i));
  66  as->mcexit = mxp;
  67  *--mxp = A64I_LE(A64I_MOVZw | A64F_U16(as->T->traceno));
  68  if (ind) {
  69    *--mxp = A64I_LE(A64I_BLR_AUTH | A64F_N(RID_LR));
  70    *--mxp = A64I_LE(A64I_LDRx | A64F_D(RID_LR) | A64F_N(RID_GL) | A64F_U12(glofs(as, &as->J->k64[LJ_K64_VM_EXIT_HANDLER]) >> 3));
  71  } else {
  72    mxp--;
  73    *mxp = A64I_LE(A64I_BL | A64F_S26(target-mxp));
  74  }
  75  *--mxp = A64I_LE(A64I_STRx | A64F_D(RID_LR) | A64F_N(RID_SP));
  76  as->mctop = mxp;
  77}
  78
  79static MCode *asm_exitstub_addr(ASMState *as, ExitNo exitno)
  80{
  81  /* Keep this in-sync with exitstub_trace_addr(). */
  82  return as->mcexit + exitno;
  83}
  84
  85/* Emit conditional branch to exit for guard. */
  86static void asm_guardcc(ASMState *as, A64CC cc)
  87{
  88  MCode *target = asm_exitstub_addr(as, as->snapno);
  89  MCode *p = as->mcp;
  90  if (LJ_UNLIKELY(p == as->invmcp)) {
  91    as->loopinv = 1;
  92    *p = A64I_B | A64F_S26(target-p);
  93    emit_cond_branch(as, cc^1, p-1);
  94    return;
  95  }
  96  emit_cond_branch(as, cc, target);
  97}
  98
  99/* Emit test and branch instruction to exit for guard, if in range. */
 100static int asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit)
 101{
 102  MCode *target = asm_exitstub_addr(as, as->snapno);
 103  MCode *p = as->mcp;
 104  ptrdiff_t delta = target - p;
 105  if (LJ_UNLIKELY(p == as->invmcp)) {
 106    if (as->orignins > 1023) return 0;  /* Delta might end up too large. */
 107    as->loopinv = 1;
 108    *p = A64I_B | A64F_S26(delta);
 109    ai ^= 0x01000000u;
 110    target = p-1;
 111  } else if (LJ_UNLIKELY(delta >= 0x1fff)) {
 112    return 0;
 113  }
 114  emit_tnb(as, ai, r, bit, target);
 115  return 1;
 116}
 117
 118/* Emit compare and branch instruction to exit for guard. */
 119static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r)
 120{
 121  MCode *target = asm_exitstub_addr(as, as->snapno);
 122  MCode *p = as->mcp;
 123  if (LJ_UNLIKELY(p == as->invmcp)) {
 124    as->loopinv = 1;
 125    *p = A64I_B | A64F_S26(target-p);
 126    emit_cnb(as, ai^0x01000000u, r, p-1);
 127    return;
 128  }
 129  emit_cnb(as, ai, r, target);
 130}
 131
 132/* -- Operand fusion ------------------------------------------------------ */
 133
 134/* Limit linear search to this distance. Avoids O(n^2) behavior. */
 135#define CONFLICT_SEARCH_LIM	31
 136
 137static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
 138{
 139  if (irref_isk(ref)) {
 140    IRIns *ir = IR(ref);
 141    if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
 142      *k = ir->i;
 143      return 1;
 144    } else if (checki32((int64_t)ir_k64(ir)->u64)) {
 145      *k = (int32_t)ir_k64(ir)->u64;
 146      return 1;
 147    }
 148  }
 149  return 0;
 150}
 151
 152/* Check if there's no conflicting instruction between curins and ref. */
 153static int noconflict(ASMState *as, IRRef ref, IROp conflict)
 154{
 155  IRIns *ir = as->ir;
 156  IRRef i = as->curins;
 157  if (i > ref + CONFLICT_SEARCH_LIM)
 158    return 0;  /* Give up, ref is too far away. */
 159  while (--i > ref)
 160    if (ir[i].o == conflict)
 161      return 0;  /* Conflict found. */
 162  return 1;  /* Ok, no conflict. */
 163}
 164
 165/* Fuse the array base of colocated arrays. */
 166static int32_t asm_fuseabase(ASMState *as, IRRef ref)
 167{
 168  IRIns *ir = IR(ref);
 169  if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
 170      !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
 171    return (int32_t)sizeof(GCtab);
 172  return 0;
 173}
 174
 175#define FUSE_REG	0x40000000
 176
 177/* Fuse array/hash/upvalue reference into register+offset operand. */
 178static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow,
 179			  A64Ins ins)
 180{
 181  IRIns *ir = IR(ref);
 182  if (ra_noreg(ir->r)) {
 183    if (ir->o == IR_AREF) {
 184      if (mayfuse(as, ref)) {
 185	if (irref_isk(ir->op2)) {
 186	  IRRef tab = IR(ir->op1)->op1;
 187	  int32_t ofs = asm_fuseabase(as, tab);
 188	  IRRef refa = ofs ? tab : ir->op1;
 189	  ofs += 8*IR(ir->op2)->i;
 190	  if (emit_checkofs(ins, ofs)) {
 191	    *ofsp = ofs;
 192	    return ra_alloc1(as, refa, allow);
 193	  }
 194	} else {
 195	  Reg base = ra_alloc1(as, ir->op1, allow);
 196	  *ofsp = FUSE_REG|ra_alloc1(as, ir->op2, rset_exclude(allow, base));
 197	  return base;
 198	}
 199      }
 200    } else if (ir->o == IR_HREFK) {
 201      if (mayfuse(as, ref)) {
 202	int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
 203	if (emit_checkofs(ins, ofs)) {
 204	  *ofsp = ofs;
 205	  return ra_alloc1(as, ir->op1, allow);
 206	}
 207      }
 208    } else if (ir->o == IR_UREFC) {
 209      if (irref_isk(ir->op1)) {
 210	GCfunc *fn = ir_kfunc(IR(ir->op1));
 211	GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
 212	int64_t ofs = glofs(as, &uv->tv);
 213	if (emit_checkofs(ins, ofs)) {
 214	  *ofsp = (int32_t)ofs;
 215	  return RID_GL;
 216	}
 217      }
 218    } else if (ir->o == IR_TMPREF) {
 219      *ofsp = (int32_t)glofs(as, &J2G(as->J)->tmptv);
 220      return RID_GL;
 221    }
 222  }
 223  *ofsp = 0;
 224  return ra_alloc1(as, ref, allow);
 225}
 226
 227/* Fuse m operand into arithmetic/logic instructions. */
 228static uint32_t asm_fuseopm(ASMState *as, A64Ins ai, IRRef ref, RegSet allow)
 229{
 230  IRIns *ir = IR(ref);
 231  int logical = (ai & 0x1f000000) == 0x0a000000;
 232  if (ra_hasreg(ir->r)) {
 233    ra_noweak(as, ir->r);
 234    return A64F_M(ir->r);
 235  } else if (irref_isk(ref)) {
 236    int64_t k = get_k64val(as, ref);
 237    uint32_t m = logical ? emit_isk13(k, irt_is64(ir->t)) :
 238			   emit_isk12(irt_is64(ir->t) ? k : (int32_t)k);
 239    if (m)
 240      return m;
 241  } else if (mayfuse(as, ref)) {
 242    if ((ir->o >= IR_BSHL && ir->o <= IR_BSAR && irref_isk(ir->op2)) ||
 243	(ir->o == IR_ADD && ir->op1 == ir->op2)) {
 244      A64Shift sh = ir->o == IR_BSHR ? A64SH_LSR :
 245		    ir->o == IR_BSAR ? A64SH_ASR : A64SH_LSL;
 246      int shift = ir->o == IR_ADD ? 1 :
 247		    (IR(ir->op2)->i & (irt_is64(ir->t) ? 63 : 31));
 248      IRIns *irl = IR(ir->op1);
 249      if (sh == A64SH_LSL &&
 250	  irl->o == IR_CONV && !logical &&
 251	  irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
 252	  shift <= 4 &&
 253	  canfuse(as, irl)) {
 254	Reg m = ra_alloc1(as, irl->op1, allow);
 255	return A64F_M(m) | A64F_EXSH(A64EX_SXTW, shift);
 256      } else {
 257	Reg m = ra_alloc1(as, ir->op1, allow);
 258	return A64F_M(m) | A64F_SH(sh, shift);
 259      }
 260    } else if (ir->o == IR_BROR && logical && irref_isk(ir->op2)) {
 261      Reg m = ra_alloc1(as, ir->op1, allow);
 262      int shift = (IR(ir->op2)->i & (irt_is64(ir->t) ? 63 : 31));
 263      return A64F_M(m) | A64F_SH(A64SH_ROR, shift);
 264    } else if (ir->o == IR_CONV && !logical &&
 265	       ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)) {
 266      Reg m = ra_alloc1(as, ir->op1, allow);
 267      return A64F_M(m) | A64F_EX(A64EX_SXTW);
 268    }
 269  }
 270  return A64F_M(ra_allocref(as, ref, allow));
 271}
 272
 273/* Fuse XLOAD/XSTORE reference into load/store operand. */
 274static void asm_fusexref(ASMState *as, A64Ins ai, Reg rd, IRRef ref,
 275			 RegSet allow)
 276{
 277  IRIns *ir = IR(ref);
 278  Reg base;
 279  int32_t ofs = 0;
 280  if (ra_noreg(ir->r) && canfuse(as, ir)) {
 281    if (ir->o == IR_ADD) {
 282      if (asm_isk32(as, ir->op2, &ofs) && emit_checkofs(ai, ofs)) {
 283	ref = ir->op1;
 284      } else {
 285	Reg rn, rm;
 286	IRRef lref = ir->op1, rref = ir->op2;
 287	IRIns *irl = IR(lref);
 288	if (mayfuse(as, irl->op1)) {
 289	  unsigned int shift = 4;
 290	  if (irl->o == IR_BSHL && irref_isk(irl->op2)) {
 291	    shift = (IR(irl->op2)->i & 63);
 292	  } else if (irl->o == IR_ADD && irl->op1 == irl->op2) {
 293	    shift = 1;
 294	  }
 295	  if ((ai >> 30) == shift) {
 296	    lref = irl->op1;
 297	    irl = IR(lref);
 298	    ai |= A64I_LS_SH;
 299	  }
 300	}
 301	if (irl->o == IR_CONV &&
 302	    irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
 303	    canfuse(as, irl)) {
 304	  lref = irl->op1;
 305	  ai |= A64I_LS_SXTWx;
 306	} else {
 307	  ai |= A64I_LS_LSLx;
 308	}
 309	rm = ra_alloc1(as, lref, allow);
 310	rn = ra_alloc1(as, rref, rset_exclude(allow, rm));
 311	emit_dnm(as, (ai^A64I_LS_R), (rd & 31), rn, rm);
 312	return;
 313      }
 314    } else if (ir->o == IR_STRREF) {
 315      if (asm_isk32(as, ir->op2, &ofs)) {
 316	ref = ir->op1;
 317      } else if (asm_isk32(as, ir->op1, &ofs)) {
 318	ref = ir->op2;
 319      } else {
 320	Reg refk = irref_isk(ir->op1) ? ir->op1 : ir->op2;
 321	Reg refv = irref_isk(ir->op1) ? ir->op2 : ir->op1;
 322	Reg rn = ra_alloc1(as, refv, allow);
 323	IRIns *irr = IR(refk);
 324	uint32_t m;
 325	if (irr+1 == ir && !ra_used(irr) &&
 326	    irr->o == IR_ADD && irref_isk(irr->op2)) {
 327	  ofs = sizeof(GCstr) + IR(irr->op2)->i;
 328	  if (emit_checkofs(ai, ofs)) {
 329	    Reg rm = ra_alloc1(as, irr->op1, rset_exclude(allow, rn));
 330	    m = A64F_M(rm) | A64F_EX(A64EX_SXTW);
 331	    goto skipopm;
 332	  }
 333	}
 334	m = asm_fuseopm(as, 0, refk, rset_exclude(allow, rn));
 335	ofs = sizeof(GCstr);
 336      skipopm:
 337	emit_lso(as, ai, rd, rd, ofs);
 338	emit_dn(as, A64I_ADDx^m, rd, rn);
 339	return;
 340      }
 341      ofs += sizeof(GCstr);
 342      if (!emit_checkofs(ai, ofs)) {
 343	Reg rn = ra_alloc1(as, ref, allow);
 344	Reg rm = ra_allock(as, ofs, rset_exclude(allow, rn));
 345	emit_dnm(as, (ai^A64I_LS_R)|A64I_LS_UXTWx, rd, rn, rm);
 346	return;
 347      }
 348    }
 349  }
 350  base = ra_alloc1(as, ref, allow);
 351  emit_lso(as, ai, (rd & 31), base, ofs);
 352}
 353
 354/* Fuse FP multiply-add/sub. */
 355static int asm_fusemadd(ASMState *as, IRIns *ir, A64Ins ai, A64Ins air)
 356{
 357  IRRef lref = ir->op1, rref = ir->op2;
 358  IRIns *irm;
 359  if ((as->flags & JIT_F_OPT_FMA) &&
 360      lref != rref &&
 361      ((mayfuse(as, lref) && (irm = IR(lref), irm->o == IR_MUL) &&
 362       ra_noreg(irm->r)) ||
 363       (mayfuse(as, rref) && (irm = IR(rref), irm->o == IR_MUL) &&
 364       (rref = lref, ai = air, ra_noreg(irm->r))))) {
 365    Reg dest = ra_dest(as, ir, RSET_FPR);
 366    Reg add = ra_hintalloc(as, rref, dest, RSET_FPR);
 367    Reg left = ra_alloc2(as, irm,
 368			 rset_exclude(rset_exclude(RSET_FPR, dest), add));
 369    Reg right = (left >> 8); left &= 255;
 370    emit_dnma(as, ai, (dest & 31), (left & 31), (right & 31), (add & 31));
 371    return 1;
 372  }
 373  return 0;
 374}
 375
 376/* Fuse BAND + BSHL/BSHR into UBFM. */
 377static int asm_fuseandshift(ASMState *as, IRIns *ir)
 378{
 379  IRIns *irl = IR(ir->op1);
 380  lj_assertA(ir->o == IR_BAND, "bad usage");
 381  if (canfuse(as, irl) && irref_isk(ir->op2)) {
 382    uint64_t mask = get_k64val(as, ir->op2);
 383    if (irref_isk(irl->op2) && (irl->o == IR_BSHR || irl->o == IR_BSHL)) {
 384      int32_t shmask = irt_is64(irl->t) ? 63 : 31;
 385      int32_t shift = (IR(irl->op2)->i & shmask);
 386      int32_t imms = shift;
 387      if (irl->o == IR_BSHL) {
 388	mask >>= shift;
 389	shift = (shmask-shift+1) & shmask;
 390	imms = 0;
 391      }
 392      if (mask && !((mask+1) & mask)) {  /* Contiguous 1-bits at the bottom. */
 393	Reg dest = ra_dest(as, ir, RSET_GPR);
 394	Reg left = ra_alloc1(as, irl->op1, RSET_GPR);
 395	A64Ins ai = shmask == 63 ? A64I_UBFMx : A64I_UBFMw;
 396	imms += 63 - emit_clz64(mask);
 397	if (imms > shmask) imms = shmask;
 398	emit_dn(as, ai | A64F_IMMS(imms) | A64F_IMMR(shift), dest, left);
 399	return 1;
 400      }
 401    }
 402  }
 403  return 0;
 404}
 405
 406/* Fuse BOR(BSHL, BSHR) into EXTR/ROR. */
 407static int asm_fuseorshift(ASMState *as, IRIns *ir)
 408{
 409  IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
 410  lj_assertA(ir->o == IR_BOR, "bad usage");
 411  if (canfuse(as, irl) && canfuse(as, irr) &&
 412      ((irl->o == IR_BSHR && irr->o == IR_BSHL) ||
 413       (irl->o == IR_BSHL && irr->o == IR_BSHR))) {
 414    if (irref_isk(irl->op2) && irref_isk(irr->op2)) {
 415      IRRef lref = irl->op1, rref = irr->op1;
 416      uint32_t lshift = IR(irl->op2)->i, rshift = IR(irr->op2)->i;
 417      if (irl->o == IR_BSHR) {  /* BSHR needs to be the right operand. */
 418	uint32_t tmp2;
 419	IRRef tmp1 = lref; lref = rref; rref = tmp1;
 420	tmp2 = lshift; lshift = rshift; rshift = tmp2;
 421      }
 422      if (rshift + lshift == (irt_is64(ir->t) ? 64 : 32)) {
 423	A64Ins ai = irt_is64(ir->t) ? A64I_EXTRx : A64I_EXTRw;
 424	Reg dest = ra_dest(as, ir, RSET_GPR);
 425	Reg left = ra_alloc1(as, lref, RSET_GPR);
 426	Reg right = ra_alloc1(as, rref, rset_exclude(RSET_GPR, left));
 427	emit_dnm(as, ai | A64F_IMMS(rshift), dest, left, right);
 428	return 1;
 429      }
 430    }
 431  }
 432  return 0;
 433}
 434
 435/* -- Calls --------------------------------------------------------------- */
 436
 437/* Generate a call to a C function. */
 438static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 439{
 440  uint32_t n, nargs = CCI_XNARGS(ci);
 441  int32_t spofs = 0, spalign = LJ_HASFFI && LJ_TARGET_OSX ? 0 : 7;
 442  Reg gpr, fpr = REGARG_FIRSTFPR;
 443  if (ci->func)
 444    emit_call(as, ci->func);
 445  for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
 446    as->cost[gpr] = REGCOST(~0u, ASMREF_L);
 447  gpr = REGARG_FIRSTGPR;
 448#if LJ_HASFFI && LJ_ABI_WIN
 449  if ((ci->flags & CCI_VARARG)) {
 450    fpr = REGARG_LASTFPR+1;
 451  }
 452#endif
 453  for (n = 0; n < nargs; n++) { /* Setup args. */
 454    IRRef ref = args[n];
 455    IRIns *ir = IR(ref);
 456    if (ref) {
 457      if (irt_isfp(ir->t)) {
 458	if (fpr <= REGARG_LASTFPR) {
 459	  lj_assertA(rset_test(as->freeset, fpr),
 460		     "reg %d not free", fpr);  /* Must have been evicted. */
 461	  ra_leftov(as, fpr, ref);
 462	  fpr++;
 463#if LJ_HASFFI && LJ_ABI_WIN
 464	} else if ((ci->flags & CCI_VARARG) && (gpr <= REGARG_LASTGPR)) {
 465	  Reg rf = ra_alloc1(as, ref, RSET_FPR);
 466	  emit_dn(as, A64I_FMOV_R_D, gpr++, rf & 31);
 467#endif
 468	} else {
 469	  Reg r = ra_alloc1(as, ref, RSET_FPR);
 470	  int32_t al = spalign;
 471#if LJ_HASFFI && LJ_TARGET_OSX
 472	  al |= irt_isnum(ir->t) ? 7 : 3;
 473#endif
 474	  spofs = (spofs + al) & ~al;
 475	  if (LJ_BE && al >= 7 && !irt_isnum(ir->t)) spofs += 4, al -= 4;
 476	  emit_spstore(as, ir, r, spofs);
 477	  spofs += al + 1;
 478	}
 479      } else {
 480	if (gpr <= REGARG_LASTGPR) {
 481	  lj_assertA(rset_test(as->freeset, gpr),
 482		     "reg %d not free", gpr);  /* Must have been evicted. */
 483	  ra_leftov(as, gpr, ref);
 484	  gpr++;
 485	} else {
 486	  Reg r = ra_alloc1(as, ref, RSET_GPR);
 487	  int32_t al = spalign;
 488#if LJ_HASFFI && LJ_TARGET_OSX
 489	  al |= irt_size(ir->t) - 1;
 490#endif
 491	  spofs = (spofs + al) & ~al;
 492	  if (al >= 3) {
 493	    if (LJ_BE && al >= 7 && !irt_is64(ir->t)) spofs += 4, al -= 4;
 494	    emit_spstore(as, ir, r, spofs);
 495	  } else {
 496	    lj_assertA(al == 0 || al == 1, "size %d unexpected", al + 1);
 497	    emit_lso(as, al ? A64I_STRH : A64I_STRB, r, RID_SP, spofs);
 498	  }
 499	  spofs += al + 1;
 500	}
 501      }
 502#if LJ_HASFFI && LJ_TARGET_OSX
 503    } else {  /* Marker for start of varargs. */
 504      gpr = REGARG_LASTGPR+1;
 505      fpr = REGARG_LASTFPR+1;
 506      spalign = 7;
 507#endif
 508    }
 509  }
 510}
 511
 512/* Setup result reg/sp for call. Evict scratch regs. */
 513static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
 514{
 515  RegSet drop = RSET_SCRATCH;
 516  int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
 517  if (ra_hasreg(ir->r))
 518    rset_clear(drop, ir->r); /* Dest reg handled below. */
 519  if (hiop && ra_hasreg((ir+1)->r))
 520    rset_clear(drop, (ir+1)->r);  /* Dest reg handled below. */
 521  ra_evictset(as, drop); /* Evictions must be performed first. */
 522  if (ra_used(ir)) {
 523    lj_assertA(!irt_ispri(ir->t), "PRI dest");
 524    if (irt_isfp(ir->t)) {
 525      if (ci->flags & CCI_CASTU64) {
 526	Reg dest = ra_dest(as, ir, RSET_FPR) & 31;
 527	emit_dn(as, irt_isnum(ir->t) ? A64I_FMOV_D_R : A64I_FMOV_S_R,
 528		dest, RID_RET);
 529      } else {
 530	ra_destreg(as, ir, RID_FPRET);
 531      }
 532    } else if (hiop) {
 533      ra_destpair(as, ir);
 534    } else {
 535      ra_destreg(as, ir, RID_RET);
 536    }
 537  }
 538  UNUSED(ci);
 539}
 540
 541static void asm_callx(ASMState *as, IRIns *ir)
 542{
 543  IRRef args[CCI_NARGS_MAX*2];
 544  CCallInfo ci;
 545  IRRef func;
 546  IRIns *irf;
 547  ci.flags = asm_callx_flags(as, ir);
 548  asm_collectargs(as, ir, &ci, args);
 549  asm_setupresult(as, ir, &ci);
 550  func = ir->op2; irf = IR(func);
 551  if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
 552  if (irref_isk(func)) {  /* Call to constant address. */
 553    ci.func = (ASMFunction)(ir_k64(irf)->u64);
 554  } else {  /* Need a non-argument register for indirect calls. */
 555    Reg freg = ra_alloc1(as, func, RSET_RANGE(RID_X8, RID_MAX_GPR)-RSET_FIXED);
 556    emit_n(as, A64I_BLR_AUTH, freg);
 557    ci.func = (ASMFunction)(void *)0;
 558  }
 559  asm_gencall(as, &ci, args);
 560}
 561
 562/* -- Returns ------------------------------------------------------------- */
 563
 564/* Return to lower frame. Guard that it goes to the right spot. */
 565static void asm_retf(ASMState *as, IRIns *ir)
 566{
 567  Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
 568  void *pc = ir_kptr(IR(ir->op2));
 569  int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
 570  as->topslot -= (BCReg)delta;
 571  if ((int32_t)as->topslot < 0) as->topslot = 0;
 572  irt_setmark(IR(REF_BASE)->t);  /* Children must not coalesce with BASE reg. */
 573  emit_setgl(as, base, jit_base);
 574  emit_addptr(as, base, -8*delta);
 575  asm_guardcc(as, CC_NE);
 576  emit_nm(as, A64I_CMPx, RID_TMP,
 577	  ra_allock(as, i64ptr(pc), rset_exclude(RSET_GPR, base)));
 578  emit_lso(as, A64I_LDRx, RID_TMP, base, -8);
 579}
 580
 581/* -- Buffer operations --------------------------------------------------- */
 582
 583#if LJ_HASBUFFER
 584static void asm_bufhdr_write(ASMState *as, Reg sb)
 585{
 586  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
 587  IRIns irgc;
 588  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
 589  emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
 590  emit_dn(as, A64I_BFMx | A64F_IMMS(lj_fls(SBUF_MASK_FLAG)) | A64F_IMMR(0), RID_TMP, tmp);
 591  emit_getgl(as, RID_TMP, cur_L);
 592  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
 593}
 594#endif
 595
 596/* -- Type conversions ---------------------------------------------------- */
 597
 598static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
 599{
 600  Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
 601  Reg dest = ra_dest(as, ir, RSET_GPR);
 602  asm_guardcc(as, CC_NE);
 603  emit_nm(as, A64I_FCMPd, (tmp & 31), (left & 31));
 604  emit_dn(as, A64I_FCVT_F64_S32, (tmp & 31), dest);
 605  emit_dn(as, A64I_FCVT_S32_F64, dest, (left & 31));
 606}
 607
 608static void asm_tobit(ASMState *as, IRIns *ir)
 609{
 610  RegSet allow = RSET_FPR;
 611  Reg left = ra_alloc1(as, ir->op1, allow);
 612  Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
 613  Reg tmp = ra_scratch(as, rset_clear(allow, right));
 614  Reg dest = ra_dest(as, ir, RSET_GPR);
 615  emit_dn(as, A64I_FMOV_R_S, dest, (tmp & 31));
 616  emit_dnm(as, A64I_FADDd, (tmp & 31), (left & 31), (right & 31));
 617}
 618
 619static void asm_conv(ASMState *as, IRIns *ir)
 620{
 621  IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 622  int st64 = (st == IRT_I64 || st == IRT_U64 || st == IRT_P64);
 623  int stfp = (st == IRT_NUM || st == IRT_FLOAT);
 624  IRRef lref = ir->op1;
 625  lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
 626  if (irt_isfp(ir->t)) {
 627    Reg dest = ra_dest(as, ir, RSET_FPR);
 628    if (stfp) {  /* FP to FP conversion. */
 629      emit_dn(as, st == IRT_NUM ? A64I_FCVT_F32_F64 : A64I_FCVT_F64_F32,
 630	      (dest & 31), (ra_alloc1(as, lref, RSET_FPR) & 31));
 631    } else {  /* Integer to FP conversion. */
 632      Reg left = ra_alloc1(as, lref, RSET_GPR);
 633      A64Ins ai = irt_isfloat(ir->t) ?
 634	(((IRT_IS64 >> st) & 1) ?
 635	 (st == IRT_I64 ? A64I_FCVT_F32_S64 : A64I_FCVT_F32_U64) :
 636	 (st == IRT_INT ? A64I_FCVT_F32_S32 : A64I_FCVT_F32_U32)) :
 637	(((IRT_IS64 >> st) & 1) ?
 638	 (st == IRT_I64 ? A64I_FCVT_F64_S64 : A64I_FCVT_F64_U64) :
 639	 (st == IRT_INT ? A64I_FCVT_F64_S32 : A64I_FCVT_F64_U32));
 640      emit_dn(as, ai, (dest & 31), left);
 641    }
 642  } else if (stfp) {  /* FP to integer conversion. */
 643    if (irt_isguard(ir->t)) {
 644      /* Checked conversions are only supported from number to int. */
 645      lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
 646		 "bad type for checked CONV");
 647      asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
 648    } else {
 649      Reg left = ra_alloc1(as, lref, RSET_FPR);
 650      Reg dest = ra_dest(as, ir, RSET_GPR);
 651      lj_assertA(!irt_isu32(ir->t), "bad CONV u32.fp emitted");
 652      if (irt_isu64(ir->t)) {
 653	emit_dnm(as, A64I_CSELx | A64F_CC(CC_VC), dest, dest, RID_TMP);
 654	emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(1), dest);
 655	emit_dn(as, st == IRT_NUM ? A64I_FCVT_U64_F64 : A64I_FCVT_U64_F32, RID_TMP, (left & 31));
 656	emit_dn(as, st == IRT_NUM ? A64I_FCVT_S64_F64 : A64I_FCVT_S64_F32, dest, (left & 31));
 657      } else {
 658	A64Ins ai = irt_is64(ir->t) ?
 659	  (st == IRT_NUM ? A64I_FCVT_S64_F64 : A64I_FCVT_S64_F32) :
 660	  (st == IRT_NUM ? A64I_FCVT_S32_F64 : A64I_FCVT_S32_F32);
 661	emit_dn(as, ai, dest, (left & 31));
 662      }
 663    }
 664  } else if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
 665    Reg dest = ra_dest(as, ir, RSET_GPR);
 666    Reg left = ra_alloc1(as, lref, RSET_GPR);
 667    A64Ins ai = st == IRT_I8 ? A64I_SXTBw :
 668		st == IRT_U8 ? A64I_UXTBw :
 669		st == IRT_I16 ? A64I_SXTHw : A64I_UXTHw;
 670    lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
 671    emit_dn(as, ai, dest, left);
 672  } else {
 673    Reg dest = ra_dest(as, ir, RSET_GPR);
 674    if (irt_is64(ir->t)) {
 675      if (st64 || !(ir->op2 & IRCONV_SEXT)) {
 676	/* 64/64 bit no-op (cast) or 32 to 64 bit zero extension. */
 677	ra_leftov(as, dest, lref);  /* Do nothing, but may need to move regs. */
 678      } else {  /* 32 to 64 bit sign extension. */
 679	Reg left = ra_alloc1(as, lref, RSET_GPR);
 680	emit_dn(as, A64I_SXTW, dest, left);
 681      }
 682    } else {
 683      if (st64 && !(ir->op2 & IRCONV_NONE)) {
 684	/* This is either a 32 bit reg/reg mov which zeroes the hiword
 685	** or a load of the loword from a 64 bit address.
 686	*/
 687	Reg left = ra_alloc1(as, lref, RSET_GPR);
 688	emit_dm(as, A64I_MOVw, dest, left);
 689      } else {  /* 32/32 bit no-op (cast). */
 690	ra_leftov(as, dest, lref);  /* Do nothing, but may need to move regs. */
 691      }
 692    }
 693  }
 694}
 695
 696static void asm_strto(ASMState *as, IRIns *ir)
 697{
 698  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
 699  IRRef args[2];
 700  Reg tmp;
 701  int32_t ofs = 0;
 702  ra_evictset(as, RSET_SCRATCH);
 703  if (ra_used(ir)) {
 704    if (ra_hasspill(ir->s)) {
 705      ofs = sps_scale(ir->s);
 706      if (ra_hasreg(ir->r)) {
 707	ra_free(as, ir->r);
 708	ra_modified(as, ir->r);
 709	emit_spload(as, ir, ir->r, ofs);
 710      }
 711    } else {
 712      Reg dest = ra_dest(as, ir, RSET_FPR);
 713      emit_lso(as, A64I_LDRd, (dest & 31), RID_SP, 0);
 714    }
 715  }
 716  asm_guardcnb(as, A64I_CBZ, RID_RET);
 717  args[0] = ir->op1; /* GCstr *str */
 718  args[1] = ASMREF_TMP1; /* TValue *n  */
 719  asm_gencall(as, ci, args);
 720  tmp = ra_releasetmp(as, ASMREF_TMP1);
 721  emit_opk(as, A64I_ADDx, tmp, RID_SP, ofs, RSET_GPR);
 722}
 723
 724/* -- Memory references --------------------------------------------------- */
 725
 726/* Store tagged value for ref at base+ofs. */
 727static void asm_tvstore64(ASMState *as, Reg base, int32_t ofs, IRRef ref)
 728{
 729  RegSet allow = rset_exclude(RSET_GPR, base);
 730  IRIns *ir = IR(ref);
 731  lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
 732	     "store of IR type %d", irt_type(ir->t));
 733  if (irref_isk(ref)) {
 734    TValue k;
 735    lj_ir_kvalue(as->J->L, &k, ir);
 736    emit_lso(as, A64I_STRx, ra_allock(as, k.u64, allow), base, ofs);
 737  } else {
 738    Reg src = ra_alloc1(as, ref, allow);
 739    rset_clear(allow, src);
 740    if (irt_isinteger(ir->t)) {
 741      Reg type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47, allow);
 742      emit_lso(as, A64I_STRx, RID_TMP, base, ofs);
 743      emit_dnm(as, A64I_ADDx | A64F_EX(A64EX_UXTW), RID_TMP, type, src);
 744    } else {
 745      Reg type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
 746      emit_lso(as, A64I_STRx, RID_TMP, base, ofs);
 747      emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 47), RID_TMP, src, type);
 748    }
 749  }
 750}
 751
 752/* Get pointer to TValue. */
 753static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
 754{
 755  if ((mode & IRTMPREF_IN1)) {
 756    IRIns *ir = IR(ref);
 757    if (irt_isnum(ir->t)) {
 758      if (irref_isk(ref) && !(mode & IRTMPREF_OUT1)) {
 759	/* Use the number constant itself as a TValue. */
 760	ra_allockreg(as, i64ptr(ir_knum(ir)), dest);
 761	return;
 762      }
 763      emit_lso(as, A64I_STRd, (ra_alloc1(as, ref, RSET_FPR) & 31), dest, 0);
 764    } else {
 765      asm_tvstore64(as, dest, 0, ref);
 766    }
 767  }
 768  /* g->tmptv holds the TValue(s). */
 769  emit_dn(as, A64I_ADDx^emit_isk12(glofs(as, &J2G(as->J)->tmptv)), dest, RID_GL);
 770}
 771
 772static void asm_aref(ASMState *as, IRIns *ir)
 773{
 774  Reg dest = ra_dest(as, ir, RSET_GPR);
 775  Reg idx, base;
 776  if (irref_isk(ir->op2)) {
 777    IRRef tab = IR(ir->op1)->op1;
 778    int32_t ofs = asm_fuseabase(as, tab);
 779    IRRef refa = ofs ? tab : ir->op1;
 780    uint32_t k = emit_isk12(ofs + 8*IR(ir->op2)->i);
 781    if (k) {
 782      base = ra_alloc1(as, refa, RSET_GPR);
 783      emit_dn(as, A64I_ADDx^k, dest, base);
 784      return;
 785    }
 786  }
 787  base = ra_alloc1(as, ir->op1, RSET_GPR);
 788  idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
 789  emit_dnm(as, A64I_ADDx | A64F_EXSH(A64EX_UXTW, 3), dest, base, idx);
 790}
 791
 792/* Inlined hash lookup. Specialized for key type and for const keys.
 793** The equivalent C code is:
 794**   Node *n = hashkey(t, key);
 795**   do {
 796**     if (lj_obj_equal(&n->key, key)) return &n->val;
 797**   } while ((n = nextnode(n)));
 798**   return niltv(L);
 799*/
 800static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 801{
 802  RegSet allow = RSET_GPR;
 803  int destused = ra_used(ir);
 804  Reg dest = ra_dest(as, ir, allow);
 805  Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
 806  Reg tmp = RID_TMP, type = RID_NONE, key = RID_NONE, tkey;
 807  IRRef refkey = ir->op2;
 808  IRIns *irkey = IR(refkey);
 809  int isk = irref_isk(refkey);
 810  IRType1 kt = irkey->t;
 811  uint32_t k = 0;
 812  uint32_t khash;
 813  MCLabel l_end, l_loop;
 814  rset_clear(allow, tab);
 815
 816  /* Allocate register for tkey outside of the loop. */
 817  if (isk) {
 818    int64_t kk;
 819    if (irt_isaddr(kt)) {
 820      kk = ((int64_t)irt_toitype(kt) << 47) | irkey[1].tv.u64;
 821    } else if (irt_isnum(kt)) {
 822      kk = (int64_t)ir_knum(irkey)->u64;
 823      /* Assumes -0.0 is already canonicalized to +0.0. */
 824    } else {
 825      lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type");
 826      kk = ~((int64_t)~irt_toitype(kt) << 47);
 827    }
 828    k = emit_isk12(kk);
 829    tkey = k ? 0 : ra_allock(as, kk, allow);
 830  } else {
 831    tkey = ra_scratch(as, allow);
 832  }
 833
 834  /* Key not found in chain: jump to exit (if merged) or load niltv. */
 835  l_end = emit_label(as);
 836  as->invmcp = NULL;
 837  if (merge == IR_NE) {
 838    asm_guardcc(as, CC_AL);
 839  } else if (destused) {
 840    uint32_t k12 = emit_isk12(offsetof(global_State, nilnode.val));
 841    lj_assertA(k12 != 0, "Cannot k12 encode niltv(L)");
 842    emit_dn(as, A64I_ADDx^k12, dest, RID_GL);
 843  }
 844
 845  /* Follow hash chain until the end. */
 846  l_loop = --as->mcp;
 847  if (destused)
 848    emit_lso(as, A64I_LDRx, dest, dest, offsetof(Node, next));
 849
 850  /* Type and value comparison. */
 851  if (merge == IR_EQ)
 852    asm_guardcc(as, CC_EQ);
 853  else
 854    emit_cond_branch(as, CC_EQ, l_end);
 855  emit_nm(as, A64I_CMPx^k, tmp, tkey);
 856  if (!destused)
 857    emit_lso(as, A64I_LDRx, dest, dest, offsetof(Node, next));
 858  emit_lso(as, A64I_LDRx, tmp, dest, offsetof(Node, key));
 859  *l_loop = A64I_X | A64I_CBNZ | A64F_S19(as->mcp - l_loop) | dest;
 860
 861  /* Construct tkey as canonicalized or tagged key. */
 862  if (!isk) {
 863    if (irt_isnum(kt)) {
 864      key = ra_alloc1(as, refkey, RSET_FPR);
 865      emit_dnm(as, A64I_CSELx | A64F_CC(CC_EQ), tkey, RID_ZERO, tkey);
 866      /* A64I_FMOV_R_D from key to tkey done below. */
 867    } else {
 868      lj_assertA(irt_isaddr(kt), "bad HREF key type");
 869      key = ra_alloc1(as, refkey, allow);
 870      type = ra_allock(as, irt_toitype(kt) << 15, rset_clear(allow, key));
 871      emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 32), tkey, key, type);
 872    }
 873  }
 874
 875  /* Load main position relative to tab->node into dest. */
 876  khash = isk ? ir_khash(as, irkey) : 1;
 877  if (khash == 0) {
 878    emit_lso(as, A64I_LDRx, dest, tab, offsetof(GCtab, node));
 879  } else {
 880    emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 3), dest, tmp, dest);
 881    emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 1), dest, dest, dest);
 882    emit_lso(as, A64I_LDRx, tmp, tab, offsetof(GCtab, node));
 883    if (isk) {
 884      Reg tmphash = ra_allock(as, khash, allow);
 885      emit_dnm(as, A64I_ANDw, dest, dest, tmphash);
 886      emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask));
 887    } else if (irt_isstr(kt)) {
 888      emit_dnm(as, A64I_ANDw, dest, dest, tmp);
 889      emit_lso(as, A64I_LDRw, tmp, key, offsetof(GCstr, sid));
 890      emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask));
 891    } else {  /* Must match with hash*() in lj_tab.c. */
 892      emit_dnm(as, A64I_ANDw, dest, dest, tmp);
 893      emit_lso(as, A64I_LDRw, tmp, tab, offsetof(GCtab, hmask));
 894      emit_dnm(as, A64I_SUBw, dest, dest, tmp);
 895      emit_dnm(as, A64I_EXTRw | (A64F_IMMS(32-HASH_ROT3)), tmp, tmp, tmp);
 896      emit_dnm(as, A64I_EORw | A64F_SH(A64SH_ROR, 32-HASH_ROT2), dest, tmp, dest);
 897      emit_dnm(as, A64I_SUBw, tmp, tmp, dest);
 898      emit_dnm(as, A64I_EXTRw | (A64F_IMMS(32-HASH_ROT1)), dest, dest, dest);
 899      if (irt_isnum(kt)) {
 900	emit_dnm(as, A64I_EORw, tmp, tkey, dest);
 901	emit_dnm(as, A64I_ADDw, dest, dest, dest);
 902	emit_dn(as, A64I_LSRx | A64F_IMMR(32)|A64F_IMMS(32), dest, tkey);
 903	emit_nm(as, A64I_FCMPZd, (key & 31), 0);
 904	emit_dn(as, A64I_FMOV_R_D, tkey, (key & 31));
 905      } else {
 906	emit_dnm(as, A64I_EORw, tmp, key, dest);
 907	emit_dnm(as, A64I_EORx | A64F_SH(A64SH_LSR, 32), dest, type, key);
 908      }
 909    }
 910  }
 911}
 912
 913static void asm_hrefk(ASMState *as, IRIns *ir)
 914{
 915  IRIns *kslot = IR(ir->op2);
 916  IRIns *irkey = IR(kslot->op1);
 917  int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
 918  int32_t kofs = ofs + (int32_t)offsetof(Node, key);
 919  int bigofs = !emit_checkofs(A64I_LDRx, kofs);
 920  Reg dest = (ra_used(ir) || bigofs) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
 921  Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
 922  Reg idx = node;
 923  RegSet allow = rset_exclude(RSET_GPR, node);
 924  uint64_t k;
 925  lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
 926  if (bigofs) {
 927    idx = dest;
 928    rset_clear(allow, dest);
 929    kofs = (int32_t)offsetof(Node, key);
 930  } else if (ra_hasreg(dest)) {
 931    emit_opk(as, A64I_ADDx, dest, node, ofs, allow);
 932  }
 933  asm_guardcc(as, CC_NE);
 934  if (irt_ispri(irkey->t)) {
 935    k = ~((int64_t)~irt_toitype(irkey->t) << 47);
 936  } else if (irt_isnum(irkey->t)) {
 937    k = ir_knum(irkey)->u64;
 938  } else {
 939    k = ((uint64_t)irt_toitype(irkey->t) << 47) | (uint64_t)ir_kgc(irkey);
 940  }
 941  emit_nm(as, A64I_CMPx, RID_TMP, ra_allock(as, k, allow));
 942  emit_lso(as, A64I_LDRx, RID_TMP, idx, kofs);
 943  if (bigofs)
 944    emit_opk(as, A64I_ADDx, dest, node, ofs, rset_exclude(RSET_GPR, node));
 945}
 946
 947static void asm_uref(ASMState *as, IRIns *ir)
 948{
 949  Reg dest = ra_dest(as, ir, RSET_GPR);
 950  int guarded = (irt_t(ir->t) & (IRT_GUARD|IRT_TYPE)) == (IRT_GUARD|IRT_PGC);
 951  if (irref_isk(ir->op1) && !guarded) {
 952    GCfunc *fn = ir_kfunc(IR(ir->op1));
 953    MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
 954    emit_lsptr(as, A64I_LDRx, dest, v);
 955  } else {
 956    if (guarded)
 957      asm_guardcnb(as, ir->o == IR_UREFC ? A64I_CBZ : A64I_CBNZ, RID_TMP);
 958    if (ir->o == IR_UREFC)
 959      emit_opk(as, A64I_ADDx, dest, dest,
 960	       (int32_t)offsetof(GCupval, tv), RSET_GPR);
 961    else
 962      emit_lso(as, A64I_LDRx, dest, dest, (int32_t)offsetof(GCupval, v));
 963    if (guarded)
 964      emit_lso(as, A64I_LDRB, RID_TMP, dest,
 965	       (int32_t)offsetof(GCupval, closed));
 966    if (irref_isk(ir->op1)) {
 967      GCfunc *fn = ir_kfunc(IR(ir->op1));
 968      uint64_t k = gcrefu(fn->l.uvptr[(ir->op2 >> 8)]);
 969      emit_loadu64(as, dest, k);
 970    } else {
 971      emit_lso(as, A64I_LDRx, dest, ra_alloc1(as, ir->op1, RSET_GPR),
 972	       (int32_t)offsetof(GCfuncL, uvptr) + 8*(int32_t)(ir->op2 >> 8));
 973    }
 974  }
 975}
 976
 977static void asm_fref(ASMState *as, IRIns *ir)
 978{
 979  UNUSED(as); UNUSED(ir);
 980  lj_assertA(!ra_used(ir), "unfused FREF");
 981}
 982
 983static void asm_strref(ASMState *as, IRIns *ir)
 984{
 985  RegSet allow = RSET_GPR;
 986  Reg dest = ra_dest(as, ir, allow);
 987  Reg base = ra_alloc1(as, ir->op1, allow);
 988  IRIns *irr = IR(ir->op2);
 989  int32_t ofs = sizeof(GCstr);
 990  uint32_t m;
 991  rset_clear(allow, base);
 992  if (irref_isk(ir->op2) && (m = emit_isk12(ofs + irr->i))) {
 993    emit_dn(as, A64I_ADDx^m, dest, base);
 994  } else {
 995    emit_dn(as, (A64I_ADDx^A64I_K12) | A64F_U12(ofs), dest, dest);
 996    emit_dnm(as, A64I_ADDx, dest, base, ra_alloc1(as, ir->op2, allow));
 997  }
 998}
 999
1000/* -- Loads and stores ---------------------------------------------------- */
1001
1002static A64Ins asm_fxloadins(IRIns *ir)
1003{
1004  switch (irt_type(ir->t)) {
1005  case IRT_I8: return A64I_LDRB ^ A64I_LS_S;
1006  case IRT_U8: return A64I_LDRB;
1007  case IRT_I16: return A64I_LDRH ^ A64I_LS_S;
1008  case IRT_U16: return A64I_LDRH;
1009  case IRT_NUM: return A64I_LDRd;
1010  case IRT_FLOAT: return A64I_LDRs;
1011  default: return irt_is64(ir->t) ? A64I_LDRx : A64I_LDRw;
1012  }
1013}
1014
1015static A64Ins asm_fxstoreins(IRIns *ir)
1016{
1017  switch (irt_type(ir->t)) {
1018  case IRT_I8: case IRT_U8: return A64I_STRB;
1019  case IRT_I16: case IRT_U16: return A64I_STRH;
1020  case IRT_NUM: return A64I_STRd;
1021  case IRT_FLOAT: return A64I_STRs;
1022  default: return irt_is64(ir->t) ? A64I_STRx : A64I_STRw;
1023  }
1024}
1025
1026static void asm_fload(ASMState *as, IRIns *ir)
1027{
1028  Reg dest = ra_dest(as, ir, RSET_GPR);
1029  Reg idx;
1030  A64Ins ai = asm_fxloadins(ir);
1031  int32_t ofs;
1032  if (ir->op1 == REF_NIL) {  /* FLOAD from GG_State with offset. */
1033    idx = RID_GL;
1034    ofs = (ir->op2 << 2) - GG_OFS(g);
1035  } else {
1036    idx = ra_alloc1(as, ir->op1, RSET_GPR);
1037    if (ir->op2 == IRFL_TAB_ARRAY) {
1038      ofs = asm_fuseabase(as, ir->op1);
1039      if (ofs) {  /* Turn the t->array load into an add for colocated arrays. */
1040	emit_dn(as, (A64I_ADDx^A64I_K12) | A64F_U12(ofs), dest, idx);
1041	return;
1042      }
1043    }
1044    ofs = field_ofs[ir->op2];
1045  }
1046  emit_lso(as, ai, (dest & 31), idx, ofs);
1047}
1048
1049static void asm_fstore(ASMState *as, IRIns *ir)
1050{
1051  if (ir->r != RID_SINK) {
1052    Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
1053    IRIns *irf = IR(ir->op1);
1054    Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
1055    int32_t ofs = field_ofs[irf->op2];
1056    emit_lso(as, asm_fxstoreins(ir), (src & 31), idx, ofs);
1057  }
1058}
1059
1060static void asm_xload(ASMState *as, IRIns *ir)
1061{
1062  Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
1063  lj_assertA(LJ_TARGET_UNALIGNED || !(ir->op2 & IRXLOAD_UNALIGNED),
1064	     "unaligned XLOAD");
1065  asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR);
1066}
1067
1068static void asm_xstore(ASMState *as, IRIns *ir)
1069{
1070  if (ir->r != RID_SINK) {
1071    Reg src = ra_alloc1(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
1072    asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
1073		 rset_exclude(RSET_GPR, src));
1074  }
1075}
1076
1077static void asm_ahuvload(ASMState *as, IRIns *ir)
1078{
1079  Reg idx, tmp;
1080  int32_t ofs = 0;
1081  RegSet gpr = RSET_GPR, allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
1082  lj_assertA(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) ||
1083	     irt_isint(ir->t),
1084	     "bad load type %d", irt_type(ir->t));
1085  if (ra_used(ir)) {
1086    Reg dest = ra_dest(as, ir, allow);
1087    tmp = irt_isnum(ir->t) ? ra_scratch(as, rset_clear(gpr, dest)) : dest;
1088    if (irt_isaddr(ir->t)) {
1089      emit_dn(as, A64I_ANDx^emit_isk13(LJ_GCVMASK, 1), dest, dest);
1090    } else if (irt_isnum(ir->t)) {
1091      emit_dn(as, A64I_FMOV_D_R, (dest & 31), tmp);
1092    } else if (irt_isint(ir->t)) {
1093      emit_dm(as, A64I_MOVw, dest, dest);
1094    }
1095  } else {
1096    tmp = ra_scratch(as, gpr);
1097  }
1098  idx = asm_fuseahuref(as, ir->op1, &ofs, rset_clear(gpr, tmp), A64I_LDRx);
1099  rset_clear(gpr, idx);
1100  if (ofs & FUSE_REG) rset_clear(gpr, ofs & 31);
1101  if (ir->o == IR_VLOAD) ofs += 8 * ir->op2;
1102  /* Always do the type check, even if the load result is unused. */
1103  asm_guardcc(as, irt_isnum(ir->t) ? CC_LS : CC_NE);
1104  if (irt_type(ir->t) >= IRT_NUM) {
1105    lj_assertA(irt_isinteger(ir->t) || irt_isnum(ir->t),
1106	       "bad load type %d", irt_type(ir->t));
1107    emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32),
1108	    ra_allock(as, LJ_TISNUM << 15, gpr), tmp);
1109  } else if (irt_isaddr(ir->t)) {
1110    emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(-irt_toitype(ir->t)), RID_TMP);
1111    emit_dn(as, A64I_ASRx | A64F_IMMR(47), RID_TMP, tmp);
1112  } else if (irt_isnil(ir->t)) {
1113    emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(1), tmp);
1114  } else {
1115    emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32),
1116	    ra_allock(as, (irt_toitype(ir->t) << 15) | 0x7fff, gpr), tmp);
1117  }
1118  if (ofs & FUSE_REG)
1119    emit_dnm(as, (A64I_LDRx^A64I_LS_R)|A64I_LS_UXTWx|A64I_LS_SH, tmp, idx, (ofs & 31));
1120  else
1121    emit_lso(as, A64I_LDRx, tmp, idx, ofs);
1122}
1123
1124static void asm_ahustore(ASMState *as, IRIns *ir)
1125{
1126  if (ir->r != RID_SINK) {
1127    RegSet allow = RSET_GPR;
1128    Reg idx, src = RID_NONE, tmp = RID_TMP, type = RID_NONE;
1129    int32_t ofs = 0;
1130    if (irt_isnum(ir->t)) {
1131      src = ra_alloc1(as, ir->op2, RSET_FPR);
1132      idx = asm_fuseahuref(as, ir->op1, &ofs, allow, A64I_STRd);
1133      if (ofs & FUSE_REG)
1134	emit_dnm(as, (A64I_STRd^A64I_LS_R)|A64I_LS_UXTWx|A64I_LS_SH, (src & 31), idx, (ofs &31));
1135      else
1136	emit_lso(as, A64I_STRd, (src & 31), idx, ofs);
1137    } else {
1138      if (!irt_ispri(ir->t)) {
1139	src = ra_alloc1(as, ir->op2, allow);
1140	rset_clear(allow, src);
1141	if (irt_isinteger(ir->t))
1142	  type = ra_allock(as, (uint64_t)(int32_t)LJ_TISNUM << 47, allow);
1143	else
1144	  type = ra_allock(as, irt_toitype(ir->t), allow);
1145      } else {
1146	tmp = type = ra_allock(as, ~((int64_t)~irt_toitype(ir->t)<<47), allow);
1147      }
1148      idx = asm_fuseahuref(as, ir->op1, &ofs, rset_exclude(allow, type),
1149			   A64I_STRx);
1150      if (ofs & FUSE_REG)
1151	emit_dnm(as, (A64I_STRx^A64I_LS_R)|A64I_LS_UXTWx|A64I_LS_SH, tmp, idx, (ofs & 31));
1152      else
1153	emit_lso(as, A64I_STRx, tmp, idx, ofs);
1154      if (ra_hasreg(src)) {
1155	if (irt_isinteger(ir->t)) {
1156	  emit_dnm(as, A64I_ADDx | A64F_EX(A64EX_UXTW), tmp, type, src);
1157	} else {
1158	  emit_dnm(as, A64I_ADDx | A64F_SH(A64SH_LSL, 47), tmp, src, type);
1159	}
1160      }
1161    }
1162  }
1163}
1164
1165static void asm_sload(ASMState *as, IRIns *ir)
1166{
1167  int32_t ofs = 8*((int32_t)ir->op1-2);
1168  IRType1 t = ir->t;
1169  Reg dest = RID_NONE, base;
1170  RegSet allow = RSET_GPR;
1171  lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1172	     "bad parent SLOAD");  /* Handled by asm_head_side(). */
1173  lj_assertA(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1174	     "inconsistent SLOAD variant");
1175  if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1176    dest = ra_scratch(as, RSET_FPR);
1177    asm_tointg(as, ir, dest);
1178    t.irt = IRT_NUM;  /* Continue with a regular number type check. */
1179  } else if (ra_used(ir)) {
1180    Reg tmp = RID_NONE;
1181    if ((ir->op2 & IRSLOAD_CONVERT))
1182      tmp = ra_scratch(as, irt_isint(t) ? RSET_FPR : RSET_GPR);
1183    lj_assertA((irt_isnum(t)) || irt_isint(t) || irt_isaddr(t),
1184	       "bad SLOAD type %d", irt_type(t));
1185    dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : allow);
1186    base = ra_alloc1(as, REF_BASE, rset_clear(allow, dest));
1187    if (irt_isaddr(t)) {
1188      emit_dn(as, A64I_ANDx^emit_isk13(LJ_GCVMASK, 1), dest, dest);
1189    } else if ((ir->op2 & IRSLOAD_CONVERT)) {
1190      if (irt_isint(t)) {
1191	emit_dn(as, A64I_FCVT_S32_F64, dest, (tmp & 31));
1192	/* If value is already loaded for type check, move it to FPR. */
1193	if ((ir->op2 & IRSLOAD_TYPECHECK))
1194	  emit_dn(as, A64I_FMOV_D_R, (tmp & 31), dest);
1195	else
1196	  dest = tmp;
1197	t.irt = IRT_NUM;  /* Check for original type. */
1198      } else {
1199	emit_dn(as, A64I_FCVT_F64_S32, (dest & 31), tmp);
1200	dest = tmp;
1201	t.irt = IRT_INT;  /* Check for original type. */
1202      }
1203    } else if (irt_isint(t) && (ir->op2 & IRSLOAD_TYPECHECK)) {
1204      emit_dm(as, A64I_MOVw, dest, dest);
1205    }
1206    goto dotypecheck;
1207  }
1208  base = ra_alloc1(as, REF_BASE, allow);
1209dotypecheck:
1210  rset_clear(allow, base);
1211  if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1212    Reg tmp;
1213    if (ra_hasreg(dest) && rset_test(RSET_GPR, dest)) {
1214      tmp = dest;
1215    } else {
1216      tmp = ra_scratch(as, allow);
1217      rset_clear(allow, tmp);
1218    }
1219    if (ra_hasreg(dest) && tmp != dest)
1220      emit_dn(as, A64I_FMOV_D_R, (dest & 31), tmp);
1221    /* Need type check, even if the load result is unused. */
1222    asm_guardcc(as, irt_isnum(t) ? CC_LS : CC_NE);
1223    if (irt_type(t) >= IRT_NUM) {
1224      lj_assertA(irt_isinteger(t) || irt_isnum(t),
1225		 "bad SLOAD type %d", irt_type(t));
1226      emit_nm(as, A64I_CMPx | A64F_SH(A64SH_LSR, 32),
1227	      ra_allock(as, (ir->op2 & IRSLOAD_KEYINDEX) ? LJ_KEYINDEX : (LJ_TISNUM << 15), allow), tmp);
1228    } else if (irt_isnil(t)) {
1229      emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(1), tmp);
1230    } else if (irt_ispri(t)) {
1231      emit_nm(as, A64I_CMPx,
1232	      ra_allock(as, ~((int64_t)~irt_toitype(t) << 47) , allow), tmp);
1233    } else {
1234      emit_n(as, (A64I_CMNx^A64I_K12) | A64F_U12(-irt_toitype(t)), RID_TMP);
1235      emit_dn(as, A64I_ASRx | A64F_IMMR(47), RID_TMP, tmp);
1236    }
1237    emit_lso(as, A64I_LDRx, tmp, base, ofs);
1238    return;
1239  }
1240  if (ra_hasreg(dest)) {
1241    emit_lso(as, irt_isnum(t) ? A64I_LDRd :
1242	     (irt_isint(t) ? A64I_LDRw : A64I_LDRx), (dest & 31), base,
1243	     ofs ^ ((LJ_BE && irt_isint(t) ? 4 : 0)));
1244  }
1245}
1246
1247/* -- Allocations --------------------------------------------------------- */
1248
1249#if LJ_HASFFI
1250static void asm_cnew(ASMState *as, IRIns *ir)
1251{
1252  CTState *cts = ctype_ctsG(J2G(as->J));
1253  CTypeID id = (CTypeID)IR(ir->op1)->i;
1254  CTSize sz;
1255  CTInfo info = lj_ctype_info(cts, id, &sz);
1256  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1257  IRRef args[4];
1258  RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1259  lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1260	     "bad CNEW/CNEWI operands");
1261
1262  as->gcsteps++;
1263  asm_setupresult(as, ir, ci);  /* GCcdata * */
1264  /* Initialize immutable cdata object. */
1265  if (ir->o == IR_CNEWI) {
1266    int32_t ofs = sizeof(GCcdata);
1267    Reg r = ra_alloc1(as, ir->op2, allow);
1268    lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1269    emit_lso(as, sz == 8 ? A64I_STRx : A64I_STRw, r, RID_RET, ofs);
1270  } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
1271    ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1272    args[0] = ASMREF_L;     /* lua_State *L */
1273    args[1] = ir->op1;      /* CTypeID id   */
1274    args[2] = ir->op2;      /* CTSize sz    */
1275    args[3] = ASMREF_TMP1;  /* CTSize align */
1276    asm_gencall(as, ci, args);
1277    emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1278    return;
1279  }
1280
1281  /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1282  {
1283    Reg r = (id < 65536) ? RID_X1 : ra_allock(as, id, allow);
1284    emit_lso(as, A64I_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct));
1285    emit_lso(as, A64I_STRH, r, RID_RET, offsetof(GCcdata, ctypeid));
1286    emit_d(as, A64I_MOVZw | A64F_U16(~LJ_TCDATA), RID_TMP);
1287    if (id < 65536) emit_d(as, A64I_MOVZw | A64F_U16(id), RID_X1);
1288  }
1289  args[0] = ASMREF_L;     /* lua_State *L */
1290  args[1] = ASMREF_TMP1;  /* MSize size   */
1291  asm_gencall(as, ci, args);
1292  ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1293	       ra_releasetmp(as, ASMREF_TMP1));
1294}
1295#endif
1296
1297/* -- Write barriers ------------------------------------------------------ */
1298
1299static void asm_tbar(ASMState *as, IRIns *ir)
1300{
1301  Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1302  Reg link = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1303  Reg mark = RID_TMP;
1304  MCLabel l_end = emit_label(as);
1305  emit_lso(as, A64I_STRB, mark, tab, (int32_t)offsetof(GCtab, marked));
1306  /* Keep STRx in the middle to avoid LDP/STP fusion with surrounding code. */
1307  emit_lso(as, A64I_STRx, link, tab, (int32_t)offsetof(GCtab, gclist));
1308  emit_setgl(as, tab, gc.grayagain);
1309  emit_dn(as, A64I_ANDw^emit_isk13(~LJ_GC_BLACK, 0), mark, mark);
1310  emit_getgl(as, link, gc.grayagain);
1311  emit_cond_branch(as, CC_EQ, l_end);
1312  emit_n(as, A64I_TSTw^emit_isk13(LJ_GC_BLACK, 0), mark);
1313  emit_lso(as, A64I_LDRB, mark, tab, (int32_t)offsetof(GCtab, marked));
1314}
1315
1316static void asm_obar(ASMState *as, IRIns *ir)
1317{
1318  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1319  IRRef args[2];
1320  MCLabel l_end;
1321  Reg obj, val, tmp;
1322  /* No need for other object barriers (yet). */
1323  lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1324  ra_evictset(as, RSET_SCRATCH);
1325  l_end = emit_label(as);
1326  args[0] = ASMREF_TMP1;  /* global_State *g */
1327  args[1] = ir->op1;      /* TValue *tv      */
1328  asm_gencall(as, ci, args);
1329  emit_dm(as, A64I_MOVx, ra_releasetmp(as, ASMREF_TMP1), RID_GL);
1330  obj = IR(ir->op1)->r;
1331  tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1332  emit_tnb(as, A64I_TBZ, tmp, lj_ffs(LJ_GC_BLACK), l_end);
1333  emit_cond_branch(as, CC_EQ, l_end);
1334  emit_n(as, A64I_TSTw^emit_isk13(LJ_GC_WHITES, 0), RID_TMP);
1335  val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1336  emit_lso(as, A64I_LDRB, tmp, obj,
1337	   (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1338  emit_lso(as, A64I_LDRB, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1339}
1340
1341/* -- Arithmetic and logic operations ------------------------------------- */
1342
1343static void asm_fparith(ASMState *as, IRIns *ir, A64Ins ai)
1344{
1345  Reg dest = ra_dest(as, ir, RSET_FPR);
1346  Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1347  right = (left >> 8); left &= 255;
1348  emit_dnm(as, ai, (dest & 31), (left & 31), (right & 31));
1349}
1350
1351static void asm_fpunary(ASMState *as, IRIns *ir, A64Ins ai)
1352{
1353  Reg dest = ra_dest(as, ir, RSET_FPR);
1354  Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1355  emit_dn(as, ai, (dest & 31), (left & 31));
1356}
1357
1358static void asm_fpmath(ASMState *as, IRIns *ir)
1359{
1360  IRFPMathOp fpm = (IRFPMathOp)ir->op2;
1361  if (fpm == IRFPM_SQRT) {
1362    asm_fpunary(as, ir, A64I_FSQRTd);
1363  } else if (fpm <= IRFPM_TRUNC) {
1364    asm_fpunary(as, ir, fpm == IRFPM_FLOOR ? A64I_FRINTMd :
1365			fpm == IRFPM_CEIL ? A64I_FRINTPd : A64I_FRINTZd);
1366  } else {
1367    asm_callid(as, ir, IRCALL_lj_vm_floor + fpm);
1368  }
1369}
1370
1371static int asm_swapops(ASMState *as, IRRef lref, IRRef rref)
1372{
1373  IRIns *ir;
1374  if (irref_isk(rref))
1375    return 0;  /* Don't swap constants to the left. */
1376  if (irref_isk(lref))
1377    return 1;  /* But swap constants to the right. */
1378  ir = IR(rref);
1379  if ((ir->o >= IR_BSHL && ir->o <= IR_BROR) ||
1380      (ir->o == IR_ADD && ir->op1 == ir->op2) ||
1381      (ir->o == IR_CONV && ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)))
1382    return 0;  /* Don't swap fusable operands to the left. */
1383  ir = IR(lref);
1384  if ((ir->o >= IR_BSHL && ir->o <= IR_BROR) ||
1385      (ir->o == IR_ADD && ir->op1 == ir->op2) ||
1386      (ir->o == IR_CONV && ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)))
1387    return 1;  /* But swap fusable operands to the right. */
1388  return 0;  /* Otherwise don't swap. */
1389}
1390
1391static void asm_intop(ASMState *as, IRIns *ir, A64Ins ai)
1392{
1393  IRRef lref = ir->op1, rref = ir->op2;
1394  Reg left, dest = ra_dest(as, ir, RSET_GPR);
1395  uint32_t m;
1396  if ((ai & ~A64I_S) != A64I_SUBw && asm_swapops(as, lref, rref)) {
1397    IRRef tmp = lref; lref = rref; rref = tmp;
1398  }
1399  left = ra_hintalloc(as, lref, dest, RSET_GPR);
1400  if (irt_is64(ir->t)) ai |= A64I_X;
1401  m = asm_fuseopm(as, ai, rref, rset_exclude(RSET_GPR, left));
1402  if (irt_isguard(ir->t)) {  /* For IR_ADDOV etc. */
1403    asm_guardcc(as, CC_VS);
1404    ai |= A64I_S;
1405  }
1406  emit_dn(as, ai^m, dest, left);
1407}
1408
1409static void asm_intop_s(ASMState *as, IRIns *ir, A64Ins ai)
1410{
1411  if (as->flagmcp == as->mcp) {  /* Drop cmp r, #0. */
1412    as->flagmcp = NULL;
1413    as->mcp++;
1414    ai |= A64I_S;
1415  }
1416  asm_intop(as, ir, ai);
1417}
1418
1419static void asm_intneg(ASMState *as, IRIns *ir)
1420{
1421  Reg dest = ra_dest(as, ir, RSET_GPR);
1422  Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1423  emit_dm(as, irt_is64(ir->t) ? A64I_NEGx : A64I_NEGw, dest, left);
1424}
1425
1426/* NYI: use add/shift for MUL(OV) with constants. FOLD only does 2^k. */
1427static void asm_intmul(ASMState *as, IRIns *ir)
1428{
1429  Reg dest = ra_dest(as, ir, RSET_GPR);
1430  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1431  Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1432  if (irt_isguard(ir->t)) {  /* IR_MULOV */
1433    asm_guardcc(as, CC_NE);
1434    emit_dm(as, A64I_MOVw, dest, dest);  /* Zero-extend. */
1435    emit_nm(as, A64I_CMPx | A64F_EX(A64EX_SXTW), dest, dest);
1436    emit_dnm(as, A64I_SMULL, dest, right, left);
1437  } else {
1438    emit_dnm(as, irt_is64(ir->t) ? A64I_MULx : A64I_MULw, dest, left, right);
1439  }
1440}
1441
1442static void asm_add(ASMState *as, IRIns *ir)
1443{
1444  if (irt_isnum(ir->t)) {
1445    if (!asm_fusemadd(as, ir, A64I_FMADDd, A64I_FMADDd))
1446      asm_fparith(as, ir, A64I_FADDd);
1447    return;
1448  }
1449  asm_intop_s(as, ir, A64I_ADDw);
1450}
1451
1452static void asm_sub(ASMState *as, IRIns *ir)
1453{
1454  if (irt_isnum(ir->t)) {
1455    if (!asm_fusemadd(as, ir, A64I_FNMSUBd, A64I_FMSUBd))
1456      asm_fparith(as, ir, A64I_FSUBd);
1457    return;
1458  }
1459  asm_intop_s(as, ir, A64I_SUBw);
1460}
1461
1462static void asm_mul(ASMState *as, IRIns *ir)
1463{
1464  if (irt_isnum(ir->t)) {
1465    asm_fparith(as, ir, A64I_FMULd);
1466    return;
1467  }
1468  asm_intmul(as, ir);
1469}
1470
1471#define asm_addov(as, ir)	asm_add(as, ir)
1472#define asm_subov(as, ir)	asm_sub(as, ir)
1473#define asm_mulov(as, ir)	asm_mul(as, ir)
1474
1475#define asm_fpdiv(as, ir)	asm_fparith(as, ir, A64I_FDIVd)
1476#define asm_abs(as, ir)		asm_fpunary(as, ir, A64I_FABS)
1477
1478static void asm_neg(ASMState *as, IRIns *ir)
1479{
1480  if (irt_isnum(ir->t)) {
1481    asm_fpunary(as, ir, A64I_FNEGd);
1482    return;
1483  }
1484  asm_intneg(as, ir);
1485}
1486
1487static void asm_band(ASMState *as, IRIns *ir)
1488{
1489  A64Ins ai = A64I_ANDw;
1490  if (asm_fuseandshift(as, ir))
1491    return;
1492  if (as->flagmcp == as->mcp) {
1493    /* Try to drop cmp r, #0. */
1494    as->flagmcp = NULL;
1495    as->mcp++;
1496    ai = A64I_ANDSw;
1497  }
1498  asm_intop(as, ir, ai);
1499}
1500
1501static void asm_borbxor(ASMState *as, IRIns *ir, A64Ins ai)
1502{
1503  IRRef lref = ir->op1, rref = ir->op2;
1504  IRIns *irl = IR(lref), *irr = IR(rref);
1505  if ((canfuse(as, irl) && irl->o == IR_BNOT && !irref_isk(rref)) ||
1506      (canfuse(as, irr) && irr->o == IR_BNOT && !irref_isk(lref))) {
1507    Reg left, dest = ra_dest(as, ir, RSET_GPR);
1508    uint32_t m;
1509    if (irl->o == IR_BNOT) {
1510      IRRef tmp = lref; lref = rref; rref = tmp;
1511    }
1512    left = ra_alloc1(as, lref, RSET_GPR);
1513    ai |= A64I_ON;
1514    if (irt_is64(ir->t)) ai |= A64I_X;
1515    m = asm_fuseopm(as, ai, IR(rref)->op1, rset_exclude(RSET_GPR, left));
1516    emit_dn(as, ai^m, dest, left);
1517  } else {
1518    asm_intop(as, ir, ai);
1519  }
1520}
1521
1522static void asm_bor(ASMState *as, IRIns *ir)
1523{
1524  if (asm_fuseorshift(as, ir))
1525    return;
1526  asm_borbxor(as, ir, A64I_ORRw);
1527}
1528
1529#define asm_bxor(as, ir)	asm_borbxor(as, ir, A64I_EORw)
1530
1531static void asm_bnot(ASMState *as, IRIns *ir)
1532{
1533  A64Ins ai = A64I_MVNw;
1534  Reg dest = ra_dest(as, ir, RSET_GPR);
1535  uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR);
1536  if (irt_is64(ir->t)) ai |= A64I_X;
1537  emit_d(as, ai^m, dest);
1538}
1539
1540static void asm_bswap(ASMState *as, IRIns *ir)
1541{
1542  Reg dest = ra_dest(as, ir, RSET_GPR);
1543  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1544  emit_dn(as, irt_is64(ir->t) ? A64I_REVx : A64I_REVw, dest, left);
1545}
1546
1547static void asm_bitshift(ASMState *as, IRIns *ir, A64Ins ai, A64Shift sh)
1548{
1549  int32_t shmask = irt_is64(ir->t) ? 63 : 31;
1550  if (irref_isk(ir->op2)) {  /* Constant shifts. */
1551    Reg left, dest = ra_dest(as, ir, RSET_GPR);
1552    int32_t shift = (IR(ir->op2)->i & shmask);
1553    IRIns *irl = IR(ir->op1);
1554    if (shmask == 63) ai += A64I_UBFMx - A64I_UBFMw;
1555
1556    /* Fuse BSHL + BSHR/BSAR into UBFM/SBFM aka UBFX/SBFX/UBFIZ/SBFIZ. */
1557    if ((sh == A64SH_LSR || sh == A64SH_ASR) && canfuse(as, irl)) {
1558      if (irl->o == IR_BSHL && irref_isk(irl->op2)) {
1559	int32_t shift2 = (IR(irl->op2)->i & shmask);
1560	shift = ((shift - shift2) & shmask);
1561	shmask -= shift2;
1562	ir = irl;
1563      }
1564    }
1565
1566    left = ra_alloc1(as, ir->op1, RSET_GPR);
1567    switch (sh) {
1568    case A64SH_LSL:
1569      emit_dn(as, ai | A64F_IMMS(shmask-shift) |
1570		  A64F_IMMR((shmask-shift+1)&shmask), dest, left);
1571      break;
1572    case A64SH_LSR: case A64SH_ASR:
1573      emit_dn(as, ai | A64F_IMMS(shmask) | A64F_IMMR(shift), dest, left);
1574      break;
1575    case A64SH_ROR:
1576      emit_dnm(as, ai | A64F_IMMS(shift), dest, left, left);
1577      break;
1578    }
1579  } else {  /* Variable-length shifts. */
1580    Reg dest = ra_dest(as, ir, RSET_GPR);
1581    Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1582    Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1583    emit_dnm(as, (shmask == 63 ? A64I_SHRx : A64I_SHRw) | A64F_BSH(sh), dest, left, right);
1584  }
1585}
1586
1587#define asm_bshl(as, ir)	asm_bitshift(as, ir, A64I_UBFMw, A64SH_LSL)
1588#define asm_bshr(as, ir)	asm_bitshift(as, ir, A64I_UBFMw, A64SH_LSR)
1589#define asm_bsar(as, ir)	asm_bitshift(as, ir, A64I_SBFMw, A64SH_ASR)
1590#define asm_bror(as, ir)	asm_bitshift(as, ir, A64I_EXTRw, A64SH_ROR)
1591#define asm_brol(as, ir)	lj_assertA(0, "unexpected BROL")
1592
1593static void asm_intmin_max(ASMState *as, IRIns *ir, A64CC cc)
1594{
1595  Reg dest = ra_dest(as, ir, RSET_GPR);
1596  Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1597  Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1598  emit_dnm(as, A64I_CSELw|A64F_CC(cc), dest, left, right);
1599  emit_nm(as, A64I_CMPw, left, right);
1600}
1601
1602static void asm_fpmin_max(ASMState *as, IRIns *ir, A64CC fcc)
1603{
1604  Reg dest = (ra_dest(as, ir, RSET_FPR) & 31);
1605  Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1606  right = ((left >> 8) & 31); left &= 31;
1607  emit_dnm(as, A64I_FCSELd | A64F_CC(fcc), dest, right, left);
1608  emit_nm(as, A64I_FCMPd, left, right);
1609}
1610
1611static void asm_min_max(ASMState *as, IRIns *ir, A64CC cc, A64CC fcc)
1612{
1613  if (irt_isnum(ir->t))
1614    asm_fpmin_max(as, ir, fcc);
1615  else
1616    asm_intmin_max(as, ir, cc);
1617}
1618
1619#define asm_min(as, ir)		asm_min_max(as, ir, CC_LT, CC_PL)
1620#define asm_max(as, ir)		asm_min_max(as, ir, CC_GT, CC_LE)
1621
1622/* -- Comparisons --------------------------------------------------------- */
1623
1624/* Map of comparisons to flags. ORDER IR. */
1625static const uint8_t asm_compmap[IR_ABC+1] = {
1626  /* op  FP swp  int cc   FP cc */
1627  /* LT       */ CC_GE + (CC_HS << 4),
1628  /* GE    x  */ CC_LT + (CC_HI << 4),
1629  /* LE       */ CC_GT + (CC_HI << 4),
1630  /* GT    x  */ CC_LE + (CC_HS << 4),
1631  /* ULT   x  */ CC_HS + (CC_LS << 4),
1632  /* UGE      */ CC_LO + (CC_LO << 4),
1633  /* ULE   x  */ CC_HI + (CC_LO << 4),
1634  /* UGT      */ CC_LS + (CC_LS << 4),
1635  /* EQ       */ CC_NE + (CC_NE << 4),
1636  /* NE       */ CC_EQ + (CC_EQ << 4),
1637  /* ABC      */ CC_LS + (CC_LS << 4)  /* Same as UGT. */
1638};
1639
1640/* FP comparisons. */
1641static void asm_fpcomp(ASMState *as, IRIns *ir)
1642{
1643  Reg left, right;
1644  A64Ins ai;
1645  int swp = ((ir->o ^ (ir->o >> 2)) & ~(ir->o >> 3) & 1);
1646  if (!swp && irref_isk(ir->op2) && ir_knum(IR(ir->op2))->u64 == 0) {
1647    left = (ra_alloc1(as, ir->op1, RSET_FPR) & 31);
1648    right = 0;
1649    ai = A64I_FCMPZd;
1650  } else {
1651    left = ra_alloc2(as, ir, RSET_FPR);
1652    if (swp) {
1653      right = (left & 31); left = ((left >> 8) & 31);
1654    } else {
1655      right = ((left >> 8) & 31); left &= 31;
1656    }
1657    ai = A64I_FCMPd;
1658  }
1659  asm_guardcc(as, (asm_compmap[ir->o] >> 4));
1660  emit_nm(as, ai, left, right);
1661}
1662
1663/* Integer comparisons. */
1664static void asm_intcomp(ASMState *as, IRIns *ir)
1665{
1666  A64CC oldcc, cc = (asm_compmap[ir->o] & 15);
1667  A64Ins ai = irt_is64(ir->t) ? A64I_CMPx : A64I_CMPw;
1668  IRRef lref = ir->op1, rref = ir->op2;
1669  Reg left;
1670  uint32_t m;
1671  int cmpprev0 = 0;
1672  lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) ||
1673	     irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t),
1674	     "bad comparison data type %d", irt_type(ir->t));
1675  if (asm_swapops(as, lref, rref)) {
1676    IRRef tmp = lref; lref = rref; rref = tmp;
1677    if (cc >= CC_GE) cc ^= 7;  /* LT <-> GT, LE <-> GE */
1678    else if (cc > CC_NE) cc ^= 11;  /* LO <-> HI, LS <-> HS */
1679  }
1680  oldcc = cc;
1681  if (irref_isk(rref) && get_k64val(as, rref) == 0) {
1682    IRIns *irl = IR(lref);
1683    if (cc == CC_GE) cc = CC_PL;
1684    else if (cc == CC_LT) cc = CC_MI;
1685    else if (cc > CC_NE) goto nocombine;  /* Other conds don't work with tst. */
1686    cmpprev0 = (irl+1 == ir);
1687    /* Combine and-cmp-bcc into tbz/tbnz or and-cmp into tst. */
1688    if (cmpprev0 && irl->o == IR_BAND && !ra_used(irl)) {
1689      IRRef blref = irl->op1, brref = irl->op2;
1690      uint32_t m2 = 0;
1691      Reg bleft;
1692      if (asm_swapops(as, blref, brref)) {
1693	Reg tmp = blref; blref = brref; brref = tmp;
1694      }
1695      bleft = ra_alloc1(as, blref, RSET_GPR);
1696      if (irref_isk(brref)) {
1697	uint64_t k = get_k64val(as, brref);
1698	if (k && !(k & (k-1)) && (cc == CC_EQ || cc == CC_NE) &&
1699	    asm_guardtnb(as, cc == CC_EQ ? A64I_TBZ : A64I_TBNZ, bleft,
1700			 emit_ctz64(k)))
1701	  return;
1702	m2 = emit_isk13(k, irt_is64(irl->t));
1703      }
1704      ai = (irt_is64(irl->t) ? A64I_TSTx : A64I_TSTw);
1705      if (!m2)
1706	m2 = asm_fuseopm(as, ai, brref, rset_exclude(RSET_GPR, bleft));
1707      asm_guardcc(as, cc);
1708      emit_n(as, ai^m2, bleft);
1709      return;
1710    }
1711    if (cc == CC_EQ || cc == CC_NE) {
1712      /* Combine cmp-bcc into cbz/cbnz. */
1713      ai = cc == CC_EQ ? A64I_CBZ : A64I_CBNZ;
1714      if (irt_is64(ir->t)) ai |= A64I_X;
1715      asm_guardcnb(as, ai, ra_alloc1(as, lref, RSET_GPR));
1716      return;
1717    }
1718  }
1719nocombine:
1720  left = ra_alloc1(as, lref, RSET_GPR);
1721  m = asm_fuseopm(as, ai, rref, rset_exclude(RSET_GPR, left));
1722  asm_guardcc(as, cc);
1723  emit_n(as, ai^m, left);
1724  /* Signed comparison with zero and referencing previous ins? */
1725  if (cmpprev0 && (oldcc <= CC_NE || oldcc >= CC_GE))
1726    as->flagmcp = as->mcp;  /* Allow elimination of the compare. */
1727}
1728
1729static void asm_comp(ASMState *as, IRIns *ir)
1730{
1731  if (irt_isnum(ir->t))
1732    asm_fpcomp(as, ir);
1733  else
1734    asm_intcomp(as, ir);
1735}
1736
1737#define asm_equal(as, ir)	asm_comp(as, ir)
1738
1739/* -- Split register ops -------------------------------------------------- */
1740
1741/* Hiword op of a split 64/64 bit op. Previous op is the loword op. */
1742static void asm_hiop(ASMState *as, IRIns *ir)
1743{
1744  /* HIOP is marked as a store because it needs its own DCE logic. */
1745  int uselo = ra_used(ir-1), usehi = ra_used(ir);  /* Loword/hiword used? */
1746  if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1747  if (!usehi) return;  /* Skip unused hiword op for all remaining ops. */
1748  switch ((ir-1)->o) {
1749  case IR_CALLN:
1750  case IR_CALLL:
1751  case IR_CALLS:
1752  case IR_CALLXS:
1753    if (!uselo)
1754      ra_allocref(as, ir->op1, RID2RSET(RID_RETLO));  /* Mark lo op as used. */
1755    break;
1756  default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
1757  }
1758}
1759
1760/* -- Profiling ----------------------------------------------------------- */
1761
1762static void asm_prof(ASMState *as, IRIns *ir)
1763{
1764  uint32_t k = emit_isk13(HOOK_PROFILE, 0);
1765  lj_assertA(k != 0, "HOOK_PROFILE does not fit in K13");
1766  UNUSED(ir);
1767  asm_guardcc(as, CC_NE);
1768  emit_n(as, A64I_TSTw^k, RID_TMP);
1769  emit_lsptr(as, A64I_LDRB, RID_TMP, (void *)&J2G(as->J)->hookmask);
1770}
1771
1772/* -- Stack handling ------------------------------------------------------ */
1773
1774/* Check Lua stack size for overflow. Use exit handler as fallback. */
1775static void asm_stack_check(ASMState *as, BCReg topslot,
1776			    IRIns *irp, RegSet allow, ExitNo exitno)
1777{
1778  uint32_t k;
1779  Reg pbase = RID_BASE;
1780  if (irp) {
1781    pbase = irp->r;
1782    if (!ra_hasreg(pbase))
1783      pbase = allow ? (0x40 | rset_pickbot(allow)) : (0xC0 | RID_RET);
1784  }
1785  emit_cond_branch(as, CC_LS, asm_exitstub_addr(as, exitno));
1786  if (pbase & 0x80)  /* Restore temp. register. */
1787    emit_lso(as, A64I_LDRx, (pbase & 31), RID_SP, 0);
1788  k = emit_isk12((8*topslot));
1789  lj_assertA(k, "slot offset %d does not fit in K12", 8*topslot);
1790  emit_n(as, A64I_CMPx^k, RID_TMP);
1791  emit_dnm(as, A64I_SUBx, RID_TMP, RID_TMP, (pbase & 31));
1792  emit_lso(as, A64I_LDRx, RID_TMP, RID_TMP,
1793	   (int32_t)offsetof(lua_State, maxstack));
1794  if (pbase & 0x40) {
1795    emit_getgl(as, (pbase & 31), jit_base);
1796    if (pbase & 0x80)  /* Save temp register. */
1797      emit_lso(as, A64I_STRx, (pbase & 31), RID_SP, 0);
1798  }
1799  emit_getgl(as, RID_TMP, cur_L);
1800}
1801
1802/* Restore Lua stack from on-trace state. */
1803static void asm_stack_restore(ASMState *as, SnapShot *snap)
1804{
1805  SnapEntry *map = &as->T->snapmap[snap->mapofs];
1806#ifdef LUA_USE_ASSERT
1807  SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
1808#endif
1809  MSize n, nent = snap->nent;
1810  /* Store the value of all modified slots to the Lua stack. */
1811  for (n = 0; n < nent; n++) {
1812    SnapEntry sn = map[n];
1813    BCReg s = snap_slot(sn);
1814    int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
1815    IRRef ref = snap_ref(sn);
1816    IRIns *ir = IR(ref);
1817    if ((sn & SNAP_NORESTORE))
1818      continue;
1819    if ((sn & SNAP_KEYINDEX)) {
1820      RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1821      Reg r = irref_isk(ref) ? ra_allock(as, ir->i, allow) :
1822			       ra_alloc1(as, ref, allow);
1823      rset_clear(allow, r);
1824      emit_lso(as, A64I_STRw, r, RID_BASE, ofs);
1825      emit_lso(as, A64I_STRw, ra_allock(as, LJ_KEYINDEX, allow), RID_BASE, ofs+4);
1826    } else if (irt_isnum(ir->t)) {
1827      Reg src = ra_alloc1(as, ref, RSET_FPR);
1828      emit_lso(as, A64I_STRd, (src & 31), RID_BASE, ofs);
1829    } else {
1830      asm_tvstore64(as, RID_BASE, ofs, ref);
1831    }
1832    checkmclim(as);
1833  }
1834  lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
1835}
1836
1837/* -- GC handling --------------------------------------------------------- */
1838
1839/* Marker to prevent patching the GC check exit. */
1840#define ARM64_NOPATCH_GC_CHECK \
1841  (A64I_ORRx|A64F_D(RID_ZERO)|A64F_M(RID_ZERO)|A64F_N(RID_ZERO))
1842
1843/* Check GC threshold and do one or more GC steps. */
1844static void asm_gc_check(ASMState *as)
1845{
1846  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
1847  IRRef args[2];
1848  MCLabel l_end;
1849  Reg tmp2;
1850  ra_evictset(as, RSET_SCRATCH);
1851  l_end = emit_label(as);
1852  /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1853  asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */
1854  *--as->mcp = ARM64_NOPATCH_GC_CHECK;
1855  args[0] = ASMREF_TMP1;  /* global_State *g */
1856  args[1] = ASMREF_TMP2;  /* MSize steps     */
1857  asm_gencall(as, ci, args);
1858  emit_dm(as, A64I_MOVx, ra_releasetmp(as, ASMREF_TMP1), RID_GL);
1859  tmp2 = ra_releasetmp(as, ASMREF_TMP2);
1860  emit_loadi(as, tmp2, as->gcsteps);
1861  /* Jump around GC step if GC total < GC threshold. */
1862  emit_cond_branch(as, CC_LS, l_end);
1863  emit_nm(as, A64I_CMPx, RID_TMP, tmp2);
1864  emit_getgl(as, tmp2, gc.threshold);
1865  emit_getgl(as, RID_TMP, gc.total);
1866  as->gcsteps = 0;
1867  checkmclim(as);
1868}
1869
1870/* -- Loop handling ------------------------------------------------------- */
1871
1872/* Fixup the loop branch. */
1873static void asm_loop_fixup(ASMState *as)
1874{
1875  MCode *p = as->mctop;
1876  MCode *target = as->mcp;
1877  if (as->loopinv) {  /* Inverted loop branch? */
1878    uint32_t mask = (p[-2] & 0x7e000000) == 0x36000000 ? 0x3fffu : 0x7ffffu;
1879    ptrdiff_t delta = target - (p - 2);
1880    /* asm_guard* already inverted the bcc/tnb/cnb and patched the final b. */
1881    p[-2] |= ((uint32_t)delta & mask) << 5;
1882  } else {
1883    ptrdiff_t delta = target - (p - 1);
1884    p[-1] = A64I_B | A64F_S26(delta);
1885  }
1886}
1887
1888/* Fixup the tail of the loop. */
1889static void asm_loop_tail_fixup(ASMState *as)
1890{
1891  UNUSED(as);  /* Nothing to do. */
1892}
1893
1894/* -- Head of trace ------------------------------------------------------- */
1895
1896/* Coalesce BASE register for a root trace. */
1897static void asm_head_root_base(ASMState *as)
1898{
1899  IRIns *ir = IR(REF_BASE);
1900  Reg r = ir->r;
1901  if (ra_hasreg(r)) {
1902    ra_free(as, r);
1903    if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1904      ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
1905    if (r != RID_BASE)
1906      emit_movrr(as, ir, r, RID_BASE);
1907  }
1908}
1909
1910/* Coalesce BASE register for a side trace. */
1911static Reg asm_head_side_base(ASMState *as, IRIns *irp)
1912{
1913  IRIns *ir = IR(REF_BASE);
1914  Reg r = ir->r;
1915  if (ra_hasreg(r)) {
1916    ra_free(as, r);
1917    if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1918      ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
1919    if (irp->r == r) {
1920      return r;  /* Same BASE register already coalesced. */
1921    } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
1922      /* Move from coalesced parent reg. */
1923      emit_movrr(as, ir, r, irp->r);
1924      return irp->r;
1925    } else {
1926      emit_getgl(as, r, jit_base);  /* Otherwise reload BASE. */
1927    }
1928  }
1929  return RID_NONE;
1930}
1931
1932/* -- Tail of trace ------------------------------------------------------- */
1933
1934/* Fixup the tail code. */
1935static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1936{
1937  MCode *mcp = as->mctail;
1938  MCode *target;
1939  /* Undo the sp adjustment in BC_JLOOP when exiting to the interpreter. */
1940  int32_t spadj = as->T->spadjust + (lnk ? 0 : sps_scale(SPS_FIXED));
1941  if (spadj) {  /* Emit stack adjustment. */
1942    uint32_t k = emit_isk12(spadj);
1943    lj_assertA(k, "stack adjustment %d does not fit in K12", spadj);
1944    *mcp++ = (A64I_ADDx^k) | A64F_D(RID_SP) | A64F_N(RID_SP);
1945  }
1946  /* Emit exit branch. */
1947  target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)(void *)lj_vm_exit_interp;
1948  if (lnk || A64F_S_OK(target - mcp, 26)) {
1949    *mcp = A64I_B | A64F_S26(target - mcp); mcp++;
1950  } else {
1951    *mcp++ = A64I_LDRx | A64F_D(RID_LR) | A64F_N(RID_GL) | A64F_U12(glofs(as, &as->J->k64[LJ_K64_VM_EXIT_INTERP]) >> 3);
1952    *mcp++ = A64I_BR_AUTH | A64F_N(RID_LR);
1953  }
1954  while (as->mctop > mcp) *--as->mctop = A64I_LE(A64I_NOP);
1955}
1956
1957/* Prepare tail of code. */
1958static void asm_tail_prep(ASMState *as, TraceNo lnk)
1959{
1960  MCode *p = as->mctop - 1;  /* Leave room for exit branch. */
1961  if (as->loopref) {
1962    as->invmcp = as->mcp = p;
1963  } else {
1964    if (!lnk) {
1965      MCode *target = (MCode *)(void *)lj_vm_exit_interp;
1966      if (!A64F_S_OK(target - p, 26) || !A64F_S_OK(target - (p+1), 26)) p--;
1967    }
1968    p--;  /* Leave room for stack pointer adjustment. */
1969    as->mcp = p;
1970    as->invmcp = NULL;
1971  }
1972  as->mctail = p;
1973  *p = 0;  /* Prevent load/store merging. */
1974}
1975
1976/* -- Trace setup --------------------------------------------------------- */
1977
1978/* Ensure there are enough stack slots for call arguments. */
1979static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
1980{
1981#if LJ_HASFFI
1982  uint32_t i, nargs = CCI_XNARGS(ci);
1983  if (nargs > (REGARG_NUMGPR < REGARG_NUMFPR ? REGARG_NUMGPR : REGARG_NUMFPR) ||
1984      (LJ_TARGET_OSX && (ci->flags & CCI_VARARG))) {
1985    IRRef args[CCI_NARGS_MAX*2];
1986    int ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
1987    int spofs = 0, spalign = LJ_TARGET_OSX ? 0 : 7, nslots;
1988    asm_collectargs(as, ir, ci, args);
1989#if LJ_ABI_WIN
1990    if ((ci->flags & CCI_VARARG)) nfpr = 0;
1991#endif
1992    for (i = 0; i < nargs; i++) {
1993      int al = spalign;
1994      if (!args[i]) {
1995#if LJ_TARGET_OSX
1996	/* Marker for start of varaargs. */
1997	nfpr = 0;
1998	ngpr = 0;
1999	spalign = 7;
2000#endif
2001      } else if (irt_isfp(IR(args[i])->t)) {
2002	if (nfpr > 0) { nfpr--; continue; }
2003#if LJ_ABI_WIN
2004	if ((ci->flags & CCI_VARARG) && ngpr > 0) { ngpr--; continue; }
2005#elif LJ_TARGET_OSX
2006	al |= irt_isnum(IR(args[i])->t) ? 7 : 3;
2007#endif
2008      } else {
2009	if (ngpr > 0) { ngpr--; continue; }
2010#if LJ_TARGET_OSX
2011	al |= irt_size(IR(args[i])->t) - 1;
2012#endif
2013      }
2014      spofs = (spofs + 2*al+1) & ~al;  /* Align and bump stack pointer. */
2015    }
2016    nslots = (spofs + 3) >> 2;
2017    if (nslots > as->evenspill)  /* Leave room for args in stack slots. */
2018      as->evenspill = nslots;
2019  }
2020#endif
2021  return REGSP_HINT(irt_isfp(ir->t) ? RID_FPRET : RID_RET);
2022}
2023
2024static void asm_setup_target(ASMState *as)
2025{
2026  /* May need extra exit for asm_stack_check on side traces. */
2027  asm_exitstub_setup(as, as->T->nsnap + (as->parent ? 1 : 0));
2028}
2029
2030#if LJ_BE
2031/* ARM64 instructions are always little-endian. Swap for ARM64BE. */
2032static void asm_mcode_fixup(MCode *mcode, MSize size)
2033{
2034  MCode *pe = (MCode *)((char *)mcode + size);
2035  while (mcode < pe) {
2036    MCode ins = *mcode;
2037    *mcode++ = lj_bswap(ins);
2038  }
2039}
2040#define LJ_TARGET_MCODE_FIXUP	1
2041#endif
2042
2043/* -- Trace patching ------------------------------------------------------ */
2044
2045/* Patch exit jumps of existing machine code to a new target. */
2046void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2047{
2048  MCode *p = T->mcode;
2049  MCode *pe = (MCode *)((char *)p + T->szmcode);
2050  MCode *cstart = NULL;
2051  MCode *mcarea = lj_mcode_patch(J, p, 0);
2052  MCode *px = exitstub_trace_addr(T, exitno);
2053  int patchlong = 1;
2054  /* Note: this assumes a trace exit is only ever patched once. */
2055  for (; p < pe; p++) {
2056    /* Look for exitstub branch, replace with branch to target. */
2057    ptrdiff_t delta = target - p;
2058    MCode ins = A64I_LE(*p);
2059    if ((ins & 0xff000000u) == 0x54000000u &&
2060	((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
2061      /* Patch bcc, if within range. */
2062      if (A64F_S_OK(delta, 19)) {
2063	*p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
2064	if (!cstart) cstart = p;
2065      }
2066    } else if ((ins & 0xfc000000u) == 0x14000000u &&
2067	       ((ins ^ (px-p)) & 0x03ffffffu) == 0) {
2068      /* Patch b. */
2069      lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
2070      *p = A64I_LE((ins & 0xfc000000u) | A64F_S26(delta));
2071      if (!cstart) cstart = p;
2072    } else if ((ins & 0x7e000000u) == 0x34000000u &&
2073	       ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
2074      /* Patch cbz/cbnz, if within range. */
2075      if (p[-1] == ARM64_NOPATCH_GC_CHECK) {
2076	patchlong = 0;
2077      } else if (A64F_S_OK(delta, 19)) {
2078	*p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
2079	if (!cstart) cstart = p;
2080      }
2081    } else if ((ins & 0x7e000000u) == 0x36000000u &&
2082	       ((ins ^ ((px-p)<<5)) & 0x0007ffe0u) == 0) {
2083      /* Patch tbz/tbnz, if within range. */
2084      if (A64F_S_OK(delta, 14)) {
2085	*p = A64I_LE((ins & 0xfff8001fu) | A64F_S14(delta));
2086	if (!cstart) cstart = p;
2087      }
2088    }
2089  }
2090  /* Always patch long-range branch in exit stub itself. Except, if we can't. */
2091  if (patchlong) {
2092    ptrdiff_t delta = target - px;
2093    lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
2094    *px = A64I_B | A64F_S26(delta);
2095    if (!cstart) cstart = px;
2096  }
2097  if (cstart) lj_mcode_sync(cstart, px+1);
2098  lj_mcode_patch(J, mcarea, 1);
2099}
2100