luna - luajit/src/lj_asm

   1/*
   2** MIPS IR assembler (SSA IR -> machine code).
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4*/
   5
   6/* -- Register allocator extensions --------------------------------------- */
   7
   8/* Allocate a register with a hint. */
   9static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
  10{
  11  Reg r = IR(ref)->r;
  12  if (ra_noreg(r)) {
  13    if (!ra_hashint(r) && !iscrossref(as, ref))
  14      ra_sethint(IR(ref)->r, hint);  /* Propagate register hint. */
  15    r = ra_allocref(as, ref, allow);
  16  }
  17  ra_noweak(as, r);
  18  return r;
  19}
  20
  21/* Allocate a register or RID_ZERO. */
  22static Reg ra_alloc1z(ASMState *as, IRRef ref, RegSet allow)
  23{
  24  Reg r = IR(ref)->r;
  25  if (ra_noreg(r)) {
  26    if (!(allow & RSET_FPR) && irref_isk(ref) && get_kval(as, ref) == 0)
  27      return RID_ZERO;
  28    r = ra_allocref(as, ref, allow);
  29  } else {
  30    ra_noweak(as, r);
  31  }
  32  return r;
  33}
  34
  35/* Allocate two source registers for three-operand instructions. */
  36static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
  37{
  38  IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
  39  Reg left = irl->r, right = irr->r;
  40  if (ra_hasreg(left)) {
  41    ra_noweak(as, left);
  42    if (ra_noreg(right))
  43      right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
  44    else
  45      ra_noweak(as, right);
  46  } else if (ra_hasreg(right)) {
  47    ra_noweak(as, right);
  48    left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
  49  } else if (ra_hashint(right)) {
  50    right = ra_alloc1z(as, ir->op2, allow);
  51    left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
  52  } else {
  53    left = ra_alloc1z(as, ir->op1, allow);
  54    right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
  55  }
  56  return left | (right << 8);
  57}
  58
  59/* -- Guard handling ------------------------------------------------------ */
  60
  61/* Need some spare long-range jump slots, for out-of-range branches. */
  62#define MIPS_SPAREJUMP		4
  63
  64/* Setup spare long-range jump slots per mcarea. */
  65static void asm_sparejump_setup(ASMState *as)
  66{
  67  MCode *mxp = as->mctop;
  68  if ((char *)mxp == (char *)as->J->mcarea + as->J->szmcarea) {
  69    mxp -= MIPS_SPAREJUMP*2;
  70    lj_assertA(MIPSI_NOP == 0, "bad NOP");
  71    memset(mxp, 0, MIPS_SPAREJUMP*2*sizeof(MCode));
  72    as->mctop = mxp;
  73  }
  74}
  75
  76static MCode *asm_sparejump_use(MCode *mcarea, MCode tjump)
  77{
  78  MCode *mxp = (MCode *)((char *)mcarea + ((MCLink *)mcarea)->size);
  79  int slot = MIPS_SPAREJUMP;
  80  while (slot--) {
  81    mxp -= 2;
  82    if (*mxp == tjump) {
  83      return mxp;
  84    } else if (*mxp == MIPSI_NOP) {
  85      *mxp = tjump;
  86      return mxp;
  87    }
  88  }
  89  return NULL;
  90}
  91
  92/* Setup exit stub after the end of each trace. */
  93static void asm_exitstub_setup(ASMState *as)
  94{
  95  uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler;
  96  MCode *mxp = as->mctop;
  97  *--mxp = MIPSI_LI | MIPSF_T(RID_TMP) | as->T->traceno;
  98  if (((uintptr_t)(mxp-1) ^ target) >> 28 == 0) {
  99    /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
 100    *--mxp = MIPSI_J | ((target >> 2) & 0x03ffffffu);
 101  } else {
 102    /* sw TMP, 0(sp); li TMP, K*_VXH(jgl); jr TMP ; li TMP, traceno */
 103    *--mxp = MIPSI_JR | MIPSF_S(RID_TMP);
 104    *--mxp = MIPSI_AL | MIPSF_T(RID_TMP) | MIPSF_S(RID_JGL) |
 105#if LJ_64
 106	     jglofs(as, &as->J->k64[LJ_K64_VM_EXIT_HANDLER]);
 107#else
 108	     jglofs(as, &as->J->k32[LJ_K32_VM_EXIT_HANDLER]);
 109#endif
 110  }
 111  *--mxp = MIPSI_SW | MIPSF_T(RID_TMP) | MIPSF_S(RID_SP) | 0;
 112  as->mctop = mxp;
 113}
 114
 115/* Keep this in-sync with exitstub_trace_addr(). */
 116#define asm_exitstub_addr(as)	((as)->mctop)
 117
 118/* Emit conditional branch to exit for guard. */
 119static void asm_guard(ASMState *as, MIPSIns mi, Reg rs, Reg rt)
 120{
 121  MCode *target = asm_exitstub_addr(as);
 122  MCode *p = as->mcp;
 123  if (LJ_UNLIKELY(p == as->invmcp)) {
 124    as->invmcp = NULL;
 125    as->loopinv = 1;
 126    as->mcp = p+1;
 127#if !LJ_TARGET_MIPSR6
 128    mi = mi ^ ((mi>>28) == 1 ? 0x04000000u : 0x00010000u);  /* Invert cond. */
 129#else
 130    mi = mi ^ ((mi>>28) == 1 ? 0x04000000u :
 131	       (mi>>28) == 4 ? 0x00800000u : 0x00010000u);  /* Invert cond. */
 132#endif
 133    target = p;  /* Patch target later in asm_loop_fixup. */
 134  }
 135  emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
 136  emit_branch(as, mi, rs, rt, target);
 137}
 138
 139/* -- Operand fusion ------------------------------------------------------ */
 140
 141/* Limit linear search to this distance. Avoids O(n^2) behavior. */
 142#define CONFLICT_SEARCH_LIM	31
 143
 144/* Check if there's no conflicting instruction between curins and ref. */
 145static int noconflict(ASMState *as, IRRef ref, IROp conflict)
 146{
 147  IRIns *ir = as->ir;
 148  IRRef i = as->curins;
 149  if (i > ref + CONFLICT_SEARCH_LIM)
 150    return 0;  /* Give up, ref is too far away. */
 151  while (--i > ref)
 152    if (ir[i].o == conflict)
 153      return 0;  /* Conflict found. */
 154  return 1;  /* Ok, no conflict. */
 155}
 156
 157/* Fuse the array base of colocated arrays. */
 158static int32_t asm_fuseabase(ASMState *as, IRRef ref)
 159{
 160  IRIns *ir = IR(ref);
 161  if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
 162      !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
 163    return (int32_t)sizeof(GCtab);
 164  return 0;
 165}
 166
 167/* Fuse array/hash/upvalue reference into register+offset operand. */
 168static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
 169{
 170  IRIns *ir = IR(ref);
 171  if (ra_noreg(ir->r)) {
 172    if (ir->o == IR_AREF) {
 173      if (mayfuse(as, ref)) {
 174	if (irref_isk(ir->op2)) {
 175	  IRRef tab = IR(ir->op1)->op1;
 176	  int32_t ofs = asm_fuseabase(as, tab);
 177	  IRRef refa = ofs ? tab : ir->op1;
 178	  ofs += 8*IR(ir->op2)->i;
 179	  if (checki16(ofs)) {
 180	    *ofsp = ofs;
 181	    return ra_alloc1(as, refa, allow);
 182	  }
 183	}
 184      }
 185    } else if (ir->o == IR_HREFK) {
 186      if (mayfuse(as, ref)) {
 187	int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
 188	if (checki16(ofs)) {
 189	  *ofsp = ofs;
 190	  return ra_alloc1(as, ir->op1, allow);
 191	}
 192      }
 193    } else if (ir->o == IR_UREFC) {
 194      if (irref_isk(ir->op1)) {
 195	GCfunc *fn = ir_kfunc(IR(ir->op1));
 196	intptr_t ofs = (intptr_t)&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv;
 197	intptr_t jgl = (intptr_t)J2G(as->J);
 198	if ((uintptr_t)(ofs-jgl) < 65536) {
 199	  *ofsp = ofs-jgl-32768;
 200	  return RID_JGL;
 201	} else {
 202	  *ofsp = (int16_t)ofs;
 203	  return ra_allock(as, ofs-(int16_t)ofs, allow);
 204	}
 205      }
 206    } else if (ir->o == IR_TMPREF) {
 207      *ofsp = (int32_t)(offsetof(global_State, tmptv)-32768);
 208      return RID_JGL;
 209    }
 210  }
 211  *ofsp = 0;
 212  return ra_alloc1(as, ref, allow);
 213}
 214
 215/* Fuse XLOAD/XSTORE reference into load/store operand. */
 216static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
 217			 RegSet allow, int32_t ofs)
 218{
 219  IRIns *ir = IR(ref);
 220  Reg base;
 221  if (ra_noreg(ir->r) && canfuse(as, ir)) {
 222    if (ir->o == IR_ADD) {
 223      intptr_t ofs2;
 224      if (irref_isk(ir->op2) && (ofs2 = ofs + get_kval(as, ir->op2),
 225				 checki16(ofs2))) {
 226	ref = ir->op1;
 227	ofs = (int32_t)ofs2;
 228      }
 229    } else if (ir->o == IR_STRREF) {
 230      intptr_t ofs2 = 65536;
 231      lj_assertA(ofs == 0, "bad usage");
 232      ofs = (int32_t)sizeof(GCstr);
 233      if (irref_isk(ir->op2)) {
 234	ofs2 = ofs + get_kval(as, ir->op2);
 235	ref = ir->op1;
 236      } else if (irref_isk(ir->op1)) {
 237	ofs2 = ofs + get_kval(as, ir->op1);
 238	ref = ir->op2;
 239      }
 240      if (!checki16(ofs2)) {
 241	/* NYI: Fuse ADD with constant. */
 242	Reg right, left = ra_alloc2(as, ir, allow);
 243	right = (left >> 8); left &= 255;
 244	emit_hsi(as, mi, rt, RID_TMP, ofs);
 245	emit_dst(as, MIPSI_AADDU, RID_TMP, left, right);
 246	return;
 247      }
 248      ofs = ofs2;
 249    }
 250  }
 251  base = ra_alloc1(as, ref, allow);
 252  emit_hsi(as, mi, rt, base, ofs);
 253}
 254
 255/* -- Calls --------------------------------------------------------------- */
 256
 257/* Generate a call to a C function. */
 258static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 259{
 260  uint32_t n, nargs = CCI_XNARGS(ci);
 261  int32_t ofs = LJ_32 ? 16 : 0;
 262#if LJ_SOFTFP
 263  Reg gpr = REGARG_FIRSTGPR;
 264#else
 265  Reg gpr, fpr = REGARG_FIRSTFPR;
 266#endif
 267  if ((void *)ci->func)
 268    emit_call(as, (void *)ci->func, 1);
 269#if !LJ_SOFTFP
 270  for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
 271    as->cost[gpr] = REGCOST(~0u, ASMREF_L);
 272  gpr = REGARG_FIRSTGPR;
 273#endif
 274  for (n = 0; n < nargs; n++) {  /* Setup args. */
 275    IRRef ref = args[n];
 276    if (ref) {
 277      IRIns *ir = IR(ref);
 278#if !LJ_SOFTFP
 279      if (irt_isfp(ir->t) && fpr <= REGARG_LASTFPR &&
 280	  !(ci->flags & CCI_VARARG)) {
 281	lj_assertA(rset_test(as->freeset, fpr),
 282		   "reg %d not free", fpr);  /* Already evicted. */
 283	ra_leftov(as, fpr, ref);
 284	fpr += LJ_32 ? 2 : 1;
 285	gpr += (LJ_32 && irt_isnum(ir->t)) ? 2 : 1;
 286      } else
 287#endif
 288      {
 289#if LJ_32 && !LJ_SOFTFP
 290	fpr = REGARG_LASTFPR+1;
 291#endif
 292	if (LJ_32 && irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
 293	if (gpr <= REGARG_LASTGPR) {
 294	  lj_assertA(rset_test(as->freeset, gpr),
 295		     "reg %d not free", gpr);  /* Already evicted. */
 296#if !LJ_SOFTFP
 297	  if (irt_isfp(ir->t)) {
 298	    RegSet of = as->freeset;
 299	    Reg r;
 300	    /* Workaround to protect argument GPRs from being used for remat. */
 301	    as->freeset &= ~RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1);
 302	    r = ra_alloc1(as, ref, RSET_FPR);
 303	    as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
 304	    if (irt_isnum(ir->t)) {
 305#if LJ_32
 306	      emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1);
 307	      emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r);
 308	      lj_assertA(rset_test(as->freeset, gpr+1),
 309			 "reg %d not free", gpr+1);  /* Already evicted. */
 310	      gpr += 2;
 311#else
 312	      emit_tg(as, MIPSI_DMFC1, gpr, r);
 313	      gpr++; fpr++;
 314#endif
 315	    } else if (irt_isfloat(ir->t)) {
 316	      emit_tg(as, MIPSI_MFC1, gpr, r);
 317	      gpr++;
 318#if LJ_64
 319	      fpr++;
 320#endif
 321	    }
 322	  } else
 323#endif
 324	  {
 325	    ra_leftov(as, gpr, ref);
 326	    gpr++;
 327#if LJ_64 && !LJ_SOFTFP
 328	    fpr++;
 329#endif
 330	  }
 331	} else {
 332	  Reg r = ra_alloc1z(as, ref, !LJ_SOFTFP && irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
 333#if LJ_32
 334	  if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
 335	  emit_spstore(as, ir, r, ofs);
 336	  ofs += irt_isnum(ir->t) ? 8 : 4;
 337#else
 338	  emit_spstore(as, ir, r, ofs + ((LJ_BE && !irt_isfp(ir->t) && !irt_is64(ir->t)) ? 4 : 0));
 339	  ofs += 8;
 340#endif
 341	}
 342      }
 343    } else {
 344#if !LJ_SOFTFP
 345      fpr = REGARG_LASTFPR+1;
 346#endif
 347      if (gpr <= REGARG_LASTGPR) {
 348	gpr++;
 349#if LJ_64 && !LJ_SOFTFP
 350	fpr++;
 351#endif
 352      } else {
 353	ofs += LJ_32 ? 4 : 8;
 354      }
 355    }
 356    checkmclim(as);
 357  }
 358}
 359
 360/* Setup result reg/sp for call. Evict scratch regs. */
 361static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
 362{
 363  RegSet drop = RSET_SCRATCH;
 364  int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
 365#if !LJ_SOFTFP
 366  if ((ci->flags & CCI_NOFPRCLOBBER))
 367    drop &= ~RSET_FPR;
 368#endif
 369  if (ra_hasreg(ir->r))
 370    rset_clear(drop, ir->r);  /* Dest reg handled below. */
 371  if (hiop && ra_hasreg((ir+1)->r))
 372    rset_clear(drop, (ir+1)->r);  /* Dest reg handled below. */
 373  ra_evictset(as, drop);  /* Evictions must be performed first. */
 374  if (ra_used(ir)) {
 375    lj_assertA(!irt_ispri(ir->t), "PRI dest");
 376    if (!LJ_SOFTFP && irt_isfp(ir->t)) {
 377      if ((ci->flags & CCI_CASTU64)) {
 378	int32_t ofs = sps_scale(ir->s);
 379	Reg dest = ir->r;
 380	if (ra_hasreg(dest)) {
 381	  ra_free(as, dest);
 382	  ra_modified(as, dest);
 383#if LJ_32
 384	  emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1);
 385	  emit_tg(as, MIPSI_MTC1, RID_RETLO, dest);
 386#else
 387	  emit_tg(as, MIPSI_DMTC1, RID_RET, dest);
 388#endif
 389	}
 390	if (ofs) {
 391#if LJ_32
 392	  emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0));
 393	  emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4));
 394#else
 395	  emit_tsi(as, MIPSI_SD, RID_RET, RID_SP, ofs);
 396#endif
 397	}
 398      } else {
 399	ra_destreg(as, ir, RID_FPRET);
 400      }
 401    } else if (hiop) {
 402      ra_destpair(as, ir);
 403    } else {
 404      ra_destreg(as, ir, RID_RET);
 405    }
 406  }
 407}
 408
 409static void asm_callx(ASMState *as, IRIns *ir)
 410{
 411  IRRef args[CCI_NARGS_MAX*2];
 412  CCallInfo ci;
 413  IRRef func;
 414  IRIns *irf;
 415  ci.flags = asm_callx_flags(as, ir);
 416  asm_collectargs(as, ir, &ci, args);
 417  asm_setupresult(as, ir, &ci);
 418  func = ir->op2; irf = IR(func);
 419  if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
 420  if (irref_isk(func)) {  /* Call to constant address. */
 421    ci.func = (ASMFunction)(void *)get_kval(as, func);
 422  } else {  /* Need specific register for indirect calls. */
 423    Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR));
 424    MCode *p = as->mcp;
 425    if (r == RID_CFUNCADDR)
 426      *--p = MIPSI_NOP;
 427    else
 428      *--p = MIPSI_MOVE | MIPSF_D(RID_CFUNCADDR) | MIPSF_S(r);
 429    *--p = MIPSI_JALR | MIPSF_S(r);
 430    as->mcp = p;
 431    ci.func = (ASMFunction)(void *)0;
 432  }
 433  asm_gencall(as, &ci, args);
 434}
 435
 436#if !LJ_SOFTFP
 437static void asm_callround(ASMState *as, IRIns *ir, IRCallID id)
 438{
 439  /* The modified regs must match with the *.dasc implementation. */
 440  RegSet drop = RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_FPRET)|
 441		RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(REGARG_FIRSTFPR)|
 442		RID2RSET(RID_CFUNCADDR)
 443#if LJ_TARGET_MIPSR6
 444		|RID2RSET(RID_F21)
 445#endif
 446		;
 447  if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
 448  ra_evictset(as, drop);
 449  ra_destreg(as, ir, RID_FPRET);
 450  emit_call(as, (void *)lj_ir_callinfo[id].func, 0);
 451  ra_leftov(as, REGARG_FIRSTFPR, ir->op1);
 452}
 453#endif
 454
 455/* -- Returns ------------------------------------------------------------- */
 456
 457/* Return to lower frame. Guard that it goes to the right spot. */
 458static void asm_retf(ASMState *as, IRIns *ir)
 459{
 460  Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
 461  void *pc = ir_kptr(IR(ir->op2));
 462  int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
 463  as->topslot -= (BCReg)delta;
 464  if ((int32_t)as->topslot < 0) as->topslot = 0;
 465  irt_setmark(IR(REF_BASE)->t);  /* Children must not coalesce with BASE reg. */
 466  emit_setgl(as, base, jit_base);
 467  emit_addptr(as, base, -8*delta);
 468  asm_guard(as, MIPSI_BNE, RID_TMP,
 469	    ra_allock(as, igcptr(pc), rset_exclude(RSET_GPR, base)));
 470  emit_tsi(as, MIPSI_AL, RID_TMP, base, (LJ_BE || LJ_FR2) ? -8 : -4);
 471}
 472
 473/* -- Buffer operations --------------------------------------------------- */
 474
 475#if LJ_HASBUFFER
 476static void asm_bufhdr_write(ASMState *as, Reg sb)
 477{
 478  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
 479  IRIns irgc;
 480  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
 481  emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
 482  if ((as->flags & JIT_F_MIPSXXR2)) {
 483    emit_tsml(as, LJ_64 ? MIPSI_DINS : MIPSI_INS, RID_TMP, tmp,
 484	      lj_fls(SBUF_MASK_FLAG), 0);
 485  } else {
 486    emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
 487    emit_tsi(as, MIPSI_ANDI, tmp, tmp, SBUF_MASK_FLAG);
 488  }
 489  emit_getgl(as, RID_TMP, cur_L);
 490  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
 491}
 492#endif
 493
 494/* -- Type conversions ---------------------------------------------------- */
 495
 496#if !LJ_SOFTFP
 497static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
 498{
 499  Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
 500  Reg dest = ra_dest(as, ir, RSET_GPR);
 501#if !LJ_TARGET_MIPSR6
 502  asm_guard(as, MIPSI_BC1F, 0, 0);
 503  emit_fgh(as, MIPSI_C_EQ_D, 0, tmp, left);
 504#else
 505  asm_guard(as, MIPSI_BC1EQZ, 0, (tmp&31));
 506  emit_fgh(as, MIPSI_CMP_EQ_D, tmp, tmp, left);
 507#endif
 508  emit_fg(as, MIPSI_CVT_D_W, tmp, tmp);
 509  emit_tg(as, MIPSI_MFC1, dest, tmp);
 510  emit_fg(as, MIPSI_CVT_W_D, tmp, left);
 511}
 512
 513static void asm_tobit(ASMState *as, IRIns *ir)
 514{
 515  RegSet allow = RSET_FPR;
 516  Reg dest = ra_dest(as, ir, RSET_GPR);
 517  Reg left = ra_alloc1(as, ir->op1, allow);
 518  Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
 519  Reg tmp = ra_scratch(as, rset_clear(allow, right));
 520  emit_tg(as, MIPSI_MFC1, dest, tmp);
 521  emit_fgh(as, MIPSI_ADD_D, tmp, left, right);
 522}
 523#elif LJ_64  /* && LJ_SOFTFP */
 524static void asm_tointg(ASMState *as, IRIns *ir, Reg r)
 525{
 526  /* The modified regs must match with the *.dasc implementation. */
 527  RegSet drop = RID2RSET(REGARG_FIRSTGPR)|RID2RSET(RID_RET)|RID2RSET(RID_RET+1)|
 528		RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_CFUNCADDR);
 529  if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
 530  ra_evictset(as, drop);
 531  /* Return values are in RID_RET (converted value) and RID_RET+1 (status). */
 532  ra_destreg(as, ir, RID_RET);
 533  asm_guard(as, MIPSI_BNE, RID_RET+1, RID_ZERO);
 534  emit_call(as, (void *)lj_ir_callinfo[IRCALL_lj_vm_tointg].func, 0);
 535  if (r == RID_NONE)
 536    ra_leftov(as, REGARG_FIRSTGPR, ir->op1);
 537  else if (r != REGARG_FIRSTGPR)
 538    emit_move(as, REGARG_FIRSTGPR, r);
 539}
 540
 541static void asm_tobit(ASMState *as, IRIns *ir)
 542{
 543  Reg dest = ra_dest(as, ir, RSET_GPR);
 544  emit_dta(as, MIPSI_SLL, dest, dest, 0);
 545  asm_callid(as, ir, IRCALL_lj_vm_tobit);
 546}
 547#endif
 548
 549static void asm_conv(ASMState *as, IRIns *ir)
 550{
 551  IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 552#if !LJ_SOFTFP32
 553  int stfp = (st == IRT_NUM || st == IRT_FLOAT);
 554#endif
 555#if LJ_64
 556  int st64 = (st == IRT_I64 || st == IRT_U64 || st == IRT_P64);
 557#endif
 558  IRRef lref = ir->op1;
 559#if LJ_32
 560  /* 64 bit integer conversions are handled by SPLIT. */
 561  lj_assertA(!(irt_isint64(ir->t) || (st == IRT_I64 || st == IRT_U64)),
 562	     "IR %04d has unsplit 64 bit type",
 563	     (int)(ir - as->ir) - REF_BIAS);
 564#endif
 565#if LJ_SOFTFP32
 566  /* FP conversions are handled by SPLIT. */
 567  lj_assertA(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT),
 568	     "IR %04d has FP type",
 569	     (int)(ir - as->ir) - REF_BIAS);
 570  /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
 571#else
 572  lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
 573#if !LJ_SOFTFP
 574  if (irt_isfp(ir->t)) {
 575    Reg dest = ra_dest(as, ir, RSET_FPR);
 576    if (stfp) {  /* FP to FP conversion. */
 577      emit_fg(as, st == IRT_NUM ? MIPSI_CVT_S_D : MIPSI_CVT_D_S,
 578	      dest, ra_alloc1(as, lref, RSET_FPR));
 579    } else if (st == IRT_U32) {  /* U32 to FP conversion. */
 580      /* y = (x ^ 0x8000000) + 2147483648.0 */
 581      Reg left = ra_alloc1(as, lref, RSET_GPR);
 582      Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
 583      if (irt_isfloat(ir->t))
 584	emit_fg(as, MIPSI_CVT_S_D, dest, dest);
 585      /* Must perform arithmetic with doubles to keep the precision. */
 586      emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
 587      emit_fg(as, MIPSI_CVT_D_W, dest, dest);
 588      emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
 589		 (void *)&as->J->k64[LJ_K64_2P31], RSET_GPR);
 590      emit_tg(as, MIPSI_MTC1, RID_TMP, dest);
 591      emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left);
 592      emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
 593#if LJ_64
 594    } else if(st == IRT_U64) {  /* U64 to FP conversion. */
 595      /* if (x >= 1u<<63) y = (double)(int64_t)(x&(1u<<63)-1) + pow(2.0, 63) */
 596      Reg left = ra_alloc1(as, lref, RSET_GPR);
 597      Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
 598      MCLabel l_end = emit_label(as);
 599      if (irt_isfloat(ir->t)) {
 600	emit_fgh(as, MIPSI_ADD_S, dest, dest, tmp);
 601	emit_lsptr(as, MIPSI_LWC1, (tmp & 31), (void *)&as->J->k32[LJ_K32_2P63],
 602		   rset_exclude(RSET_GPR, left));
 603	emit_fg(as, MIPSI_CVT_S_L, dest, dest);
 604      } else {
 605	emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
 606	emit_lsptr(as, MIPSI_LDC1, (tmp & 31), (void *)&as->J->k64[LJ_K64_2P63],
 607		   rset_exclude(RSET_GPR, left));
 608	emit_fg(as, MIPSI_CVT_D_L, dest, dest);
 609      }
 610      emit_branch(as, MIPSI_BGEZ, left, RID_ZERO, l_end);
 611      emit_tg(as, MIPSI_DMTC1, RID_TMP, dest);
 612      emit_tsml(as, MIPSI_DEXTM, RID_TMP, left, 30, 0);
 613#endif
 614    } else {  /* Integer to FP conversion. */
 615      Reg left = ra_alloc1(as, lref, RSET_GPR);
 616#if LJ_32
 617      emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
 618	      dest, dest);
 619      emit_tg(as, MIPSI_MTC1, left, dest);
 620#else
 621      MIPSIns mi = irt_isfloat(ir->t) ?
 622	(st64 ? MIPSI_CVT_S_L : MIPSI_CVT_S_W) :
 623	(st64 ? MIPSI_CVT_D_L : MIPSI_CVT_D_W);
 624      emit_fg(as, mi, dest, dest);
 625      emit_tg(as, st64 ? MIPSI_DMTC1 : MIPSI_MTC1, left, dest);
 626#endif
 627    }
 628  } else if (stfp) {  /* FP to integer conversion. */
 629    if (irt_isguard(ir->t)) {
 630      /* Checked conversions are only supported from number to int. */
 631      lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
 632		 "bad type for checked CONV");
 633      asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
 634    } else {
 635      Reg dest = ra_dest(as, ir, RSET_GPR);
 636      Reg left = ra_alloc1(as, lref, RSET_FPR);
 637      Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
 638      lj_assertA(!irt_isu32(ir->t), "bad CONV u32.fp emitted");
 639#if LJ_64
 640      if (irt_isu64(ir->t)) {  /* FP to U64 conversion. */
 641	MCLabel l_end = emit_label(as);
 642	emit_tg(as, MIPSI_DMFC1, dest, tmp);
 643	/* For result == INT64_MAX add -2^64 and convert again. */
 644	if (st == IRT_NUM) {
 645	  emit_fg(as, MIPSI_TRUNC_L_D, tmp, tmp);
 646	  emit_fgh(as, MIPSI_ADD_D, tmp, left, tmp);
 647	  emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
 648		     (void *)&as->J->k64[LJ_K64_M2P64],
 649		     rset_exclude(RSET_GPR, dest));  /* Delay slot. */
 650	  emit_branch(as, MIPSI_BNE, RID_TMP, dest, l_end);  /* != INT64_MAX? */
 651	  emit_dta(as, MIPSI_DSRL, RID_TMP, RID_TMP, 1);
 652	  emit_ti(as, MIPSI_LI, RID_TMP, -1);
 653	  emit_tg(as, MIPSI_DMFC1, dest, tmp);
 654	  emit_fg(as, MIPSI_TRUNC_L_D, tmp, left);
 655	} else {
 656	  emit_fg(as, MIPSI_TRUNC_L_S, tmp, tmp);
 657	  emit_fgh(as, MIPSI_ADD_S, tmp, left, tmp);
 658	  emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
 659		     (void *)&as->J->k32[LJ_K32_M2P64],
 660		     rset_exclude(RSET_GPR, dest));  /* Delay slot. */
 661	  emit_branch(as, MIPSI_BNE, RID_TMP, dest, l_end);  /* != INT64_MAX? */
 662	  emit_dta(as, MIPSI_DSRL, RID_TMP, RID_TMP, 1);
 663	  emit_ti(as, MIPSI_LI, RID_TMP, -1);
 664	  emit_tg(as, MIPSI_DMFC1, dest, tmp);
 665	  emit_fg(as, MIPSI_TRUNC_L_S, tmp, left);
 666	}
 667      } else
 668#endif
 669      {
 670#if LJ_32
 671	emit_tg(as, MIPSI_MFC1, dest, tmp);
 672	emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D,
 673		tmp, left);
 674#else
 675	MIPSIns mi = irt_is64(ir->t) ?
 676	  (st == IRT_NUM ? MIPSI_TRUNC_L_D : MIPSI_TRUNC_L_S) :
 677	  (st == IRT_NUM ? MIPSI_TRUNC_W_D : MIPSI_TRUNC_W_S);
 678	emit_tg(as, irt_is64(ir->t) ? MIPSI_DMFC1 : MIPSI_MFC1, dest, tmp);
 679	emit_fg(as, mi, tmp, left);
 680#endif
 681      }
 682    }
 683  } else
 684#else
 685  if (irt_isfp(ir->t)) {
 686#if LJ_64 && LJ_HASFFI
 687    if (stfp) {  /* FP to FP conversion. */
 688      asm_callid(as, ir, irt_isnum(ir->t) ? IRCALL_softfp_f2d :
 689					    IRCALL_softfp_d2f);
 690    } else {  /* Integer to FP conversion. */
 691      IRCallID cid = ((IRT_IS64 >> st) & 1) ?
 692	(irt_isnum(ir->t) ?
 693	 (st == IRT_I64 ? IRCALL_fp64_l2d : IRCALL_fp64_ul2d) :
 694	 (st == IRT_I64 ? IRCALL_fp64_l2f : IRCALL_fp64_ul2f)) :
 695	(irt_isnum(ir->t) ?
 696	 (st == IRT_INT ? IRCALL_softfp_i2d : IRCALL_softfp_ui2d) :
 697	 (st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f));
 698      asm_callid(as, ir, cid);
 699    }
 700#else
 701    asm_callid(as, ir, IRCALL_softfp_i2d);
 702#endif
 703  } else if (stfp) {  /* FP to integer conversion. */
 704    if (irt_isguard(ir->t)) {
 705      /* Checked conversions are only supported from number to int. */
 706      lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
 707		 "bad type for checked CONV");
 708      asm_tointg(as, ir, RID_NONE);
 709    } else {
 710      IRCallID cid;
 711      lj_assertA(!irt_isu32(ir->t), "bad CONV u32.fp emitted");
 712      lj_assertA(!(irt_is64(ir->t) && st != IRT_NUM), "bad CONV *64.float emitted");
 713      cid = irt_is64(ir->t) ? IRCALL_lj_vm_num2u64 :
 714	    (st == IRT_NUM ? IRCALL_softfp_d2i : IRCALL_softfp_f2i);
 715      asm_callid(as, ir, cid);
 716    }
 717  } else
 718#endif
 719#endif
 720  {
 721    Reg dest = ra_dest(as, ir, RSET_GPR);
 722    if (st >= IRT_I8 && st <= IRT_U16) {  /* Extend to 32 bit integer. */
 723      Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
 724      lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
 725      if ((ir->op2 & IRCONV_SEXT)) {
 726	if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
 727	  emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left);
 728	} else {
 729	  uint32_t shift = st == IRT_I8 ? 24 : 16;
 730	  emit_dta(as, MIPSI_SRA, dest, dest, shift);
 731	  emit_dta(as, MIPSI_SLL, dest, left, shift);
 732	}
 733      } else {
 734	emit_tsi(as, MIPSI_ANDI, dest, left,
 735		 (int32_t)(st == IRT_U8 ? 0xff : 0xffff));
 736      }
 737    } else {  /* 32/64 bit integer conversions. */
 738#if LJ_32
 739      /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
 740      ra_leftov(as, dest, lref);  /* Do nothing, but may need to move regs. */
 741#else
 742      if (irt_is64(ir->t)) {
 743	if (st64) {
 744	  /* 64/64 bit no-op (cast)*/
 745	  ra_leftov(as, dest, lref);
 746	} else {
 747	  Reg left = ra_alloc1(as, lref, RSET_GPR);
 748	  if ((ir->op2 & IRCONV_SEXT)) {  /* 32 to 64 bit sign extension. */
 749	    emit_dta(as, MIPSI_SLL, dest, left, 0);
 750	  } else {  /* 32 to 64 bit zero extension. */
 751	    emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
 752	  }
 753	}
 754      } else {
 755	if (!irt_isu32(ir->t)) {  /* Implicit sign extension. */
 756	  Reg left = ra_alloc1(as, lref, RSET_GPR);
 757	  emit_dta(as, MIPSI_SLL, dest, left, 0);
 758	} else if (st64 && !(ir->op2 & IRCONV_NONE)) {
 759	  /* This is either a 32 bit reg/reg mov which zeroes the hiword
 760	  ** or a load of the loword from a 64 bit address.
 761	  */
 762	  Reg left = ra_alloc1(as, lref, RSET_GPR);
 763	  emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
 764	} else {  /* 32/32 bit no-op (cast). */
 765	  /* Do nothing, but may need to move regs. */
 766	  ra_leftov(as, dest, lref);
 767	}
 768      }
 769#endif
 770    }
 771  }
 772}
 773
 774static void asm_strto(ASMState *as, IRIns *ir)
 775{
 776  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
 777  IRRef args[2];
 778  int32_t ofs = 0;
 779#if LJ_SOFTFP32
 780  ra_evictset(as, RSET_SCRATCH);
 781  if (ra_used(ir)) {
 782    if (ra_hasspill(ir->s) && ra_hasspill((ir+1)->s) &&
 783	(ir->s & 1) == LJ_BE && (ir->s ^ 1) == (ir+1)->s) {
 784      int i;
 785      for (i = 0; i < 2; i++) {
 786	Reg r = (ir+i)->r;
 787	if (ra_hasreg(r)) {
 788	  ra_free(as, r);
 789	  ra_modified(as, r);
 790	  emit_spload(as, ir+i, r, sps_scale((ir+i)->s));
 791	}
 792      }
 793      ofs = sps_scale(ir->s & ~1);
 794    } else {
 795      Reg rhi = ra_dest(as, ir+1, RSET_GPR);
 796      Reg rlo = ra_dest(as, ir, rset_exclude(RSET_GPR, rhi));
 797      emit_tsi(as, MIPSI_LW, rhi, RID_SP, ofs+(LJ_BE?0:4));
 798      emit_tsi(as, MIPSI_LW, rlo, RID_SP, ofs+(LJ_BE?4:0));
 799    }
 800  }
 801#else
 802  RegSet drop = RSET_SCRATCH;
 803  if (ra_hasreg(ir->r)) rset_set(drop, ir->r);  /* Spill dest reg (if any). */
 804  ra_evictset(as, drop);
 805  ofs = sps_scale(ir->s);
 806#endif
 807  asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO);  /* Test return status. */
 808  args[0] = ir->op1;      /* GCstr *str */
 809  args[1] = ASMREF_TMP1;  /* TValue *n  */
 810  asm_gencall(as, ci, args);
 811  /* Store the result to the spill slot or temp slots. */
 812  emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1),
 813	   RID_SP, ofs);
 814}
 815
 816/* -- Memory references --------------------------------------------------- */
 817
 818#if LJ_64
 819/* Store tagged value for ref at base+ofs. */
 820static void asm_tvstore64(ASMState *as, Reg base, int32_t ofs, IRRef ref)
 821{
 822  RegSet allow = rset_exclude(RSET_GPR, base);
 823  IRIns *ir = IR(ref);
 824  lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
 825	     "store of IR type %d", irt_type(ir->t));
 826  if (irref_isk(ref)) {
 827    TValue k;
 828    lj_ir_kvalue(as->J->L, &k, ir);
 829    emit_tsi(as, MIPSI_SD, ra_allock(as, (int64_t)k.u64, allow), base, ofs);
 830  } else {
 831    Reg src = ra_alloc1(as, ref, allow);
 832    Reg type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47,
 833			 rset_exclude(allow, src));
 834    emit_tsi(as, MIPSI_SD, RID_TMP, base, ofs);
 835    if (irt_isinteger(ir->t)) {
 836      emit_dst(as, MIPSI_DADDU, RID_TMP, RID_TMP, type);
 837      emit_tsml(as, MIPSI_DEXT, RID_TMP, src, 31, 0);
 838    } else {
 839      emit_dst(as, MIPSI_DADDU, RID_TMP, src, type);
 840    }
 841  }
 842}
 843#endif
 844
 845/* Get pointer to TValue. */
 846static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode)
 847{
 848  int32_t tmpofs = (int32_t)(offsetof(global_State, tmptv)-32768);
 849  if ((mode & IRTMPREF_IN1)) {
 850    IRIns *ir = IR(ref);
 851    if (irt_isnum(ir->t)) {
 852      if ((mode & IRTMPREF_OUT1)) {
 853#if LJ_SOFTFP
 854	emit_tsi(as, MIPSI_AADDIU, dest, RID_JGL, tmpofs);
 855#if LJ_64
 856	emit_setgl(as, ra_alloc1(as, ref, RSET_GPR), tmptv.u64);
 857#else
 858	lj_assertA(irref_isk(ref), "unsplit FP op");
 859	emit_setgl(as,
 860		   ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, RSET_GPR),
 861		   tmptv.u32.lo);
 862	emit_setgl(as,
 863		   ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, RSET_GPR),
 864		   tmptv.u32.hi);
 865#endif
 866#else
 867	Reg src = ra_alloc1(as, ref, RSET_FPR);
 868	emit_tsi(as, MIPSI_AADDIU, dest, RID_JGL, tmpofs);
 869	emit_tsi(as, MIPSI_SDC1, (src & 31),  RID_JGL, tmpofs);
 870#endif
 871      } else if (irref_isk(ref)) {
 872	/* Use the number constant itself as a TValue. */
 873	ra_allockreg(as, igcptr(ir_knum(ir)), dest);
 874      } else {
 875#if LJ_SOFTFP32
 876	lj_assertA(0, "unsplit FP op");
 877#else
 878	/* Otherwise force a spill and use the spill slot. */
 879	emit_tsi(as, MIPSI_AADDIU, dest, RID_SP, ra_spill(as, ir));
 880#endif
 881      }
 882    } else {
 883      /* Otherwise use g->tmptv to hold the TValue. */
 884#if LJ_32
 885      Reg type;
 886      emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, tmpofs);
 887      if (!irt_ispri(ir->t)) {
 888	Reg src = ra_alloc1(as, ref, RSET_GPR);
 889	emit_setgl(as, src, tmptv.gcr);
 890      }
 891      if (LJ_SOFTFP && (ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t))
 892	type = ra_alloc1(as, ref+1, RSET_GPR);
 893      else
 894	type = ra_allock(as, (int32_t)irt_toitype(ir->t), RSET_GPR);
 895      emit_setgl(as, type, tmptv.it);
 896#else
 897      asm_tvstore64(as, dest, 0, ref);
 898      emit_tsi(as, MIPSI_DADDIU, dest, RID_JGL, tmpofs);
 899#endif
 900    }
 901  } else {
 902    emit_tsi(as, MIPSI_AADDIU, dest, RID_JGL, tmpofs);
 903  }
 904}
 905
 906static void asm_aref(ASMState *as, IRIns *ir)
 907{
 908  Reg dest = ra_dest(as, ir, RSET_GPR);
 909  Reg idx, base;
 910  if (irref_isk(ir->op2)) {
 911    IRRef tab = IR(ir->op1)->op1;
 912    int32_t ofs = asm_fuseabase(as, tab);
 913    IRRef refa = ofs ? tab : ir->op1;
 914    ofs += 8*IR(ir->op2)->i;
 915    if (checki16(ofs)) {
 916      base = ra_alloc1(as, refa, RSET_GPR);
 917      emit_tsi(as, MIPSI_AADDIU, dest, base, ofs);
 918      return;
 919    }
 920  }
 921  base = ra_alloc1(as, ir->op1, RSET_GPR);
 922  idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
 923#if !LJ_TARGET_MIPSR6
 924  emit_dst(as, MIPSI_AADDU, dest, RID_TMP, base);
 925  emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3);
 926#else
 927  emit_dst(as, MIPSI_ALSA | MIPSF_A(3-1), dest, idx, base);
 928#endif
 929}
 930
 931/* Inlined hash lookup. Specialized for key type and for const keys.
 932** The equivalent C code is:
 933**   Node *n = hashkey(t, key);
 934**   do {
 935**     if (lj_obj_equal(&n->key, key)) return &n->val;
 936**   } while ((n = nextnode(n)));
 937**   return niltv(L);
 938*/
 939static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 940{
 941  RegSet allow = RSET_GPR;
 942  int destused = ra_used(ir);
 943  Reg dest = ra_dest(as, ir, allow);
 944  Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
 945  Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2;
 946#if LJ_64
 947  Reg cmp64 = RID_NONE;
 948#endif
 949  IRRef refkey = ir->op2;
 950  IRIns *irkey = IR(refkey);
 951  int isk = irref_isk(refkey);
 952  IRType1 kt = irkey->t;
 953  uint32_t khash;
 954  MCLabel l_end, l_loop, l_next;
 955
 956  rset_clear(allow, tab);
 957  if (!LJ_SOFTFP && irt_isnum(kt)) {
 958    key = ra_alloc1(as, refkey, RSET_FPR);
 959    tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
 960  } else {
 961    if (!irt_ispri(kt)) {
 962      key = ra_alloc1(as, refkey, allow);
 963      rset_clear(allow, key);
 964    }
 965#if LJ_32
 966    if (LJ_SOFTFP && irkey[1].o == IR_HIOP) {
 967      if (ra_hasreg((irkey+1)->r)) {
 968	type = tmpnum = (irkey+1)->r;
 969	tmp1 = ra_scratch(as, allow);
 970	rset_clear(allow, tmp1);
 971	ra_noweak(as, tmpnum);
 972      } else {
 973	type = tmpnum = ra_allocref(as, refkey+1, allow);
 974      }
 975      rset_clear(allow, tmpnum);
 976    } else {
 977      type = ra_allock(as, (int32_t)irt_toitype(kt), allow);
 978      rset_clear(allow, type);
 979    }
 980#endif
 981  }
 982  tmp2 = ra_scratch(as, allow);
 983  rset_clear(allow, tmp2);
 984#if LJ_64
 985  if (LJ_SOFTFP || !irt_isnum(kt)) {
 986    /* Allocate cmp64 register used for 64-bit comparisons */
 987    if (LJ_SOFTFP && irt_isnum(kt)) {
 988      cmp64 = key;
 989    } else if (!isk && irt_isaddr(kt)) {
 990      cmp64 = tmp2;
 991    } else {
 992      int64_t k;
 993      if (isk && irt_isaddr(kt)) {
 994	k = ((int64_t)irt_toitype(kt) << 47) | irkey[1].tv.u64;
 995      } else {
 996	lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type");
 997	k = ~((int64_t)~irt_toitype(kt) << 47);
 998      }
 999      cmp64 = ra_allock(as, k, allow);
1000      rset_clear(allow, cmp64);
1001    }
1002  }
1003#endif
1004
1005  /* Key not found in chain: jump to exit (if merged) or load niltv. */
1006  l_end = emit_label(as);
1007  as->invmcp = NULL;
1008  if (merge == IR_NE)
1009    asm_guard(as, MIPSI_B, RID_ZERO, RID_ZERO);
1010  else if (destused)
1011    emit_loada(as, dest, niltvg(J2G(as->J)));
1012  /* Follow hash chain until the end. */
1013  emit_move(as, dest, tmp1);
1014  l_loop = --as->mcp;
1015  emit_tsi(as, MIPSI_AL, tmp1, dest, (int32_t)offsetof(Node, next));
1016  l_next = emit_label(as);
1017
1018  /* Type and value comparison. */
1019  if (merge == IR_EQ) {  /* Must match asm_guard(). */
1020    emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
1021    l_end = asm_exitstub_addr(as);
1022  }
1023  if (!LJ_SOFTFP && irt_isnum(kt)) {
1024#if !LJ_TARGET_MIPSR6
1025    emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
1026    emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
1027#else
1028    emit_branch(as, MIPSI_BC1NEZ, 0, (tmpnum&31), l_end);
1029    emit_fgh(as, MIPSI_CMP_EQ_D, tmpnum, tmpnum, key);
1030#endif
1031    *--as->mcp = MIPSI_NOP;  /* Avoid NaN comparison overhead. */
1032    emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next);
1033    emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM);
1034#if LJ_32
1035    emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
1036  } else {
1037    if (irt_ispri(kt)) {
1038      emit_branch(as, MIPSI_BEQ, tmp1, type, l_end);
1039    } else {
1040      emit_branch(as, MIPSI_BEQ, tmp2, key, l_end);
1041      emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
1042      emit_branch(as, MIPSI_BNE, tmp1, type, l_next);
1043    }
1044  }
1045  emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it));
1046  *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
1047#else
1048    emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 15);
1049    emit_tg(as, MIPSI_DMTC1, tmp1, tmpnum);
1050    emit_tsi(as, MIPSI_LD, tmp1, dest, (int32_t)offsetof(Node, key.u64));
1051  } else {
1052    emit_branch(as, MIPSI_BEQ, tmp1, cmp64, l_end);
1053    emit_tsi(as, MIPSI_LD, tmp1, dest, (int32_t)offsetof(Node, key.u64));
1054  }
1055  *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
1056  if (!isk && irt_isaddr(kt)) {
1057    type = ra_allock(as, (int64_t)irt_toitype(kt) << 47, allow);
1058    emit_dst(as, MIPSI_DADDU, tmp2, key, type);
1059    rset_clear(allow, type);
1060  }
1061#endif
1062
1063  /* Load main position relative to tab->node into dest. */
1064  khash = isk ? ir_khash(as, irkey) : 1;
1065  if (khash == 0) {
1066    emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
1067  } else {
1068    Reg tmphash = tmp1;
1069    if (isk)
1070      tmphash = ra_allock(as, khash, allow);
1071    emit_dst(as, MIPSI_AADDU, dest, dest, tmp1);
1072    lj_assertA(sizeof(Node) == 24, "bad Node size");
1073    emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1);
1074    emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3);
1075    emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5);
1076    emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash);
1077    emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
1078    emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
1079    if (isk) {
1080      /* Nothing to do. */
1081    } else if (irt_isstr(kt)) {
1082      emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, sid));
1083    } else {  /* Must match with hash*() in lj_tab.c. */
1084      emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2);
1085      emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31);
1086      emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2);
1087      emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31);
1088      emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest);
1089#if LJ_32
1090      if (LJ_SOFTFP ? (irkey[1].o == IR_HIOP) : irt_isnum(kt)) {
1091	emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
1092	if ((as->flags & JIT_F_MIPSXXR2)) {
1093	  emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
1094	} else {
1095	  emit_dst(as, MIPSI_OR, dest, dest, tmp1);
1096	  emit_dta(as, MIPSI_SLL, tmp1, tmp1, HASH_ROT1);
1097	  emit_dta(as, MIPSI_SRL, dest, tmp1, (-HASH_ROT1)&31);
1098	}
1099	emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
1100#if LJ_SOFTFP
1101	emit_ds(as, MIPSI_MOVE, tmp1, type);
1102	emit_ds(as, MIPSI_MOVE, tmp2, key);
1103#else
1104	emit_tg(as, MIPSI_MFC1, tmp2, key);
1105	emit_tg(as, MIPSI_MFC1, tmp1, key+1);
1106#endif
1107      } else {
1108	emit_dst(as, MIPSI_XOR, tmp2, key, tmp1);
1109	emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31);
1110	emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow));
1111      }
1112#else
1113      emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
1114      emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
1115      if (irt_isnum(kt)) {
1116	emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
1117	emit_dta(as, MIPSI_DSRA32, tmp1, LJ_SOFTFP ? key : tmp1, 0);
1118	emit_dta(as, MIPSI_SLL, tmp2, LJ_SOFTFP ? key : tmp1, 0);
1119#if !LJ_SOFTFP
1120	emit_tg(as, MIPSI_DMFC1, tmp1, key);
1121#endif
1122      } else {
1123	checkmclim(as);
1124	emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 0);
1125	emit_dta(as, MIPSI_SLL, tmp2, key, 0);
1126	emit_dst(as, MIPSI_DADDU, tmp1, key, type);
1127      }
1128#endif
1129    }
1130  }
1131}
1132
1133static void asm_hrefk(ASMState *as, IRIns *ir)
1134{
1135  IRIns *kslot = IR(ir->op2);
1136  IRIns *irkey = IR(kslot->op1);
1137  int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
1138  int32_t kofs = ofs + (int32_t)offsetof(Node, key);
1139  Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
1140  Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
1141  RegSet allow = rset_exclude(RSET_GPR, node);
1142  Reg idx = node;
1143#if LJ_32
1144  Reg key = RID_NONE, type = RID_TMP;
1145  int32_t lo, hi;
1146#else
1147  Reg key = ra_scratch(as, allow);
1148  int64_t k;
1149#endif
1150  lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
1151  if (ofs > 32736) {
1152    idx = dest;
1153    rset_clear(allow, dest);
1154    kofs = (int32_t)offsetof(Node, key);
1155  } else if (ra_hasreg(dest)) {
1156    emit_tsi(as, MIPSI_AADDIU, dest, node, ofs);
1157  }
1158#if LJ_32
1159  if (!irt_ispri(irkey->t)) {
1160    key = ra_scratch(as, allow);
1161    rset_clear(allow, key);
1162  }
1163  if (irt_isnum(irkey->t)) {
1164    lo = (int32_t)ir_knum(irkey)->u32.lo;
1165    hi = (int32_t)ir_knum(irkey)->u32.hi;
1166  } else {
1167    lo = irkey->i;
1168    hi = irt_toitype(irkey->t);
1169    if (!ra_hasreg(key))
1170      goto nolo;
1171  }
1172  asm_guard(as, MIPSI_BNE, key, lo ? ra_allock(as, lo, allow) : RID_ZERO);
1173nolo:
1174  asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO);
1175  if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0));
1176  emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4));
1177#else
1178  if (irt_ispri(irkey->t)) {
1179    lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type");
1180    k = ~((int64_t)~irt_toitype(irkey->t) << 47);
1181  } else if (irt_isnum(irkey->t)) {
1182    k = (int64_t)ir_knum(irkey)->u64;
1183  } else {
1184    k = ((int64_t)irt_toitype(irkey->t) << 47) | (int64_t)ir_kgc(irkey);
1185  }
1186  asm_guard(as, MIPSI_BNE, key, ra_allock(as, k, allow));
1187  emit_tsi(as, MIPSI_LD, key, idx, kofs);
1188#endif
1189  if (ofs > 32736)
1190    emit_tsi(as, MIPSI_AADDU, dest, node, ra_allock(as, ofs, allow));
1191}
1192
1193static void asm_uref(ASMState *as, IRIns *ir)
1194{
1195  Reg dest = ra_dest(as, ir, RSET_GPR);
1196  int guarded = (irt_t(ir->t) & (IRT_GUARD|IRT_TYPE)) == (IRT_GUARD|IRT_PGC);
1197  if (irref_isk(ir->op1) && !guarded) {
1198    GCfunc *fn = ir_kfunc(IR(ir->op1));
1199    MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
1200    emit_lsptr(as, MIPSI_AL, dest, v, RSET_GPR);
1201  } else {
1202    if (guarded)
1203      asm_guard(as, ir->o == IR_UREFC ? MIPSI_BEQ : MIPSI_BNE, RID_TMP, RID_ZERO);
1204    if (ir->o == IR_UREFC)
1205      emit_tsi(as, MIPSI_AADDIU, dest, dest, (int32_t)offsetof(GCupval, tv));
1206    else
1207      emit_tsi(as, MIPSI_AL, dest, dest, (int32_t)offsetof(GCupval, v));
1208    if (guarded)
1209      emit_tsi(as, MIPSI_LBU, RID_TMP, dest, (int32_t)offsetof(GCupval, closed));
1210    if (irref_isk(ir->op1)) {
1211      GCfunc *fn = ir_kfunc(IR(ir->op1));
1212      GCobj *o = gcref(fn->l.uvptr[(ir->op2 >> 8)]);
1213      emit_loada(as, dest, o);
1214    } else {
1215      emit_tsi(as, MIPSI_AL, dest, ra_alloc1(as, ir->op1, RSET_GPR),
1216	       (int32_t)offsetof(GCfuncL, uvptr) +
1217	       (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
1218    }
1219  }
1220}
1221
1222static void asm_fref(ASMState *as, IRIns *ir)
1223{
1224  UNUSED(as); UNUSED(ir);
1225  lj_assertA(!ra_used(ir), "unfused FREF");
1226}
1227
1228static void asm_strref(ASMState *as, IRIns *ir)
1229{
1230#if LJ_32
1231  Reg dest = ra_dest(as, ir, RSET_GPR);
1232  IRRef ref = ir->op2, refk = ir->op1;
1233  int32_t ofs = (int32_t)sizeof(GCstr);
1234  Reg r;
1235  if (irref_isk(ref)) {
1236    IRRef tmp = refk; refk = ref; ref = tmp;
1237  } else if (!irref_isk(refk)) {
1238    Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1239    IRIns *irr = IR(ir->op2);
1240    if (ra_hasreg(irr->r)) {
1241      ra_noweak(as, irr->r);
1242      right = irr->r;
1243    } else if (mayfuse(as, irr->op2) &&
1244	       irr->o == IR_ADD && irref_isk(irr->op2) &&
1245	       checki16(ofs + IR(irr->op2)->i)) {
1246      ofs += IR(irr->op2)->i;
1247      right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left));
1248    } else {
1249      right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left));
1250    }
1251    emit_tsi(as, MIPSI_ADDIU, dest, dest, ofs);
1252    emit_dst(as, MIPSI_ADDU, dest, left, right);
1253    return;
1254  }
1255  r = ra_alloc1(as, ref, RSET_GPR);
1256  ofs += IR(refk)->i;
1257  if (checki16(ofs))
1258    emit_tsi(as, MIPSI_ADDIU, dest, r, ofs);
1259  else
1260    emit_dst(as, MIPSI_ADDU, dest, r,
1261	     ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
1262#else
1263  RegSet allow = RSET_GPR;
1264  Reg dest = ra_dest(as, ir, allow);
1265  Reg base = ra_alloc1(as, ir->op1, allow);
1266  IRIns *irr = IR(ir->op2);
1267  int32_t ofs = sizeof(GCstr);
1268  rset_clear(allow, base);
1269  if (irref_isk(ir->op2) && checki16(ofs + irr->i)) {
1270    emit_tsi(as, MIPSI_DADDIU, dest, base, ofs + irr->i);
1271  } else {
1272    emit_tsi(as, MIPSI_DADDIU, dest, dest, ofs);
1273    emit_dst(as, MIPSI_DADDU, dest, base, ra_alloc1(as, ir->op2, allow));
1274  }
1275#endif
1276}
1277
1278/* -- Loads and stores ---------------------------------------------------- */
1279
1280static MIPSIns asm_fxloadins(ASMState *as, IRIns *ir)
1281{
1282  UNUSED(as);
1283  switch (irt_type(ir->t)) {
1284  case IRT_I8: return MIPSI_LB;
1285  case IRT_U8: return MIPSI_LBU;
1286  case IRT_I16: return MIPSI_LH;
1287  case IRT_U16: return MIPSI_LHU;
1288  case IRT_NUM:
1289    lj_assertA(!LJ_SOFTFP32, "unsplit FP op");
1290    if (!LJ_SOFTFP) return MIPSI_LDC1;
1291  /* fallthrough */
1292  case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_LWC1;
1293  /* fallthrough */
1294  default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_LD : MIPSI_LW;
1295  }
1296}
1297
1298static MIPSIns asm_fxstoreins(ASMState *as, IRIns *ir)
1299{
1300  UNUSED(as);
1301  switch (irt_type(ir->t)) {
1302  case IRT_I8: case IRT_U8: return MIPSI_SB;
1303  case IRT_I16: case IRT_U16: return MIPSI_SH;
1304  case IRT_NUM:
1305    lj_assertA(!LJ_SOFTFP32, "unsplit FP op");
1306    if (!LJ_SOFTFP) return MIPSI_SDC1;
1307  /* fallthrough */
1308  case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_SWC1;
1309  /* fallthrough */
1310  default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_SD : MIPSI_SW;
1311  }
1312}
1313
1314static void asm_fload(ASMState *as, IRIns *ir)
1315{
1316  Reg dest = ra_dest(as, ir, RSET_GPR);
1317  MIPSIns mi = asm_fxloadins(as, ir);
1318  Reg idx;
1319  int32_t ofs;
1320  if (ir->op1 == REF_NIL) {  /* FLOAD from GG_State with offset. */
1321    idx = RID_JGL;
1322    ofs = (ir->op2 << 2) - 32768 - GG_OFS(g);
1323  } else {
1324    idx = ra_alloc1(as, ir->op1, RSET_GPR);
1325    if (ir->op2 == IRFL_TAB_ARRAY) {
1326      ofs = asm_fuseabase(as, ir->op1);
1327      if (ofs) {  /* Turn the t->array load into an add for colocated arrays. */
1328	emit_tsi(as, MIPSI_AADDIU, dest, idx, ofs);
1329	return;
1330      }
1331    }
1332    ofs = field_ofs[ir->op2];
1333    lj_assertA(!irt_isfp(ir->t), "bad FP FLOAD");
1334  }
1335  emit_tsi(as, mi, dest, idx, ofs);
1336}
1337
1338static void asm_fstore(ASMState *as, IRIns *ir)
1339{
1340  if (ir->r != RID_SINK) {
1341    Reg src = ra_alloc1z(as, ir->op2, RSET_GPR);
1342    IRIns *irf = IR(ir->op1);
1343    Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
1344    int32_t ofs = field_ofs[irf->op2];
1345    MIPSIns mi = asm_fxstoreins(as, ir);
1346    lj_assertA(!irt_isfp(ir->t), "bad FP FSTORE");
1347    emit_tsi(as, mi, src, idx, ofs);
1348  }
1349}
1350
1351static void asm_xload(ASMState *as, IRIns *ir)
1352{
1353  Reg dest = ra_dest(as, ir,
1354    (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1355  lj_assertA(LJ_TARGET_UNALIGNED || !(ir->op2 & IRXLOAD_UNALIGNED),
1356	     "unaligned XLOAD");
1357  asm_fusexref(as, asm_fxloadins(as, ir), dest, ir->op1, RSET_GPR, 0);
1358}
1359
1360static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
1361{
1362  if (ir->r != RID_SINK) {
1363    Reg src = ra_alloc1z(as, ir->op2,
1364      (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1365    asm_fusexref(as, asm_fxstoreins(as, ir), src, ir->op1,
1366		 rset_exclude(RSET_GPR, src), ofs);
1367  }
1368}
1369
1370#define asm_xstore(as, ir)	asm_xstore_(as, ir, 0)
1371
1372static void asm_ahuvload(ASMState *as, IRIns *ir)
1373{
1374  int hiop = (LJ_SOFTFP32 && (ir+1)->o == IR_HIOP);
1375  Reg dest = RID_NONE, type = RID_TMP, idx;
1376  RegSet allow = RSET_GPR;
1377  int32_t ofs = 0;
1378  IRType1 t = ir->t;
1379  if (hiop) {
1380    t.irt = IRT_NUM;
1381    if (ra_used(ir+1)) {
1382      type = ra_dest(as, ir+1, allow);
1383      rset_clear(allow, type);
1384    }
1385  }
1386  if (ra_used(ir)) {
1387    lj_assertA((LJ_SOFTFP32 ? 0 : irt_isnum(ir->t)) ||
1388	       irt_isint(ir->t) || irt_isaddr(ir->t),
1389	       "bad load type %d", irt_type(ir->t));
1390    dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
1391    rset_clear(allow, dest);
1392#if LJ_64
1393    if (irt_isaddr(t))
1394      emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
1395    else if (irt_isint(t))
1396      emit_dta(as, MIPSI_SLL, dest, dest, 0);
1397#endif
1398  }
1399  idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1400  if (ir->o == IR_VLOAD) ofs += 8 * ir->op2;
1401  rset_clear(allow, idx);
1402  if (irt_isnum(t)) {
1403    asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1404    emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
1405  } else {
1406    asm_guard(as, MIPSI_BNE, type,
1407	      ra_allock(as, (int32_t)irt_toitype(t), allow));
1408  }
1409#if LJ_32
1410  if (ra_hasreg(dest)) {
1411    if (!LJ_SOFTFP && irt_isnum(t))
1412      emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
1413    else
1414      emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0));
1415  }
1416  emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4));
1417#else
1418  if (ra_hasreg(dest)) {
1419    if (!LJ_SOFTFP && irt_isnum(t)) {
1420      emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
1421      dest = type;
1422    }
1423  } else {
1424    dest = type;
1425  }
1426  emit_dta(as, MIPSI_DSRA32, type, dest, 15);
1427  emit_tsi(as, MIPSI_LD, dest, idx, ofs);
1428#endif
1429}
1430
1431static void asm_ahustore(ASMState *as, IRIns *ir)
1432{
1433  RegSet allow = RSET_GPR;
1434  Reg idx, src = RID_NONE, type = RID_NONE;
1435  int32_t ofs = 0;
1436  if (ir->r == RID_SINK)
1437    return;
1438  if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
1439    src = ra_alloc1(as, ir->op2, LJ_SOFTFP ? RSET_GPR : RSET_FPR);
1440    idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1441    emit_hsi(as, LJ_SOFTFP ? MIPSI_SD : MIPSI_SDC1, src, idx, ofs);
1442  } else {
1443#if LJ_32
1444    if (!irt_ispri(ir->t)) {
1445      src = ra_alloc1(as, ir->op2, allow);
1446      rset_clear(allow, src);
1447    }
1448    if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
1449      type = ra_alloc1(as, (ir+1)->op2, allow);
1450    else
1451      type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1452    rset_clear(allow, type);
1453    idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1454    if (ra_hasreg(src))
1455      emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0));
1456    emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4));
1457#else
1458    Reg tmp = RID_TMP;
1459    if (irt_ispri(ir->t)) {
1460      tmp = ra_allock(as, ~((int64_t)~irt_toitype(ir->t) << 47), allow);
1461      rset_clear(allow, tmp);
1462    } else {
1463      src = ra_alloc1(as, ir->op2, allow);
1464      rset_clear(allow, src);
1465      type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47, allow);
1466      rset_clear(allow, type);
1467    }
1468    idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1469    emit_tsi(as, MIPSI_SD, tmp, idx, ofs);
1470    if (ra_hasreg(src)) {
1471      if (irt_isinteger(ir->t)) {
1472	emit_dst(as, MIPSI_DADDU, tmp, tmp, type);
1473	emit_tsml(as, MIPSI_DEXT, tmp, src, 31, 0);
1474      } else {
1475	emit_dst(as, MIPSI_DADDU, tmp, src, type);
1476      }
1477    }
1478#endif
1479  }
1480}
1481
1482static void asm_sload(ASMState *as, IRIns *ir)
1483{
1484  Reg dest = RID_NONE, type = RID_NONE, base;
1485  RegSet allow = RSET_GPR;
1486  IRType1 t = ir->t;
1487#if LJ_32
1488  int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1489  int hiop = (LJ_SOFTFP32 && (ir+1)->o == IR_HIOP);
1490  if (hiop)
1491    t.irt = IRT_NUM;
1492#else
1493  int32_t ofs = 8*((int32_t)ir->op1-2);
1494#endif
1495  lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1496	     "bad parent SLOAD");  /* Handled by asm_head_side(). */
1497  lj_assertA(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1498	     "inconsistent SLOAD variant");
1499#if LJ_SOFTFP32
1500  lj_assertA(!(ir->op2 & IRSLOAD_CONVERT),
1501	     "unsplit SLOAD convert");  /* Handled by LJ_SOFTFP SPLIT. */
1502  if (hiop && ra_used(ir+1)) {
1503    type = ra_dest(as, ir+1, allow);
1504    rset_clear(allow, type);
1505  }
1506#else
1507  if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1508    dest = ra_scratch(as, LJ_SOFTFP ? allow : RSET_FPR);
1509    asm_tointg(as, ir, dest);
1510    t.irt = IRT_NUM;  /* Continue with a regular number type check. */
1511  } else
1512#endif
1513  if (ra_used(ir)) {
1514    lj_assertA((LJ_SOFTFP32 ? 0 : irt_isnum(ir->t)) ||
1515	       irt_isint(ir->t) || irt_isaddr(ir->t),
1516	       "bad SLOAD type %d", irt_type(ir->t));
1517    dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
1518    rset_clear(allow, dest);
1519    base = ra_alloc1(as, REF_BASE, allow);
1520    rset_clear(allow, base);
1521    if (!LJ_SOFTFP32 && (ir->op2 & IRSLOAD_CONVERT)) {
1522      if (irt_isint(t)) {
1523	Reg tmp = ra_scratch(as, LJ_SOFTFP ? RSET_GPR : RSET_FPR);
1524#if LJ_SOFTFP
1525	ra_evictset(as, rset_exclude(RSET_SCRATCH, dest));
1526	ra_destreg(as, ir, RID_RET);
1527	emit_call(as, (void *)lj_ir_callinfo[IRCALL_softfp_d2i].func, 0);
1528	if (tmp != REGARG_FIRSTGPR)
1529	  emit_move(as, REGARG_FIRSTGPR, tmp);
1530#else
1531	emit_tg(as, MIPSI_MFC1, dest, tmp);
1532	emit_fg(as, MIPSI_TRUNC_W_D, tmp, tmp);
1533#endif
1534	dest = tmp;
1535	t.irt = IRT_NUM;  /* Check for original type. */
1536      } else {
1537	Reg tmp = ra_scratch(as, RSET_GPR);
1538#if LJ_SOFTFP
1539	ra_evictset(as, rset_exclude(RSET_SCRATCH, dest));
1540	ra_destreg(as, ir, RID_RET);
1541	emit_call(as, (void *)lj_ir_callinfo[IRCALL_softfp_i2d].func, 0);
1542	emit_dta(as, MIPSI_SLL, REGARG_FIRSTGPR, tmp, 0);
1543#else
1544	emit_fg(as, MIPSI_CVT_D_W, dest, dest);
1545	emit_tg(as, MIPSI_MTC1, tmp, dest);
1546#endif
1547	dest = tmp;
1548	t.irt = IRT_INT;  /* Check for original type. */
1549      }
1550    }
1551#if LJ_64
1552    else if (irt_isaddr(t)) {
1553      /* Clear type from pointers. */
1554      emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
1555    } else if (irt_isint(t) && (ir->op2 & IRSLOAD_TYPECHECK)) {
1556      /* Sign-extend integers. */
1557      emit_dta(as, MIPSI_SLL, dest, dest, 0);
1558    }
1559#endif
1560    goto dotypecheck;
1561  }
1562  base = ra_alloc1(as, REF_BASE, allow);
1563  rset_clear(allow, base);
1564dotypecheck:
1565#if LJ_32
1566  if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1567    if (ra_noreg(type))
1568      type = RID_TMP;
1569    if (irt_isnum(t)) {
1570      asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1571      emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
1572    } else {
1573      Reg ktype = ra_allock(as, (ir->op2 & IRSLOAD_KEYINDEX) ? LJ_KEYINDEX : irt_toitype(t), allow);
1574      asm_guard(as, MIPSI_BNE, type, ktype);
1575    }
1576  }
1577  if (ra_hasreg(dest)) {
1578    if (!LJ_SOFTFP && irt_isnum(t))
1579      emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1580    else
1581      emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0));
1582  }
1583  if (ra_hasreg(type))
1584    emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4));
1585#else
1586  if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1587    type = dest < RID_MAX_GPR ? dest : RID_TMP;
1588    if (irt_ispri(t)) {
1589      asm_guard(as, MIPSI_BNE, type,
1590		ra_allock(as, ~((int64_t)~irt_toitype(t) << 47) , allow));
1591    } else if ((ir->op2 & IRSLOAD_KEYINDEX)) {
1592      asm_guard(as, MIPSI_BNE, RID_TMP,
1593		ra_allock(as, (int32_t)LJ_KEYINDEX, allow));
1594      emit_dta(as, MIPSI_DSRA32, RID_TMP, type, 0);
1595    } else {
1596      if (irt_isnum(t)) {
1597	asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1598	emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)LJ_TISNUM);
1599	if (!LJ_SOFTFP && ra_hasreg(dest))
1600	  emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1601      } else {
1602	asm_guard(as, MIPSI_BNE, RID_TMP,
1603		  ra_allock(as, (int32_t)irt_toitype(t), allow));
1604      }
1605      emit_dta(as, MIPSI_DSRA32, RID_TMP, type, 15);
1606    }
1607    emit_tsi(as, MIPSI_LD, type, base, ofs);
1608  } else if (ra_hasreg(dest)) {
1609    if (!LJ_SOFTFP && irt_isnum(t))
1610      emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1611    else
1612      emit_tsi(as, irt_isint(t) ? MIPSI_LW : MIPSI_LD, dest, base,
1613	       ofs ^ ((LJ_BE && irt_isint(t)) ? 4 : 0));
1614  }
1615#endif
1616}
1617
1618/* -- Allocations --------------------------------------------------------- */
1619
1620#if LJ_HASFFI
1621static void asm_cnew(ASMState *as, IRIns *ir)
1622{
1623  CTState *cts = ctype_ctsG(J2G(as->J));
1624  CTypeID id = (CTypeID)IR(ir->op1)->i;
1625  CTSize sz;
1626  CTInfo info = lj_ctype_info(cts, id, &sz);
1627  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1628  IRRef args[4];
1629  RegSet drop = RSET_SCRATCH;
1630  lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1631	     "bad CNEW/CNEWI operands");
1632
1633  as->gcsteps++;
1634  if (ra_hasreg(ir->r))
1635    rset_clear(drop, ir->r);  /* Dest reg handled below. */
1636  ra_evictset(as, drop);
1637  if (ra_used(ir))
1638    ra_destreg(as, ir, RID_RET);  /* GCcdata * */
1639
1640  /* Initialize immutable cdata object. */
1641  if (ir->o == IR_CNEWI) {
1642    RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1643#if LJ_32
1644    int32_t ofs = sizeof(GCcdata);
1645    if (sz == 8) {
1646      ofs += 4;
1647      lj_assertA((ir+1)->o == IR_HIOP, "expected HIOP for CNEWI");
1648      if (LJ_LE) ir++;
1649    }
1650    for (;;) {
1651      Reg r = ra_alloc1z(as, ir->op2, allow);
1652      emit_tsi(as, MIPSI_SW, r, RID_RET, ofs);
1653      rset_clear(allow, r);
1654      if (ofs == sizeof(GCcdata)) break;
1655      ofs -= 4; if (LJ_BE) ir++; else ir--;
1656    }
1657#else
1658    emit_tsi(as, sz == 8 ? MIPSI_SD : MIPSI_SW, ra_alloc1(as, ir->op2, allow),
1659	     RID_RET, sizeof(GCcdata));
1660#endif
1661    lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1662  } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
1663    ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1664    args[0] = ASMREF_L;     /* lua_State *L */
1665    args[1] = ir->op1;      /* CTypeID id   */
1666    args[2] = ir->op2;      /* CTSize sz    */
1667    args[3] = ASMREF_TMP1;  /* CTSize align */
1668    asm_gencall(as, ci, args);
1669    emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1670    return;
1671  }
1672
1673  /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1674  emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1675  emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1676  emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA);
1677  emit_ti(as, MIPSI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */
1678  args[0] = ASMREF_L;     /* lua_State *L */
1679  args[1] = ASMREF_TMP1;  /* MSize size   */
1680  asm_gencall(as, ci, args);
1681  ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1682	       ra_releasetmp(as, ASMREF_TMP1));
1683}
1684#endif
1685
1686/* -- Write barriers ------------------------------------------------------ */
1687
1688static void asm_tbar(ASMState *as, IRIns *ir)
1689{
1690  Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1691  Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1692  Reg link = RID_TMP;
1693  MCLabel l_end = emit_label(as);
1694  emit_tsi(as, MIPSI_AS, link, tab, (int32_t)offsetof(GCtab, gclist));
1695  emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked));
1696  emit_setgl(as, tab, gc.grayagain);
1697  emit_getgl(as, link, gc.grayagain);
1698  emit_dst(as, MIPSI_XOR, mark, mark, RID_TMP);  /* Clear black bit. */
1699  emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1700  emit_tsi(as, MIPSI_ANDI, RID_TMP, mark, LJ_GC_BLACK);
1701  emit_tsi(as, MIPSI_LBU, mark, tab, (int32_t)offsetof(GCtab, marked));
1702}
1703
1704static void asm_obar(ASMState *as, IRIns *ir)
1705{
1706  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1707  IRRef args[2];
1708  MCLabel l_end;
1709  Reg obj, val, tmp;
1710  /* No need for other object barriers (yet). */
1711  lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1712  ra_evictset(as, RSET_SCRATCH);
1713  l_end = emit_label(as);
1714  args[0] = ASMREF_TMP1;  /* global_State *g */
1715  args[1] = ir->op1;      /* TValue *tv      */
1716  asm_gencall(as, ci, args);
1717  emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1718  obj = IR(ir->op1)->r;
1719  tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1720  emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1721  emit_tsi(as, MIPSI_ANDI, tmp, tmp, LJ_GC_BLACK);
1722  emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1723  emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, LJ_GC_WHITES);
1724  val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1725  emit_tsi(as, MIPSI_LBU, tmp, obj,
1726	   (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1727  emit_tsi(as, MIPSI_LBU, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1728}
1729
1730/* -- Arithmetic and logic operations ------------------------------------- */
1731
1732#if !LJ_SOFTFP
1733static void asm_fparith(ASMState *as, IRIns *ir, MIPSIns mi)
1734{
1735  Reg dest = ra_dest(as, ir, RSET_FPR);
1736  Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1737  right = (left >> 8); left &= 255;
1738  emit_fgh(as, mi, dest, left, right);
1739}
1740
1741static void asm_fpunary(ASMState *as, IRIns *ir, MIPSIns mi)
1742{
1743  Reg dest = ra_dest(as, ir, RSET_FPR);
1744  Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1745  emit_fg(as, mi, dest, left);
1746}
1747#endif
1748
1749#if !LJ_SOFTFP32
1750static void asm_fpmath(ASMState *as, IRIns *ir)
1751{
1752#if !LJ_SOFTFP
1753  if (ir->op2 <= IRFPM_TRUNC)
1754    asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
1755  else if (ir->op2 == IRFPM_SQRT)
1756    asm_fpunary(as, ir, MIPSI_SQRT_D);
1757  else
1758#endif
1759    asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1760}
1761#endif
1762
1763#if !LJ_SOFTFP
1764#define asm_fpadd(as, ir)	asm_fparith(as, ir, MIPSI_ADD_D)
1765#define asm_fpsub(as, ir)	asm_fparith(as, ir, MIPSI_SUB_D)
1766#define asm_fpmul(as, ir)	asm_fparith(as, ir, MIPSI_MUL_D)
1767#elif LJ_64  /* && LJ_SOFTFP */
1768#define asm_fpadd(as, ir)	asm_callid(as, ir, IRCALL_softfp_add)
1769#define asm_fpsub(as, ir)	asm_callid(as, ir, IRCALL_softfp_sub)
1770#define asm_fpmul(as, ir)	asm_callid(as, ir, IRCALL_softfp_mul)
1771#endif
1772
1773static void asm_add(ASMState *as, IRIns *ir)
1774{
1775  IRType1 t = ir->t;
1776#if !LJ_SOFTFP32
1777  if (irt_isnum(t)) {
1778    asm_fpadd(as, ir);
1779  } else
1780#endif
1781  {
1782    /* TODO MIPSR6: Fuse ADD(BSHL(a,1-4),b) or ADD(ADD(a,a),b) to MIPSI_ALSA. */
1783    Reg dest = ra_dest(as, ir, RSET_GPR);
1784    Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1785    if (irref_isk(ir->op2)) {
1786      intptr_t k = get_kval(as, ir->op2);
1787      if (checki16(k)) {
1788	emit_tsi(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDIU : MIPSI_ADDIU, dest,
1789		 left, k);
1790	return;
1791      }
1792    }
1793    right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1794    emit_dst(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDU : MIPSI_ADDU, dest,
1795	     left, right);
1796  }
1797}
1798
1799static void asm_sub(ASMState *as, IRIns *ir)
1800{
1801#if !LJ_SOFTFP32
1802  if (irt_isnum(ir->t)) {
1803    asm_fpsub(as, ir);
1804  } else
1805#endif
1806  {
1807    Reg dest = ra_dest(as, ir, RSET_GPR);
1808    Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1809    right = (left >> 8); left &= 255;
1810    emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
1811	     left, right);
1812  }
1813}
1814
1815static void asm_mul(ASMState *as, IRIns *ir)
1816{
1817#if !LJ_SOFTFP32
1818  if (irt_isnum(ir->t)) {
1819    asm_fpmul(as, ir);
1820  } else
1821#endif
1822  {
1823    Reg dest = ra_dest(as, ir, RSET_GPR);
1824    Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1825    right = (left >> 8); left &= 255;
1826    if (LJ_64 && irt_is64(ir->t)) {
1827#if !LJ_TARGET_MIPSR6
1828      emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1829      emit_dst(as, MIPSI_DMULT, 0, left, right);
1830#else
1831      emit_dst(as, MIPSI_DMUL, dest, left, right);
1832#endif
1833    } else {
1834      emit_dst(as, MIPSI_MUL, dest, left, right);
1835    }
1836  }
1837}
1838
1839#if !LJ_SOFTFP32
1840static void asm_fpdiv(ASMState *as, IRIns *ir)
1841{
1842#if !LJ_SOFTFP
1843    asm_fparith(as, ir, MIPSI_DIV_D);
1844#else
1845    asm_callid(as, ir, IRCALL_softfp_div);
1846#endif
1847}
1848#endif
1849
1850static void asm_neg(ASMState *as, IRIns *ir)
1851{
1852#if !LJ_SOFTFP
1853  if (irt_isnum(ir->t)) {
1854    asm_fpunary(as, ir, MIPSI_NEG_D);
1855  } else
1856#elif LJ_64  /* && LJ_SOFTFP */
1857  if (irt_isnum(ir->t)) {
1858    Reg dest = ra_dest(as, ir, RSET_GPR);
1859    Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1860    emit_dst(as, MIPSI_XOR, dest, left,
1861	    ra_allock(as, 0x8000000000000000ll, rset_exclude(RSET_GPR, dest)));
1862  } else
1863#endif
1864  {
1865    Reg dest = ra_dest(as, ir, RSET_GPR);
1866    Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1867    emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
1868	     RID_ZERO, left);
1869  }
1870}
1871
1872#if !LJ_SOFTFP
1873#define asm_abs(as, ir)		asm_fpunary(as, ir, MIPSI_ABS_D)
1874#elif LJ_64   /* && LJ_SOFTFP */
1875static void asm_abs(ASMState *as, IRIns *ir)
1876{
1877  Reg dest = ra_dest(as, ir, RSET_GPR);
1878  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1879  emit_tsml(as, MIPSI_DEXTM, dest, left, 30, 0);
1880}
1881#endif
1882
1883static void asm_arithov(ASMState *as, IRIns *ir)
1884{
1885  /* TODO MIPSR6: bovc/bnvc. Caveat: no delay slot to load RID_TMP. */
1886  Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
1887  lj_assertA(!irt_is64(ir->t), "bad usage");
1888  if (irref_isk(ir->op2)) {
1889    int k = IR(ir->op2)->i;
1890    if (ir->o == IR_SUBOV) k = (int)(~(unsigned int)k+1u);
1891    if (checki16(k)) {  /* (dest < left) == (k >= 0 ? 1 : 0) */
1892      left = ra_alloc1(as, ir->op1, RSET_GPR);
1893      asm_guard(as, k >= 0 ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1894      emit_dst(as, MIPSI_SLT, RID_TMP, dest, dest == left ? RID_TMP : left);
1895      emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1896      if (dest == left) emit_move(as, RID_TMP, left);
1897      return;
1898    }
1899  }
1900  left = ra_alloc2(as, ir, RSET_GPR);
1901  right = (left >> 8); left &= 255;
1902  tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1903						 right), dest));
1904  asm_guard(as, MIPSI_BLTZ, RID_TMP, 0);
1905  emit_dst(as, MIPSI_AND, RID_TMP, RID_TMP, tmp);
1906  if (ir->o == IR_ADDOV) {  /* ((dest^left) & (dest^right)) < 0 */
1907    emit_dst(as, MIPSI_XOR, RID_TMP, dest, dest == right ? RID_TMP : right);
1908  } else {  /* ((dest^left) & (dest^~right)) < 0 */
1909    emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, dest);
1910    emit_dst(as, MIPSI_NOR, RID_TMP, dest == right ? RID_TMP : right, RID_ZERO);
1911  }
1912  emit_dst(as, MIPSI_XOR, tmp, dest, dest == left ? RID_TMP : left);
1913  emit_dst(as, ir->o == IR_ADDOV ? MIPSI_ADDU : MIPSI_SUBU, dest, left, right);
1914  if (dest == left || dest == right)
1915    emit_move(as, RID_TMP, dest == left ? left : right);
1916}
1917
1918#define asm_addov(as, ir)	asm_arithov(as, ir)
1919#define asm_subov(as, ir)	asm_arithov(as, ir)
1920
1921static void asm_mulov(ASMState *as, IRIns *ir)
1922{
1923  Reg dest = ra_dest(as, ir, RSET_GPR);
1924  Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
1925  right = (left >> 8); left &= 255;
1926  tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1927						 right), dest));
1928  asm_guard(as, MIPSI_BNE, RID_TMP, tmp);
1929  emit_dta(as, MIPSI_SRA, RID_TMP, dest, 31);
1930#if !LJ_TARGET_MIPSR6
1931  emit_dst(as, MIPSI_MFHI, tmp, 0, 0);
1932  emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1933  emit_dst(as, MIPSI_MULT, 0, left, right);
1934#else
1935  emit_dst(as, MIPSI_MUL, dest, left, right);
1936  emit_dst(as, MIPSI_MUH, tmp, left, right);
1937#endif
1938}
1939
1940#if LJ_32 && LJ_HASFFI
1941static void asm_add64(ASMState *as, IRIns *ir)
1942{
1943  Reg dest = ra_dest(as, ir, RSET_GPR);
1944  Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1945  if (irref_isk(ir->op2)) {
1946    int32_t k = IR(ir->op2)->i;
1947    if (k == 0) {
1948      emit_dst(as, MIPSI_ADDU, dest, left, RID_TMP);
1949      goto loarith;
1950    } else if (checki16(k)) {
1951      emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1952      emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1953      goto loarith;
1954    }
1955  }
1956  emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1957  right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1958  emit_dst(as, MIPSI_ADDU, dest, left, right);
1959loarith:
1960  ir--;
1961  dest = ra_dest(as, ir, RSET_GPR);
1962  left = ra_alloc1(as, ir->op1, RSET_GPR);
1963  if (irref_isk(ir->op2)) {
1964    int32_t k = IR(ir->op2)->i;
1965    if (k == 0) {
1966      if (dest != left)
1967	emit_move(as, dest, left);
1968      return;
1969    } else if (checki16(k)) {
1970      if (dest == left) {
1971	Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, left));
1972	emit_move(as, dest, tmp);
1973	dest = tmp;
1974      }
1975      emit_dst(as, MIPSI_SLTU, RID_TMP, dest, left);
1976      emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1977      return;
1978    }
1979  }
1980  right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1981  if (dest == left && dest == right) {
1982    Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1983    emit_move(as, dest, tmp);
1984    dest = tmp;
1985  }
1986  emit_dst(as, MIPSI_SLTU, RID_TMP, dest, dest == left ? right : left);
1987  emit_dst(as, MIPSI_ADDU, dest, left, right);
1988}
1989
1990static void asm_sub64(ASMState *as, IRIns *ir)
1991{
1992  Reg dest = ra_dest(as, ir, RSET_GPR);
1993  Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1994  right = (left >> 8); left &= 255;
1995  emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1996  emit_dst(as, MIPSI_SUBU, dest, left, right);
1997  ir--;
1998  dest = ra_dest(as, ir, RSET_GPR);
1999  left = ra_alloc2(as, ir, RSET_GPR);
2000  right = (left >> 8); left &= 255;
2001  if (dest == left) {
2002    Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
2003    emit_move(as, dest, tmp);
2004    dest = tmp;
2005  }
2006  emit_dst(as, MIPSI_SLTU, RID_TMP, left, dest);
2007  emit_dst(as, MIPSI_SUBU, dest, left, right);
2008}
2009
2010static void asm_neg64(ASMState *as, IRIns *ir)
2011{
2012  Reg dest = ra_dest(as, ir, RSET_GPR);
2013  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
2014  emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
2015  emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
2016  ir--;
2017  dest = ra_dest(as, ir, RSET_GPR);
2018  left = ra_alloc1(as, ir->op1, RSET_GPR);
2019  emit_dst(as, MIPSI_SLTU, RID_TMP, RID_ZERO, dest);
2020  emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
2021}
2022#endif
2023
2024static void asm_bnot(ASMState *as, IRIns *ir)
2025{
2026  Reg left, right, dest = ra_dest(as, ir, RSET_GPR);
2027  IRIns *irl = IR(ir->op1);
2028  if (mayfuse(as, ir->op1) && irl->o == IR_BOR) {
2029    left = ra_alloc2(as, irl, RSET_GPR);
2030    right = (left >> 8); left &= 255;
2031  } else {
2032    left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
2033    right = RID_ZERO;
2034  }
2035  emit_dst(as, MIPSI_NOR, dest, left, right);
2036}
2037
2038static void asm_bswap(ASMState *as, IRIns *ir)
2039{
2040  Reg dest = ra_dest(as, ir, RSET_GPR);
2041  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
2042#if LJ_32
2043  if ((as->flags & JIT_F_MIPSXXR2)) {
2044    emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
2045    emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
2046  } else {
2047    Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), dest));
2048    emit_dst(as, MIPSI_OR, dest, dest, tmp);
2049    emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
2050    emit_tsi(as, MIPSI_ANDI, dest, dest, 0xff00);
2051    emit_dta(as, MIPSI_SLL, RID_TMP, RID_TMP, 8);
2052    emit_dta(as, MIPSI_SRL, dest, left, 8);
2053    emit_tsi(as, MIPSI_ANDI, RID_TMP, left, 0xff00);
2054    emit_dst(as, MIPSI_OR, tmp, tmp, RID_TMP);
2055    emit_dta(as, MIPSI_SRL, tmp, left, 24);
2056    emit_dta(as, MIPSI_SLL, RID_TMP, left, 24);
2057  }
2058#else
2059  if (irt_is64(ir->t)) {
2060    emit_dst(as, MIPSI_DSHD, dest, 0, RID_TMP);
2061    emit_dst(as, MIPSI_DSBH, RID_TMP, 0, left);
2062  } else {
2063    emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
2064    emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
2065  }
2066#endif
2067}
2068
2069static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
2070{
2071  Reg dest = ra_dest(as, ir, RSET_GPR);
2072  Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
2073  if (irref_isk(ir->op2)) {
2074    intptr_t k = get_kval(as, ir->op2);
2075    if (checku16(k)) {
2076      emit_tsi(as, mik, dest, left, k);
2077      return;
2078    }
2079  }
2080  right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
2081  emit_dst(as, mi, dest, left, right);
2082}
2083
2084#define asm_band(as, ir)	asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI)
2085#define asm_bor(as, ir)		asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI)
2086#define asm_bxor(as, ir)	asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI)
2087
2088static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
2089{
2090  Reg dest = ra_dest(as, ir, RSET_GPR);
2091  if (irref_isk(ir->op2)) {  /* Constant shifts. */
2092    uint32_t shift = (uint32_t)IR(ir->op2)->i;
2093    if (LJ_64 && irt_is64(ir->t)) mik |= (shift & 32) ? MIPSI_D32 : MIPSI_D;
2094    emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR),
2095	     (shift & 31));
2096  } else {
2097    Reg right, left = ra_alloc2(as, ir, RSET_GPR);
2098    right = (left >> 8); left &= 255;
2099    if (LJ_64 && irt_is64(ir->t)) mi |= MIPSI_DV;
2100    emit_dst(as, mi, dest, right, left);  /* Shift amount is in rs. */
2101  }
2102}
2103
2104#define asm_bshl(as, ir)	asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL)
2105#define asm_bshr(as, ir)	asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL)
2106#define asm_bsar(as, ir)	asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA)
2107#define asm_brol(as, ir)	lj_assertA(0, "unexpected BROL")
2108
2109static void asm_bror(ASMState *as, IRIns *ir)
2110{
2111  if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
2112    asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
2113  } else {
2114    Reg dest = ra_dest(as, ir, RSET_GPR);
2115    if (irref_isk(ir->op2)) {  /* Constant shifts. */
2116      uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
2117      Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
2118      emit_rotr(as, dest, left, RID_TMP, shift);
2119    } else {
2120      Reg right, left = ra_alloc2(as, ir, RSET_GPR);
2121      right = (left >> 8); left &= 255;
2122      emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
2123      emit_dst(as, MIPSI_SRLV, dest, right, left);
2124      emit_dst(as, MIPSI_SLLV, RID_TMP, RID_TMP, left);
2125      emit_dst(as, MIPSI_SUBU, RID_TMP, ra_allock(as, 32, RSET_GPR), right);
2126    }
2127  }
2128}
2129
2130#if LJ_SOFTFP
2131static void asm_sfpmin_max(ASMState *as, IRIns *ir)
2132{
2133  CCallInfo ci = lj_ir_callinfo[(IROp)ir->o == IR_MIN ? IRCALL_lj_vm_sfmin : IRCALL_lj_vm_sfmax];
2134#if LJ_64
2135  IRRef args[2];
2136  args[0] = ir->op1;
2137  args[1] = ir->op2;
2138#else
2139  IRRef args[4];
2140  args[0^LJ_BE] = ir->op1;
2141  args[1^LJ_BE] = (ir+1)->op1;
2142  args[2^LJ_BE] = ir->op2;
2143  args[3^LJ_BE] = (ir+1)->op2;
2144#endif
2145  asm_setupresult(as, ir, &ci);
2146  emit_call(as, (void *)ci.func, 0);
2147  ci.func = NULL;
2148  asm_gencall(as, &ci, args);
2149}
2150#endif
2151
2152static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
2153{
2154  if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
2155#if LJ_SOFTFP
2156    asm_sfpmin_max(as, ir);
2157#else
2158    Reg dest = ra_dest(as, ir, RSET_FPR);
2159    Reg right, left = ra_alloc2(as, ir, RSET_FPR);
2160    right = (left >> 8); left &= 255;
2161#if !LJ_TARGET_MIPSR6
2162    if (dest == left) {
2163      emit_fg(as, MIPSI_MOVF_D, dest, right);
2164    } else {
2165      emit_fg(as, MIPSI_MOVT_D, dest, left);
2166      if (dest != right) emit_fg(as, MIPSI_MOV_D, dest, right);
2167    }
2168    emit_fgh(as, MIPSI_C_OLT_D, 0, ismax ? right : left, ismax ? left : right);
2169#else
2170    emit_fgh(as, ismax ? MIPSI_MAX_D : MIPSI_MIN_D, dest, left, right);
2171#endif
2172#endif
2173  } else {
2174    Reg dest = ra_dest(as, ir, RSET_GPR);
2175    Reg right, left = ra_alloc2(as, ir, RSET_GPR);
2176    right = (left >> 8); left &= 255;
2177    if (left == right) {
2178      if (dest != left) emit_move(as, dest, left);
2179    } else {
2180#if !LJ_TARGET_MIPSR6
2181      if (dest == left) {
2182	emit_dst(as, MIPSI_MOVN, dest, right, RID_TMP);
2183      } else {
2184	emit_dst(as, MIPSI_MOVZ, dest, left, RID_TMP);
2185	if (dest != right) emit_move(as, dest, right);
2186      }
2187#else
2188      emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
2189      if (dest != right) {
2190	emit_dst(as, MIPSI_SELNEZ, RID_TMP, right, RID_TMP);
2191	emit_dst(as, MIPSI_SELEQZ, dest, left, RID_TMP);
2192      } else {
2193	emit_dst(as, MIPSI_SELEQZ, RID_TMP, left, RID_TMP);
2194	emit_dst(as, MIPSI_SELNEZ, dest, right, RID_TMP);
2195      }
2196#endif
2197      emit_dst(as, MIPSI_SLT, RID_TMP,
2198	       ismax ? left : right, ismax ? right : left);
2199    }
2200  }
2201}
2202
2203#define asm_min(as, ir)		asm_min_max(as, ir, 0)
2204#define asm_max(as, ir)		asm_min_max(as, ir, 1)
2205
2206/* -- Comparisons --------------------------------------------------------- */
2207
2208#if LJ_SOFTFP
2209/* SFP comparisons. */
2210static void asm_sfpcomp(ASMState *as, IRIns *ir)
2211{
2212  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp];
2213  RegSet drop = RSET_SCRATCH;
2214  Reg r;
2215#if LJ_64
2216  IRRef args[2];
2217  args[0] = ir->op1;
2218  args[1] = ir->op2;
2219#else
2220  IRRef args[4];
2221  args[LJ_LE ? 0 : 1] = ir->op1; args[LJ_LE ? 1 : 0] = (ir+1)->op1;
2222  args[LJ_LE ? 2 : 3] = ir->op2; args[LJ_LE ? 3 : 2] = (ir+1)->op2;
2223#endif
2224
2225  for (r = REGARG_FIRSTGPR; r <= REGARG_FIRSTGPR+(LJ_64?1:3); r++) {
2226    if (!rset_test(as->freeset, r) &&
2227	regcost_ref(as->cost[r]) == args[r-REGARG_FIRSTGPR])
2228      rset_clear(drop, r);
2229  }
2230  ra_evictset(as, drop);
2231
2232  asm_setupresult(as, ir, ci);
2233
2234  switch ((IROp)ir->o) {
2235  case IR_LT:
2236    asm_guard(as, MIPSI_BGEZ, RID_RET, 0);
2237    break;
2238  case IR_ULT:
2239    asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2240    emit_loadi(as, RID_TMP, 1);
2241    asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO);
2242    break;
2243  case IR_GE:
2244    asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2245    emit_loadi(as, RID_TMP, 2);
2246    asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
2247    break;
2248  case IR_LE:
2249    asm_guard(as, MIPSI_BGTZ, RID_RET, 0);
2250    break;
2251  case IR_GT:
2252    asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2253    emit_loadi(as, RID_TMP, 2);
2254    asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
2255    break;
2256  case IR_UGE:
2257    asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
2258    break;
2259  case IR_ULE:
2260    asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2261    emit_loadi(as, RID_TMP, 1);
2262    break;
2263  case IR_UGT: case IR_ABC:
2264    asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
2265    break;
2266  case IR_EQ: case IR_NE:
2267    asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_RET, RID_ZERO);
2268  default:
2269    break;
2270  }
2271  asm_gencall(as, ci, args);
2272}
2273#endif
2274
2275static void asm_comp(ASMState *as, IRIns *ir)
2276{
2277  /* ORDER IR: LT GE LE GT  ULT UGE ULE UGT. */
2278  IROp op = ir->o;
2279  if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
2280#if LJ_SOFTFP
2281    asm_sfpcomp(as, ir);
2282#else
2283#if !LJ_TARGET_MIPSR6
2284    Reg right, left = ra_alloc2(as, ir, RSET_FPR);
2285    right = (left >> 8); left &= 255;
2286    asm_guard(as, (op&1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
2287    emit_fgh(as, MIPSI_C_OLT_D + ((op&3) ^ ((op>>2)&1)), 0, left, right);
2288#else
2289    Reg tmp, right, left = ra_alloc2(as, ir, RSET_FPR);
2290    right = (left >> 8); left &= 255;
2291    tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_FPR, left), right));
2292    asm_guard(as, (op&1) ? MIPSI_BC1NEZ : MIPSI_BC1EQZ, 0, (tmp&31));
2293    emit_fgh(as, MIPSI_CMP_LT_D + ((op&3) ^ ((op>>2)&1)), tmp, left, right);
2294#endif
2295#endif
2296  } else {
2297    Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
2298    if (op == IR_ABC) op = IR_UGT;
2299    if ((op&4) == 0 && irref_isk(ir->op2) && get_kval(as, ir->op2) == 0) {
2300      MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) :
2301			    ((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ);
2302      asm_guard(as, mi, left, 0);
2303    } else {
2304      if (irref_isk(ir->op2)) {
2305	intptr_t k = get_kval(as, ir->op2);
2306	if ((op&2)) k++;
2307	if (checki16(k)) {
2308	  asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
2309	  emit_tsi(as, (op&4) ? MIPSI_SLTIU : MIPSI_SLTI,
2310		   RID_TMP, left, k);
2311	  return;
2312	}
2313      }
2314      right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
2315      asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
2316      emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT,
2317	       RID_TMP, (op&2) ? right : left, (op&2) ? left : right);
2318    }
2319  }
2320}
2321
2322static void asm_equal(ASMState *as, IRIns *ir)
2323{
2324  Reg right, left = ra_alloc2(as, ir, (!LJ_SOFTFP && irt_isnum(ir->t)) ?
2325				       RSET_FPR : RSET_GPR);
2326  right = (left >> 8); left &= 255;
2327  if (!LJ_SOFTFP32 && irt_isnum(ir->t)) {
2328#if LJ_SOFTFP
2329    asm_sfpcomp(as, ir);
2330#elif !LJ_TARGET_MIPSR6
2331    asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
2332    emit_fgh(as, MIPSI_C_EQ_D, 0, left, right);
2333#else
2334    Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_FPR, left), right));
2335    asm_guard(as, (ir->o & 1) ? MIPSI_BC1NEZ : MIPSI_BC1EQZ, 0, (tmp&31));
2336    emit_fgh(as, MIPSI_CMP_EQ_D, tmp, left, right);
2337#endif
2338  } else {
2339    asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, left, right);
2340  }
2341}
2342
2343#if LJ_32 && LJ_HASFFI
2344/* 64 bit integer comparisons. */
2345static void asm_comp64(ASMState *as, IRIns *ir)
2346{
2347  /* ORDER IR: LT GE LE GT  ULT UGE ULE UGT. */
2348  IROp op = (ir-1)->o;
2349  MCLabel l_end;
2350  Reg rightlo, leftlo, righthi, lefthi = ra_alloc2(as, ir, RSET_GPR);
2351  righthi = (lefthi >> 8); lefthi &= 255;
2352  leftlo = ra_alloc2(as, ir-1,
2353		     rset_exclude(rset_exclude(RSET_GPR, lefthi), righthi));
2354  rightlo = (leftlo >> 8); leftlo &= 255;
2355  asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
2356  l_end = emit_label(as);
2357  if (lefthi != righthi)
2358    emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT, RID_TMP,
2359	     (op&2) ? righthi : lefthi, (op&2) ? lefthi : righthi);
2360  emit_dst(as, MIPSI_SLTU, RID_TMP,
2361	   (op&2) ? rightlo : leftlo, (op&2) ? leftlo : rightlo);
2362  if (lefthi != righthi)
2363    emit_branch(as, MIPSI_BEQ, lefthi, righthi, l_end);
2364}
2365
2366static void asm_comp64eq(ASMState *as, IRIns *ir)
2367{
2368  Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
2369  right = (left >> 8); left &= 255;
2370  asm_guard(as, ((ir-1)->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_TMP, RID_ZERO);
2371  tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
2372  emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
2373  emit_dst(as, MIPSI_XOR, tmp, left, right);
2374  left = ra_alloc2(as, ir-1, RSET_GPR);
2375  right = (left >> 8); left &= 255;
2376  emit_dst(as, MIPSI_XOR, RID_TMP, left, right);
2377}
2378#endif
2379
2380/* -- Split register ops -------------------------------------------------- */
2381
2382/* Hiword op of a split 32/32 or 64/64 bit op. Previous op is the loword op. */
2383static void asm_hiop(ASMState *as, IRIns *ir)
2384{
2385  /* HIOP is marked as a store because it needs its own DCE logic. */
2386  int uselo = ra_used(ir-1), usehi = ra_used(ir);  /* Loword/hiword used? */
2387  if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
2388#if LJ_32 && (LJ_HASFFI || LJ_SOFTFP)
2389  if ((ir-1)->o == IR_CONV) {  /* Conversions to/from 64 bit. */
2390    as->curins--;  /* Always skip the CONV. */
2391#if LJ_HASFFI && !LJ_SOFTFP
2392    if (usehi || uselo)
2393      asm_conv64(as, ir);
2394    return;
2395#endif
2396  } else if ((ir-1)->o < IR_EQ) {  /* 64 bit integer comparisons. ORDER IR. */
2397    as->curins--;  /* Always skip the loword comparison. */
2398#if LJ_SOFTFP
2399    if (!irt_isint(ir->t)) {
2400      asm_sfpcomp(as, ir-1);
2401      return;
2402    }
2403#endif
2404#if LJ_HASFFI
2405    asm_comp64(as, ir);
2406#endif
2407    return;
2408  } else if ((ir-1)->o <= IR_NE) {  /* 64 bit integer comparisons. ORDER IR. */
2409    as->curins--;  /* Always skip the loword comparison. */
2410#if LJ_SOFTFP
2411    if (!irt_isint(ir->t)) {
2412      asm_sfpcomp(as, ir-1);
2413      return;
2414    }
2415#endif
2416#if LJ_HASFFI
2417    asm_comp64eq(as, ir);
2418#endif
2419    return;
2420#if LJ_SOFTFP
2421  } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) {
2422      as->curins--;  /* Always skip the loword min/max. */
2423    if (uselo || usehi)
2424      asm_sfpmin_max(as, ir-1);
2425    return;
2426#endif
2427  } else if ((ir-1)->o == IR_XSTORE) {
2428    as->curins--;  /* Handle both stores here. */
2429    if ((ir-1)->r != RID_SINK) {
2430      asm_xstore_(as, ir, LJ_LE ? 4 : 0);
2431      asm_xstore_(as, ir-1, LJ_LE ? 0 : 4);
2432    }
2433    return;
2434  }
2435#endif
2436  if (!usehi) return;  /* Skip unused hiword op for all remaining ops. */
2437  switch ((ir-1)->o) {
2438#if LJ_32 && LJ_HASFFI
2439  case IR_ADD: as->curins--; asm_add64(as, ir); break;
2440  case IR_SUB: as->curins--; asm_sub64(as, ir); break;
2441  case IR_NEG: as->curins--; asm_neg64(as, ir); break;
2442  case IR_CNEWI:
2443    /* Nothing to do here. Handled by lo op itself. */
2444    break;
2445#endif
2446#if LJ_32 && LJ_SOFTFP
2447  case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2448  case IR_STRTO:
2449    if (!uselo)
2450      ra_allocref(as, ir->op1, RSET_GPR);  /* Mark lo op as used. */
2451    break;
2452  case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR: case IR_TMPREF:
2453    /* Nothing to do here. Handled by lo op itself. */
2454    break;
2455#endif
2456  case IR_CALLN: case IR_CALLL: case IR_CALLS: case IR_CALLXS:
2457    if (!uselo)
2458      ra_allocref(as, ir->op1, RID2RSET(RID_RETLO));  /* Mark lo op as used. */
2459    break;
2460  default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
2461  }
2462}
2463
2464/* -- Profiling ----------------------------------------------------------- */
2465
2466static void asm_prof(ASMState *as, IRIns *ir)
2467{
2468  UNUSED(ir);
2469  asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
2470  emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, HOOK_PROFILE);
2471  emit_lsglptr(as, MIPSI_LBU, RID_TMP,
2472	       (int32_t)offsetof(global_State, hookmask));
2473}
2474
2475/* -- Stack handling ------------------------------------------------------ */
2476
2477/* Check Lua stack size for overflow. Use exit handler as fallback. */
2478static void asm_stack_check(ASMState *as, BCReg topslot,
2479			    IRIns *irp, RegSet allow, ExitNo exitno)
2480{
2481  /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
2482  Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
2483  ExitNo oldsnap = as->snapno;
2484  rset_clear(allow, pbase);
2485#if LJ_32
2486  tmp = allow ? rset_pickbot(allow) :
2487		(pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
2488#else
2489  tmp = allow ? rset_pickbot(allow) : RID_RET;
2490#endif
2491  as->snapno = exitno;
2492  asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
2493  as->snapno = oldsnap;
2494  if (allow == RSET_EMPTY)  /* Restore temp. register. */
2495    emit_tsi(as, MIPSI_AL, tmp, RID_SP, 0);
2496  else
2497    ra_modified(as, tmp);
2498  emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot));
2499  emit_dst(as, MIPSI_ASUBU, RID_TMP, tmp, pbase);
2500  emit_tsi(as, MIPSI_AL, tmp, tmp, offsetof(lua_State, maxstack));
2501  if (pbase == RID_TMP)
2502    emit_getgl(as, RID_TMP, jit_base);
2503  emit_getgl(as, tmp, cur_L);
2504  if (allow == RSET_EMPTY)  /* Spill temp. register. */
2505    emit_tsi(as, MIPSI_AS, tmp, RID_SP, 0);
2506}
2507
2508/* Restore Lua stack from on-trace state. */
2509static void asm_stack_restore(ASMState *as, SnapShot *snap)
2510{
2511  SnapEntry *map = &as->T->snapmap[snap->mapofs];
2512#if LJ_32 || defined(LUA_USE_ASSERT)
2513  SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
2514#endif
2515  MSize n, nent = snap->nent;
2516  /* Store the value of all modified slots to the Lua stack. */
2517  for (n = 0; n < nent; n++) {
2518    SnapEntry sn = map[n];
2519    BCReg s = snap_slot(sn);
2520    int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
2521    IRRef ref = snap_ref(sn);
2522    IRIns *ir = IR(ref);
2523    if ((sn & SNAP_NORESTORE))
2524      continue;
2525    if (irt_isnum(ir->t)) {
2526#if LJ_SOFTFP32
2527      Reg tmp;
2528      RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2529      /* LJ_SOFTFP: must be a number constant. */
2530      lj_assertA(irref_isk(ref), "unsplit FP op");
2531      tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, allow);
2532      emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?4:0));
2533      if (rset_test(as->freeset, tmp+1)) allow = RID2RSET(tmp+1);
2534      tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, allow);
2535      emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?0:4));
2536#elif LJ_SOFTFP  /* && LJ_64 */
2537      Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
2538      emit_tsi(as, MIPSI_SD, src, RID_BASE, ofs);
2539#else
2540      Reg src = ra_alloc1(as, ref, RSET_FPR);
2541      emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs);
2542#endif
2543    } else {
2544#if LJ_32
2545      RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2546      Reg type;
2547      lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t),
2548		 "restore of IR type %d", irt_type(ir->t));
2549      if (!irt_ispri(ir->t)) {
2550	Reg src = ra_alloc1(as, ref, allow);
2551	rset_clear(allow, src);
2552	emit_tsi(as, MIPSI_SW, src, RID_BASE, ofs+(LJ_BE?4:0));
2553      }
2554      if ((sn & (SNAP_CONT|SNAP_FRAME))) {
2555	if (s == 0) continue;  /* Do not overwrite link to previous frame. */
2556	type = ra_allock(as, (int32_t)(*flinks--), allow);
2557#if LJ_SOFTFP
2558      } else if ((sn & SNAP_SOFTFPNUM)) {
2559	type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPR, RID_BASE));
2560#endif
2561      } else if ((sn & SNAP_KEYINDEX)) {
2562	type = ra_allock(as, (int32_t)LJ_KEYINDEX, allow);
2563      } else {
2564	type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
2565      }
2566      emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4));
2567#else
2568      if ((sn & SNAP_KEYINDEX)) {
2569	RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2570	int64_t kki = (int64_t)LJ_KEYINDEX << 32;
2571	if (irref_isk(ref)) {
2572	  emit_tsi(as, MIPSI_SD,
2573		   ra_allock(as, kki | (int64_t)(uint32_t)ir->i, allow),
2574		   RID_BASE, ofs);
2575	} else {
2576	  Reg src = ra_alloc1(as, ref, allow);
2577	  Reg rki = ra_allock(as, kki, rset_exclude(allow, src));
2578	  emit_tsi(as, MIPSI_SD, RID_TMP, RID_BASE, ofs);
2579	  emit_dst(as, MIPSI_DADDU, RID_TMP, src, rki);
2580	}
2581      } else {
2582	asm_tvstore64(as, RID_BASE, ofs, ref);
2583      }
2584#endif
2585    }
2586    checkmclim(as);
2587  }
2588  lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
2589}
2590
2591/* -- GC handling --------------------------------------------------------- */
2592
2593/* Marker to prevent patching the GC check exit. */
2594#define MIPS_NOPATCH_GC_CHECK	MIPSI_OR
2595
2596/* Check GC threshold and do one or more GC steps. */
2597static void asm_gc_check(ASMState *as)
2598{
2599  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
2600  IRRef args[2];
2601  MCLabel l_end;
2602  Reg tmp;
2603  ra_evictset(as, RSET_SCRATCH);
2604  l_end = emit_label(as);
2605  /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
2606  /* Assumes asm_snap_prep() already done. */
2607  asm_guard(as, MIPSI_BNE, RID_RET, RID_ZERO);
2608  args[0] = ASMREF_TMP1;  /* global_State *g */
2609  args[1] = ASMREF_TMP2;  /* MSize steps     */
2610  asm_gencall(as, ci, args);
2611  l_end[-3] = MIPS_NOPATCH_GC_CHECK;  /* Replace the nop after the call. */
2612  emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
2613  tmp = ra_releasetmp(as, ASMREF_TMP2);
2614  emit_loadi(as, tmp, as->gcsteps);
2615  /* Jump around GC step if GC total < GC threshold. */
2616  emit_branch(as, MIPSI_BNE, RID_TMP, RID_ZERO, l_end);
2617  emit_dst(as, MIPSI_SLTU, RID_TMP, RID_TMP, tmp);
2618  emit_getgl(as, tmp, gc.threshold);
2619  emit_getgl(as, RID_TMP, gc.total);
2620  as->gcsteps = 0;
2621  checkmclim(as);
2622}
2623
2624/* -- Loop handling ------------------------------------------------------- */
2625
2626/* Fixup the loop branch. */
2627static void asm_loop_fixup(ASMState *as)
2628{
2629  MCode *p = as->mctop;
2630  MCode *target = as->mcp;
2631  p[-1] = MIPSI_NOP;
2632  if (as->loopinv) {  /* Inverted loop branch? */
2633    /* asm_guard already inverted the cond branch. Only patch the target. */
2634    p[-3] |= ((target-p+2) & 0x0000ffffu);
2635  } else {
2636    p[-2] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2637  }
2638}
2639
2640/* Fixup the tail of the loop. */
2641static void asm_loop_tail_fixup(ASMState *as)
2642{
2643  if (as->loopinv) as->mctop--;
2644}
2645
2646/* -- Head of trace ------------------------------------------------------- */
2647
2648/* Coalesce BASE register for a root trace. */
2649static void asm_head_root_base(ASMState *as)
2650{
2651  IRIns *ir = IR(REF_BASE);
2652  Reg r = ir->r;
2653  if (ra_hasreg(r)) {
2654    ra_free(as, r);
2655    if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2656      ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
2657    if (r != RID_BASE)
2658      emit_move(as, r, RID_BASE);
2659  }
2660}
2661
2662/* Coalesce BASE register for a side trace. */
2663static Reg asm_head_side_base(ASMState *as, IRIns *irp)
2664{
2665  IRIns *ir = IR(REF_BASE);
2666  Reg r = ir->r;
2667  if (ra_hasreg(r)) {
2668    ra_free(as, r);
2669    if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2670      ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
2671    if (irp->r == r) {
2672      return r;  /* Same BASE register already coalesced. */
2673    } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
2674      emit_move(as, r, irp->r);  /* Move from coalesced parent reg. */
2675      return irp->r;
2676    } else {
2677      emit_getgl(as, r, jit_base);  /* Otherwise reload BASE. */
2678    }
2679  }
2680  return RID_NONE;
2681}
2682
2683/* -- Tail of trace ------------------------------------------------------- */
2684
2685/* Fixup the tail code. */
2686static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2687{
2688  uintptr_t target = lnk ? (uintptr_t)traceref(as->J, lnk)->mcode : (uintptr_t)(void *)lj_vm_exit_interp;
2689  MCode *mcp = as->mctail;
2690  int32_t spadj = as->T->spadjust;
2691  if (((uintptr_t)mcp ^ target) >> 28 == 0) {
2692    *mcp++ = MIPSI_J | ((target >> 2) & 0x03ffffffu);
2693  } else {
2694    *mcp++ = MIPSI_AL | MIPSF_T(RID_TMP) | MIPSF_S(RID_JGL) |
2695#if LJ_64
2696	     jglofs(as, &as->J->k64[LJ_K64_VM_EXIT_INTERP]);
2697#else
2698	     jglofs(as, &as->J->k32[LJ_K32_VM_EXIT_INTERP]);
2699#endif
2700    *mcp++ = MIPSI_JR | MIPSF_S(RID_TMP);
2701  }
2702  *mcp++ = spadj ? (MIPSI_AADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
2703}
2704
2705/* Prepare tail of code. */
2706static void asm_tail_prep(ASMState *as, TraceNo lnk)
2707{
2708  as->mcp = as->mctop - 2;  /* Leave room for branch plus nop or stack adj. */
2709  if (as->loopref) {
2710    as->invmcp = as->mcp;
2711  } else {
2712    if (!lnk) {
2713      uintptr_t target = (uintptr_t)(void *)lj_vm_exit_interp;
2714      if (((uintptr_t)as->mcp ^ target) >> 28 != 0) as->mcp--;
2715    }
2716    as->invmcp = NULL;
2717  }
2718  as->mctail = as->mcp;
2719}
2720
2721/* -- Trace setup --------------------------------------------------------- */
2722
2723/* Ensure there are enough stack slots for call arguments. */
2724static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2725{
2726  IRRef args[CCI_NARGS_MAX*2];
2727  uint32_t i, nargs = CCI_XNARGS(ci);
2728#if LJ_32
2729  int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2730#else
2731  int nslots = 0, ngpr = REGARG_NUMGPR;
2732#endif
2733  asm_collectargs(as, ir, ci, args);
2734  for (i = 0; i < nargs; i++) {
2735#if LJ_32
2736    if (!LJ_SOFTFP && args[i] && irt_isfp(IR(args[i])->t) &&
2737	nfpr > 0 && !(ci->flags & CCI_VARARG)) {
2738      nfpr--;
2739      ngpr -= irt_isnum(IR(args[i])->t) ? 2 : 1;
2740    } else if (!LJ_SOFTFP && args[i] && irt_isnum(IR(args[i])->t)) {
2741      nfpr = 0;
2742      ngpr = ngpr & ~1;
2743      if (ngpr > 0) ngpr -= 2; else nslots = (nslots+3) & ~1;
2744    } else {
2745      nfpr = 0;
2746      if (ngpr > 0) ngpr--; else nslots++;
2747    }
2748#else
2749    if (ngpr > 0) ngpr--; else nslots += 2;
2750#endif
2751  }
2752  if (nslots > as->evenspill)  /* Leave room for args in stack slots. */
2753    as->evenspill = nslots;
2754  return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
2755}
2756
2757static void asm_setup_target(ASMState *as)
2758{
2759  asm_sparejump_setup(as);
2760  asm_exitstub_setup(as);
2761}
2762
2763/* -- Trace patching ------------------------------------------------------ */
2764
2765/* Patch exit jumps of existing machine code to a new target. */
2766void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2767{
2768  MCode *p = T->mcode;
2769  MCode *pe = (MCode *)((char *)p + T->szmcode);
2770  MCode *px = exitstub_trace_addr(T, exitno);
2771  MCode *cstart = NULL, *cstop = NULL;
2772  MCode *mcarea = lj_mcode_patch(J, p, 0);
2773  MCode exitload = MIPSI_LI | MIPSF_T(RID_TMP) | exitno;
2774  MCode tjump = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2775  for (p++; p < pe; p++) {
2776    if (*p == exitload) {  /* Look for load of exit number. */
2777      /* Look for exitstub branch. Yes, this covers all used branch variants. */
2778      if (((p[-1] ^ (px-p)) & 0xffffu) == 0 &&
2779	  ((p[-1] & 0xf0000000u) == MIPSI_BEQ ||
2780	   (p[-1] & 0xfc1e0000u) == MIPSI_BLTZ ||
2781#if !LJ_TARGET_MIPSR6
2782	   (p[-1] & 0xffe00000u) == MIPSI_BC1F
2783#else
2784	   (p[-1] & 0xff600000u) == MIPSI_BC1EQZ
2785#endif
2786	  ) && p[-2] != MIPS_NOPATCH_GC_CHECK) {
2787	ptrdiff_t delta = target - p;
2788	if (((delta + 0x8000) >> 16) == 0) {  /* Patch in-range branch. */
2789	patchbranch:
2790	  p[-1] = (p[-1] & 0xffff0000u) | (delta & 0xffffu);
2791	  *p = MIPSI_NOP;  /* Replace the load of the exit number. */
2792	  cstop = p+1;
2793	  if (!cstart) cstart = p-1;
2794	} else {  /* Branch out of range. Use spare jump slot in mcarea. */
2795	  MCode *mcjump = asm_sparejump_use(mcarea, tjump);
2796	  if (mcjump) {
2797	    lj_mcode_sync(mcjump, mcjump+1);
2798	    delta = mcjump - p;
2799	    if (((delta + 0x8000) >> 16) == 0) {
2800	      goto patchbranch;
2801	    } else {
2802	      lj_assertJ(0, "spare jump out of range: -Osizemcode too big");
2803	    }
2804	  }
2805	  /* Ignore jump slot overflow. Child trace is simply not attached. */
2806	}
2807      } else if (p+1 == pe) {
2808	/* Patch NOP after code for inverted loop branch. Use of J is ok. */
2809	lj_assertJ(p[1] == MIPSI_NOP, "expected NOP");
2810	p[1] = tjump;
2811	*p = MIPSI_NOP;  /* Replace the load of the exit number. */
2812	cstop = p+2;
2813	if (!cstart) cstart = p+1;
2814      }
2815    }
2816  }
2817  if (cstart) lj_mcode_sync(cstart, cstop);
2818  lj_mcode_patch(J, mcarea, 1);
2819}
2820