luna - luajit/src/lj

   1/*
   2** Trace recorder (bytecode -> SSA IR).
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4*/
   5
   6#define lj_record_c
   7#define LUA_CORE
   8
   9#include "lj_obj.h"
  10
  11#if LJ_HASJIT
  12
  13#include "lj_err.h"
  14#include "lj_str.h"
  15#include "lj_tab.h"
  16#include "lj_meta.h"
  17#include "lj_frame.h"
  18#if LJ_HASFFI
  19#include "lj_ctype.h"
  20#endif
  21#include "lj_bc.h"
  22#include "lj_ff.h"
  23#if LJ_HASPROFILE
  24#include "lj_debug.h"
  25#endif
  26#include "lj_ir.h"
  27#include "lj_jit.h"
  28#include "lj_ircall.h"
  29#include "lj_iropt.h"
  30#include "lj_trace.h"
  31#include "lj_record.h"
  32#include "lj_ffrecord.h"
  33#include "lj_snap.h"
  34#include "lj_dispatch.h"
  35#include "lj_vm.h"
  36#include "lj_prng.h"
  37
  38/* Some local macros to save typing. Undef'd at the end. */
  39#define IR(ref)			(&J->cur.ir[(ref)])
  40
  41/* Pass IR on to next optimization in chain (FOLD). */
  42#define emitir(ot, a, b)	(lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  43
  44/* Emit raw IR without passing through optimizations. */
  45#define emitir_raw(ot, a, b)	(lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
  46
  47/* -- Sanity checks ------------------------------------------------------- */
  48
  49#ifdef LUA_USE_ASSERT
  50/* Sanity check the whole IR -- sloooow. */
  51static void rec_check_ir(jit_State *J)
  52{
  53  IRRef i, nins = J->cur.nins, nk = J->cur.nk;
  54  lj_assertJ(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536,
  55	     "inconsistent IR layout");
  56  for (i = nk; i < nins; i++) {
  57    IRIns *ir = IR(i);
  58    uint32_t mode = lj_ir_mode[ir->o];
  59    IRRef op1 = ir->op1;
  60    IRRef op2 = ir->op2;
  61    const char *err = NULL;
  62    switch (irm_op1(mode)) {
  63    case IRMnone:
  64      if (op1 != 0) err = "IRMnone op1 used";
  65      break;
  66    case IRMref:
  67      if (op1 < nk || (i >= REF_BIAS ? op1 >= i : op1 <= i))
  68	err = "IRMref op1 out of range";
  69      break;
  70    case IRMlit: break;
  71    case IRMcst:
  72      if (i >= REF_BIAS) { err = "constant in IR range"; break; }
  73      if (irt_is64(ir->t) && ir->o != IR_KNULL)
  74	i++;
  75      continue;
  76    }
  77    switch (irm_op2(mode)) {
  78    case IRMnone:
  79      if (op2) err = "IRMnone op2 used";
  80      break;
  81    case IRMref:
  82      if (op2 < nk || (i >= REF_BIAS ? op2 >= i : op2 <= i))
  83	err = "IRMref op2 out of range";
  84      break;
  85    case IRMlit: break;
  86    case IRMcst: err = "IRMcst op2"; break;
  87    }
  88    if (!err && ir->prev) {
  89      if (ir->prev < nk || (i >= REF_BIAS ? ir->prev >= i : ir->prev <= i))
  90	err = "chain out of range";
  91      else if (ir->o != IR_NOP && IR(ir->prev)->o != ir->o)
  92	err = "chain to different op";
  93    }
  94    lj_assertJ(!err, "bad IR %04d op %d(%04d,%04d): %s",
  95	       i-REF_BIAS,
  96	       ir->o,
  97	       irm_op1(mode) == IRMref ? op1-REF_BIAS : op1,
  98	       irm_op2(mode) == IRMref ? op2-REF_BIAS : op2,
  99	       err);
 100  }
 101}
 102
 103/* Compare stack slots and frames of the recorder and the VM. */
 104static void rec_check_slots(jit_State *J)
 105{
 106  BCReg s, nslots = J->baseslot + J->maxslot;
 107  int32_t depth = 0;
 108  cTValue *base = J->L->base - J->baseslot;
 109  lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot");
 110  lj_assertJ(J->baseslot == 1+LJ_FR2 || (J->slot[J->baseslot-1] & TREF_FRAME),
 111	     "baseslot does not point to frame");
 112  lj_assertJ(nslots <= LJ_MAX_JSLOTS, "slot overflow");
 113  for (s = 0; s < nslots; s++) {
 114    TRef tr = J->slot[s];
 115    if (tr) {
 116      cTValue *tv = &base[s];
 117      IRRef ref = tref_ref(tr);
 118      IRIns *ir = NULL;  /* Silence compiler. */
 119      lj_assertJ(tv < J->L->top, "slot %d above top of Lua stack", s);
 120      if (!LJ_FR2 || ref || !(tr & (TREF_FRAME | TREF_CONT))) {
 121	lj_assertJ(ref >= J->cur.nk && ref < J->cur.nins,
 122		   "slot %d ref %04d out of range", s, ref - REF_BIAS);
 123	ir = IR(ref);
 124	lj_assertJ(irt_t(ir->t) == tref_t(tr), "slot %d IR type mismatch", s);
 125      }
 126      if (s == 0) {
 127	lj_assertJ(tref_isfunc(tr), "frame slot 0 is not a function");
 128#if LJ_FR2
 129      } else if (s == 1) {
 130	lj_assertJ((tr & ~TREF_FRAME) == 0, "bad frame slot 1");
 131#endif
 132      } else if ((tr & TREF_FRAME)) {
 133	GCfunc *fn = gco2func(frame_gc(tv));
 134	BCReg delta = (BCReg)(tv - frame_prev(tv));
 135#if LJ_FR2
 136	lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
 137		   "frame slot %d PC mismatch", s);
 138	tr = J->slot[s-1];
 139	ir = IR(tref_ref(tr));
 140#endif
 141	lj_assertJ(tref_isfunc(tr),
 142		   "frame slot %d is not a function", s-LJ_FR2);
 143	lj_assertJ(!tref_isk(tr) || fn == ir_kfunc(ir),
 144		   "frame slot %d function mismatch", s-LJ_FR2);
 145	lj_assertJ(s > delta + LJ_FR2 ? (J->slot[s-delta] & TREF_FRAME)
 146				      : (s == delta + LJ_FR2),
 147		   "frame slot %d broken chain", s-LJ_FR2);
 148	depth++;
 149      } else if ((tr & TREF_CONT)) {
 150#if LJ_FR2
 151	lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
 152		   "cont slot %d continuation mismatch", s);
 153#else
 154	lj_assertJ(ir_kptr(ir) == gcrefp(tv->gcr, void),
 155		   "cont slot %d continuation mismatch", s);
 156#endif
 157	lj_assertJ((J->slot[s+1+LJ_FR2] & TREF_FRAME),
 158		   "cont slot %d not followed by frame", s);
 159	depth++;
 160      } else if ((tr & TREF_KEYINDEX)) {
 161	lj_assertJ(tref_isint(tr), "keyindex slot %d bad type %d",
 162				   s, tref_type(tr));
 163      } else {
 164	/* Number repr. may differ, but other types must be the same. */
 165	lj_assertJ(tvisnumber(tv) ? tref_isnumber(tr) :
 166				    itype2irt(tv) == tref_type(tr),
 167		   "slot %d type mismatch: stack type %d vs IR type %d",
 168		   s, itypemap(tv), tref_type(tr));
 169	if (tref_isk(tr)) {  /* Compare constants. */
 170	  TValue tvk;
 171	  lj_ir_kvalue(J->L, &tvk, ir);
 172	  lj_assertJ((tvisnum(&tvk) && tvisnan(&tvk)) ?
 173		     (tvisnum(tv) && tvisnan(tv)) :
 174		     lj_obj_equal(tv, &tvk),
 175		     "slot %d const mismatch: stack %016llx vs IR %016llx",
 176		     s, tv->u64, tvk.u64);
 177	}
 178      }
 179    }
 180  }
 181  lj_assertJ(J->framedepth == depth,
 182	     "frame depth mismatch %d vs %d", J->framedepth, depth);
 183}
 184#endif
 185
 186/* -- Type handling and specialization ------------------------------------ */
 187
 188/* Note: these functions return tagged references (TRef). */
 189
 190/* Specialize a slot to a specific type. Note: slot can be negative! */
 191static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode)
 192{
 193  /* Caller may set IRT_GUARD in t. */
 194  TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode);
 195  J->base[slot] = ref;
 196  return ref;
 197}
 198
 199/* Specialize a slot to the runtime type. Note: slot can be negative! */
 200static TRef sload(jit_State *J, int32_t slot)
 201{
 202  IRType t = itype2irt(&J->L->base[slot]);
 203  TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot,
 204			IRSLOAD_TYPECHECK);
 205  if (irtype_ispri(t)) ref = TREF_PRI(t);  /* Canonicalize primitive refs. */
 206  J->base[slot] = ref;
 207  return ref;
 208}
 209
 210/* Get TRef from slot. Load slot and specialize if not done already. */
 211#define getslot(J, s)	(J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
 212
 213/* Get TRef for current function. */
 214static TRef getcurrf(jit_State *J)
 215{
 216  if (J->base[-1-LJ_FR2])
 217    return J->base[-1-LJ_FR2];
 218  /* Non-base frame functions ought to be loaded already. */
 219  lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot");
 220  return sloadt(J, -1-LJ_FR2, IRT_FUNC, IRSLOAD_READONLY);
 221}
 222
 223/* Compare for raw object equality.
 224** Returns 0 if the objects are the same.
 225** Returns 1 if they are different, but the same type.
 226** Returns 2 for two different types.
 227** Comparisons between primitives always return 1 -- no caller cares about it.
 228*/
 229int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
 230{
 231  int diff = !lj_obj_equal(av, bv);
 232  if (!tref_isk2(a, b)) {  /* Shortcut, also handles primitives. */
 233    IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a);
 234    IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b);
 235    if (ta != tb) {
 236      /* Widen mixed number/int comparisons to number/number comparison. */
 237      if (ta == IRT_INT && tb == IRT_NUM) {
 238	a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT);
 239	ta = IRT_NUM;
 240      } else if (ta == IRT_NUM && tb == IRT_INT) {
 241	b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT);
 242      } else {
 243	return 2;  /* Two different types are never equal. */
 244      }
 245    }
 246    emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b);
 247  }
 248  return diff;
 249}
 250
 251/* Constify a value. Returns 0 for non-representable object types. */
 252TRef lj_record_constify(jit_State *J, cTValue *o)
 253{
 254  if (tvisgcv(o))
 255    return lj_ir_kgc(J, gcV(o), itype2irt(o));
 256  else if (tvisint(o))
 257    return lj_ir_kint(J, intV(o));
 258  else if (tvisnum(o))
 259    return lj_ir_knumint(J, numV(o));
 260  else if (tvisbool(o))
 261    return TREF_PRI(itype2irt(o));
 262  else
 263    return 0;  /* Can't represent lightuserdata (pointless). */
 264}
 265
 266/* Emit a VLOAD with the correct type. */
 267TRef lj_record_vload(jit_State *J, TRef ref, MSize idx, IRType t)
 268{
 269  TRef tr = emitir(IRTG(IR_VLOAD, t), ref, idx);
 270  if (irtype_ispri(t)) tr = TREF_PRI(t);  /* Canonicalize primitives. */
 271  return tr;
 272}
 273
 274/* -- Record loop ops ----------------------------------------------------- */
 275
 276/* Loop event. */
 277typedef enum {
 278  LOOPEV_LEAVE,		/* Loop is left or not entered. */
 279  LOOPEV_ENTERLO,	/* Loop is entered with a low iteration count left. */
 280  LOOPEV_ENTER		/* Loop is entered. */
 281} LoopEvent;
 282
 283/* Canonicalize slots: convert integers to numbers. */
 284static void canonicalize_slots(jit_State *J)
 285{
 286  BCReg s;
 287  if (LJ_DUALNUM) return;
 288  for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
 289    TRef tr = J->slot[s];
 290    if (tref_isinteger(tr) && !(tr & TREF_KEYINDEX)) {
 291      IRIns *ir = IR(tref_ref(tr));
 292      if (!(ir->o == IR_SLOAD && (ir->op2 & (IRSLOAD_READONLY))))
 293	J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
 294    }
 295  }
 296}
 297
 298/* Stop recording. */
 299void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
 300{
 301#ifdef LUAJIT_ENABLE_TABLE_BUMP
 302  if (J->retryrec)
 303    lj_trace_err(J, LJ_TRERR_RETRY);
 304#endif
 305  lj_trace_end(J);
 306  J->cur.linktype = (uint8_t)linktype;
 307  J->cur.link = (uint16_t)lnk;
 308  /* Looping back at the same stack level? */
 309  if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) {
 310    if ((J->flags & JIT_F_OPT_LOOP))  /* Shall we try to create a loop? */
 311      goto nocanon;  /* Do not canonicalize or we lose the narrowing. */
 312    if (J->cur.root)  /* Otherwise ensure we always link to the root trace. */
 313      J->cur.link = J->cur.root;
 314  }
 315  canonicalize_slots(J);
 316nocanon:
 317  /* Note: all loop ops must set J->pc to the following instruction! */
 318  lj_snap_add(J);  /* Add loop snapshot. */
 319  J->needsnap = 0;
 320  J->mergesnap = 1;  /* In case recording continues. */
 321}
 322
 323/* Search bytecode backwards for a int/num constant slot initializer. */
 324static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t)
 325{
 326  /* This algorithm is rather simplistic and assumes quite a bit about
 327  ** how the bytecode is generated. It works fine for FORI initializers,
 328  ** but it won't necessarily work in other cases (e.g. iterator arguments).
 329  ** It doesn't do anything fancy, either (like backpropagating MOVs).
 330  */
 331  const BCIns *pc, *startpc = proto_bc(J->pt);
 332  for (pc = endpc-1; pc > startpc; pc--) {
 333    BCIns ins = *pc;
 334    BCOp op = bc_op(ins);
 335    /* First try to find the last instruction that stores to this slot. */
 336    if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) {
 337      return 0;  /* Multiple results, e.g. from a CALL or KNIL. */
 338    } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) {
 339      if (op == BC_KSHORT || op == BC_KNUM) {  /* Found const. initializer. */
 340	/* Now try to verify there's no forward jump across it. */
 341	const BCIns *kpc = pc;
 342	for (; pc > startpc; pc--)
 343	  if (bc_op(*pc) == BC_JMP) {
 344	    const BCIns *target = pc+bc_j(*pc)+1;
 345	    if (target > kpc && target <= endpc)
 346	      return 0;  /* Conditional assignment. */
 347	  }
 348	if (op == BC_KSHORT) {
 349	  int32_t k = (int32_t)(int16_t)bc_d(ins);
 350	  return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k);
 351	} else {
 352	  cTValue *tv = proto_knumtv(J->pt, bc_d(ins));
 353	  if (t == IRT_INT) {
 354	    if (tvisint(tv)) {
 355	      return lj_ir_kint(J, intV(tv));
 356	    } else {
 357	      int64_t i64;
 358	      int32_t k;
 359	      if (lj_num2int_check(numV(tv), i64, k))  /* -0 is ok here. */
 360		return lj_ir_kint(J, k);
 361	    }
 362	    return 0;  /* Type mismatch. */
 363	  } else {
 364	    return lj_ir_knum(J, numberVnum(tv));
 365	  }
 366	}
 367      }
 368      return 0;  /* Non-constant initializer. */
 369    }
 370  }
 371  return 0;  /* No assignment to this slot found? */
 372}
 373
 374/* Load and optionally convert a FORI argument from a slot. */
 375static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode)
 376{
 377  int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0;
 378  return sloadt(J, (int32_t)slot,
 379		t + (((mode & IRSLOAD_TYPECHECK) ||
 380		      (conv && t == IRT_INT && !(mode >> 16))) ?
 381		     IRT_GUARD : 0),
 382		mode + conv);
 383}
 384
 385/* Convert FORI argument to expected target type. */
 386static TRef fori_conv(jit_State *J, TRef tr, IRType t)
 387{
 388  if (t == IRT_INT) {
 389    if (!tref_isinteger(tr))
 390      return emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_CHECK);
 391  } else {
 392    if (!tref_isnum(tr))
 393      return emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
 394  }
 395  return tr;
 396}
 397
 398/* Peek before FORI to find a const initializer. Otherwise load from slot. */
 399static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot,
 400		     IRType t, int mode)
 401{
 402  TRef tr = J->base[slot];
 403  if (tr) {
 404    tr = fori_conv(J, tr, t);
 405  } else {
 406    tr = find_kinit(J, fori, slot, t);
 407    if (!tr)
 408      tr = fori_load(J, slot, t, mode);
 409  }
 410  return tr;
 411}
 412
 413/* Return the direction of the FOR loop iterator.
 414** It's important to exactly reproduce the semantics of the interpreter.
 415*/
 416static int rec_for_direction(cTValue *o)
 417{
 418  return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0;
 419}
 420
 421/* Simulate the runtime behavior of the FOR loop iterator. */
 422static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl)
 423{
 424  lua_Number stopv = numberVnum(&o[FORL_STOP]);
 425  lua_Number idxv = numberVnum(&o[FORL_IDX]);
 426  lua_Number stepv = numberVnum(&o[FORL_STEP]);
 427  if (isforl)
 428    idxv += stepv;
 429  if (rec_for_direction(&o[FORL_STEP])) {
 430    if (idxv <= stopv) {
 431      *op = IR_LE;
 432      return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
 433    }
 434    *op = IR_GT; return LOOPEV_LEAVE;
 435  } else {
 436    if (stopv <= idxv) {
 437      *op = IR_GE;
 438      return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
 439    }
 440    *op = IR_LT; return LOOPEV_LEAVE;
 441  }
 442}
 443
 444/* Record checks for FOR loop overflow and step direction. */
 445static void rec_for_check(jit_State *J, IRType t, int dir,
 446			  TRef stop, TRef step, int init)
 447{
 448  if (!tref_isk(step)) {
 449    /* Non-constant step: need a guard for the direction. */
 450    TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J);
 451    emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero);
 452    /* Add hoistable overflow checks for a narrowed FORL index. */
 453    if (init && t == IRT_INT) {
 454      if (tref_isk(stop)) {
 455	/* Constant stop: optimize check away or to a range check for step. */
 456	int32_t k = IR(tref_ref(stop))->i;
 457	if (dir) {
 458	  if (k > 0)
 459	    emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k));
 460	} else {
 461	  if (k < 0)
 462	    emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k));
 463	}
 464      } else {
 465	/* Stop+step variable: need full overflow check. */
 466	TRef tr = emitir(IRTGI(IR_ADDOV), step, stop);
 467	emitir(IRTI(IR_USE), tr, 0);  /* ADDOV is weak. Avoid dead result. */
 468      }
 469    }
 470  } else if (init && t == IRT_INT && !tref_isk(stop)) {
 471    /* Constant step: optimize overflow check to a range check for stop. */
 472    int32_t k = IR(tref_ref(step))->i;
 473    k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k;
 474    emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k));
 475  }
 476}
 477
 478/* Record a FORL instruction. */
 479static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev,
 480			 int init)
 481{
 482  BCReg ra = bc_a(*fori);
 483  cTValue *tv = &J->L->base[ra];
 484  TRef idx = J->base[ra+FORL_IDX];
 485  IRType t = idx ? tref_type(idx) :
 486	     (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM;
 487  int mode = IRSLOAD_INHERIT +
 488    ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0);
 489  TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode);
 490  TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode);
 491  int tc, dir = rec_for_direction(&tv[FORL_STEP]);
 492  lj_assertJ(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI,
 493	     "bad bytecode %d instead of FORI/JFORI", bc_op(*fori));
 494  scev->t.irt = t;
 495  scev->dir = dir;
 496  scev->stop = tref_ref(stop);
 497  scev->step = tref_ref(step);
 498  rec_for_check(J, t, dir, stop, step, init);
 499  scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT));
 500  tc = (LJ_DUALNUM &&
 501	!(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) &&
 502	  tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ?
 503	IRSLOAD_TYPECHECK : 0;
 504  if (tc) {
 505    J->base[ra+FORL_STOP] = stop;
 506    J->base[ra+FORL_STEP] = step;
 507  }
 508  if (!idx)
 509    idx = fori_load(J, ra+FORL_IDX, t,
 510		    IRSLOAD_INHERIT + tc + (J->scev.start << 16));
 511  if (!init)
 512    J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step);
 513  J->base[ra+FORL_EXT] = idx;
 514  scev->idx = tref_ref(idx);
 515  setmref(scev->pc, fori);
 516  J->maxslot = ra+FORL_EXT+1;
 517}
 518
 519/* Record FORL/JFORL or FORI/JFORI. */
 520static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
 521{
 522  BCReg ra = bc_a(*fori);
 523  TValue *tv = &J->L->base[ra];
 524  TRef *tr = &J->base[ra];
 525  IROp op;
 526  LoopEvent ev;
 527  TRef stop;
 528  IRType t;
 529  /* Avoid semantic mismatches and always failing guards. */
 530  if ((tvisnum(&tv[FORL_IDX]) && tvisnan(&tv[FORL_IDX])) ||
 531      (tvisnum(&tv[FORL_STOP]) && tvisnan(&tv[FORL_STOP])) ||
 532      (tvisnum(&tv[FORL_STEP]) && tvisnan(&tv[FORL_STEP])) ||
 533      tvismzero(&tv[FORL_STEP]))
 534    lj_trace_err(J, LJ_TRERR_GFAIL);
 535  if (isforl) {  /* Handle FORL/JFORL opcodes. */
 536    TRef idx = tr[FORL_IDX];
 537    if (mref(J->scev.pc, const BCIns) == fori && tref_ref(idx) == J->scev.idx) {
 538      t = J->scev.t.irt;
 539      stop = J->scev.stop;
 540      idx = emitir(IRT(IR_ADD, t), idx, J->scev.step);
 541      tr[FORL_EXT] = tr[FORL_IDX] = idx;
 542    } else {
 543      ScEvEntry scev;
 544      rec_for_loop(J, fori, &scev, 0);
 545      t = scev.t.irt;
 546      stop = scev.stop;
 547    }
 548  } else {  /* Handle FORI/JFORI opcodes. */
 549    BCReg i;
 550    lj_meta_for(J->L, tv);
 551    t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) :
 552						   IRT_NUM;
 553    for (i = FORL_IDX; i <= FORL_STEP; i++) {
 554      if (!tr[i]) sload(J, ra+i);
 555      lj_assertJ(tref_isnumber_str(tr[i]), "bad FORI argument type");
 556      if (tref_isstr(tr[i]))
 557	tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0);
 558      tr[i] = fori_conv(J, tr[i], t);
 559    }
 560    tr[FORL_EXT] = tr[FORL_IDX];
 561    stop = tr[FORL_STOP];
 562    rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]),
 563		  stop, tr[FORL_STEP], 1);
 564  }
 565
 566  ev = rec_for_iter(&op, tv, isforl);
 567  if (ev == LOOPEV_LEAVE) {
 568    J->maxslot = ra+FORL_EXT+1;
 569    J->pc = fori+1;
 570  } else {
 571    J->maxslot = ra;
 572    J->pc = fori+bc_j(*fori)+1;
 573  }
 574  lj_snap_add(J);
 575
 576  emitir(IRTG(op, t), tr[FORL_IDX], stop);
 577
 578  if (ev == LOOPEV_LEAVE) {
 579    J->maxslot = ra;
 580    J->pc = fori+bc_j(*fori)+1;
 581  } else {
 582    J->maxslot = ra+FORL_EXT+1;
 583    J->pc = fori+1;
 584  }
 585  J->needsnap = 1;
 586  return ev;
 587}
 588
 589/* Record ITERL/JITERL. */
 590static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
 591{
 592  BCReg ra = bc_a(iterins);
 593  if (!tref_isnil(getslot(J, ra))) {  /* Looping back? */
 594    J->base[ra-1] = J->base[ra];  /* Copy result of ITERC to control var. */
 595    J->maxslot = ra-1+bc_b(J->pc[-1]);
 596    J->pc += bc_j(iterins)+1;
 597    return LOOPEV_ENTER;
 598  } else {
 599    J->maxslot = ra-3;
 600    J->pc++;
 601    return LOOPEV_LEAVE;
 602  }
 603}
 604
 605/* Record LOOP/JLOOP. Now, that was easy. */
 606static LoopEvent rec_loop(jit_State *J, BCReg ra, int skip)
 607{
 608  if (ra < J->maxslot) J->maxslot = ra;
 609  J->pc += skip;
 610  return LOOPEV_ENTER;
 611}
 612
 613/* Check if a loop repeatedly failed to trace because it didn't loop back. */
 614static int innerloopleft(jit_State *J, const BCIns *pc)
 615{
 616  ptrdiff_t i;
 617  for (i = 0; i < PENALTY_SLOTS; i++)
 618    if (mref(J->penalty[i].pc, const BCIns) == pc) {
 619      if ((J->penalty[i].reason == LJ_TRERR_LLEAVE ||
 620	   J->penalty[i].reason == LJ_TRERR_LINNER) &&
 621	  J->penalty[i].val >= 2*PENALTY_MIN)
 622	return 1;
 623      break;
 624    }
 625  return 0;
 626}
 627
 628/* Handle the case when an interpreted loop op is hit. */
 629static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
 630{
 631  if (J->parent == 0 && J->exitno == 0) {
 632    if (pc == J->startpc && J->framedepth + J->retdepth == 0) {
 633      if (bc_op(J->cur.startins) == BC_ITERN) return;  /* See rec_itern(). */
 634      /* Same loop? */
 635      if (ev == LOOPEV_LEAVE)  /* Must loop back to form a root trace. */
 636	lj_trace_err(J, LJ_TRERR_LLEAVE);
 637      lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Looping trace. */
 638    } else if (ev != LOOPEV_LEAVE) {  /* Entering inner loop? */
 639      /* It's usually better to abort here and wait until the inner loop
 640      ** is traced. But if the inner loop repeatedly didn't loop back,
 641      ** this indicates a low trip count. In this case try unrolling
 642      ** an inner loop even in a root trace. But it's better to be a bit
 643      ** more conservative here and only do it for very short loops.
 644      */
 645      if (bc_j(*pc) != -1 && !innerloopleft(J, pc))
 646	lj_trace_err(J, LJ_TRERR_LINNER);  /* Root trace hit an inner loop. */
 647      if ((ev != LOOPEV_ENTERLO &&
 648	   J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0)
 649	lj_trace_err(J, LJ_TRERR_LUNROLL);  /* Limit loop unrolling. */
 650      J->loopref = J->cur.nins;
 651    }
 652  } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters an inner loop. */
 653    J->loopref = J->cur.nins;
 654    if (--J->loopunroll < 0)
 655      lj_trace_err(J, LJ_TRERR_LUNROLL);  /* Limit loop unrolling. */
 656  }  /* Side trace continues across a loop that's left or not entered. */
 657}
 658
 659/* Handle the case when an already compiled loop op is hit. */
 660static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
 661{
 662  if (J->parent == 0 && J->exitno == 0) {  /* Root trace hit an inner loop. */
 663    /* Better let the inner loop spawn a side trace back here. */
 664    lj_trace_err(J, LJ_TRERR_LINNER);
 665  } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters a compiled loop. */
 666    J->instunroll = 0;  /* Cannot continue across a compiled loop op. */
 667    if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
 668      lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Form extra loop. */
 669    else
 670      lj_record_stop(J, LJ_TRLINK_ROOT, lnk);  /* Link to the loop. */
 671  }  /* Side trace continues across a loop that's left or not entered. */
 672}
 673
 674/* Record ITERN. */
 675static LoopEvent rec_itern(jit_State *J, BCReg ra, BCReg rb)
 676{
 677#if LJ_BE
 678  /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
 679  ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
 680  */
 681  UNUSED(ra); UNUSED(rb);
 682  setintV(&J->errinfo, (int32_t)BC_ITERN);
 683  lj_trace_err_info(J, LJ_TRERR_NYIBC);
 684#else
 685  RecordIndex ix;
 686  /* Since ITERN is recorded at the start, we need our own loop detection. */
 687  if (J->pc == J->startpc &&
 688      J->framedepth + J->retdepth == 0 && J->parent == 0 && J->exitno == 0) {
 689    IRRef ref = REF_FIRST + LJ_HASPROFILE;
 690#ifdef LUAJIT_ENABLE_CHECKHOOK
 691    ref += 3;
 692#endif
 693    if (J->cur.nins > ref ||
 694       (LJ_HASPROFILE && J->cur.nins == ref && J->cur.ir[ref-1].o != IR_PROF)) {
 695      J->instunroll = 0;  /* Cannot continue unrolling across an ITERN. */
 696      lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Looping trace. */
 697      return LOOPEV_ENTER;
 698    }
 699  }
 700  J->maxslot = ra;
 701  lj_snap_add(J);  /* Required to make JLOOP the first ins in a side-trace. */
 702  ix.tab = getslot(J, ra-2);
 703  ix.key = J->base[ra-1] ? J->base[ra-1] :
 704	   sloadt(J, (int32_t)(ra-1), IRT_GUARD|IRT_INT,
 705		  IRSLOAD_TYPECHECK|IRSLOAD_KEYINDEX);
 706  copyTV(J->L, &ix.tabv, &J->L->base[ra-2]);
 707  copyTV(J->L, &ix.keyv, &J->L->base[ra-1]);
 708  ix.idxchain = (rb < 3);  /* Omit value type check, if unused. */
 709  ix.mobj = 1;  /* We need the next index, too. */
 710  J->maxslot = ra + lj_record_next(J, &ix);
 711  J->needsnap = 1;
 712  if (!tref_isnil(ix.key)) {  /* Looping back? */
 713    J->base[ra-1] = ix.mobj | TREF_KEYINDEX;  /* Control var has next index. */
 714    J->base[ra] = ix.key;
 715    J->base[ra+1] = ix.val;
 716    J->pc += bc_j(J->pc[1])+2;
 717    return LOOPEV_ENTER;
 718  } else {
 719    J->maxslot = ra-3;
 720    J->pc += 2;
 721    return LOOPEV_LEAVE;
 722  }
 723#endif
 724}
 725
 726/* Record ISNEXT. */
 727static void rec_isnext(jit_State *J, BCReg ra)
 728{
 729  cTValue *b = &J->L->base[ra-3];
 730  if (tvisfunc(b) && funcV(b)->c.ffid == FF_next &&
 731      tvistab(b+1) && tvisnil(b+2)) {
 732    /* These checks are folded away for a compiled pairs(). */
 733    TRef func = getslot(J, ra-3);
 734    TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), func, IRFL_FUNC_FFID);
 735    emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, FF_next));
 736    (void)getslot(J, ra-2); /* Type check for table. */
 737    (void)getslot(J, ra-1); /* Type check for nil key. */
 738    J->base[ra-1] = lj_ir_kint(J, 0) | TREF_KEYINDEX;
 739    J->maxslot = ra;
 740  } else {  /* Abort trace. Interpreter will despecialize bytecode. */
 741    lj_trace_err(J, LJ_TRERR_RECERR);
 742  }
 743}
 744
 745/* -- Record profiler hook checks ----------------------------------------- */
 746
 747#if LJ_HASPROFILE
 748
 749/* Need to insert profiler hook check? */
 750static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc)
 751{
 752  GCproto *ppt;
 753  lj_assertJ(J->prof_mode == 'f' || J->prof_mode == 'l',
 754	     "bad profiler mode %c", J->prof_mode);
 755  if (!pt)
 756    return 0;
 757  ppt = J->prev_pt;
 758  J->prev_pt = pt;
 759  if (pt != ppt && ppt) {
 760    J->prev_line = -1;
 761    return 1;
 762  }
 763  if (J->prof_mode == 'l') {
 764    BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc));
 765    BCLine pline = J->prev_line;
 766    J->prev_line = line;
 767    if (pline != line)
 768      return 1;
 769  }
 770  return 0;
 771}
 772
 773static void rec_profile_ins(jit_State *J, const BCIns *pc)
 774{
 775  if (J->prof_mode && rec_profile_need(J, J->pt, pc)) {
 776    emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
 777    lj_snap_add(J);
 778  }
 779}
 780
 781static void rec_profile_ret(jit_State *J)
 782{
 783  if (J->prof_mode == 'f') {
 784    emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
 785    J->prev_pt = NULL;
 786    lj_snap_add(J);
 787  }
 788}
 789
 790#endif
 791
 792/* -- Record calls and returns -------------------------------------------- */
 793
 794/* Specialize to the runtime value of the called function or its prototype. */
 795static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
 796{
 797  TRef kfunc;
 798  if (isluafunc(fn)) {
 799    GCproto *pt = funcproto(fn);
 800    /* Too many closures created? Probably not a monomorphic function. */
 801    if (pt->flags >= PROTO_CLC_POLY) {  /* Specialize to prototype instead. */
 802      TRef trpt = emitir(IRT(IR_FLOAD, IRT_PGC), tr, IRFL_FUNC_PC);
 803      emitir(IRTG(IR_EQ, IRT_PGC), trpt, lj_ir_kptr(J, proto_bc(pt)));
 804      (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);  /* Prevent GC of proto. */
 805      return tr;
 806    }
 807  } else {
 808    /* Don't specialize to non-monomorphic builtins. */
 809    switch (fn->c.ffid) {
 810    case FF_coroutine_wrap_aux:
 811    case FF_string_gmatch_aux:
 812      /* NYI: io_file_iter doesn't have an ffid, yet. */
 813      {  /* Specialize to the ffid. */
 814	TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID);
 815	emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, fn->c.ffid));
 816      }
 817      return tr;
 818    default:
 819      /* NYI: don't specialize to non-monomorphic C functions. */
 820      break;
 821    }
 822  }
 823  /* Otherwise specialize to the function (closure) value itself. */
 824  kfunc = lj_ir_kfunc(J, fn);
 825  emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc);
 826  return kfunc;
 827}
 828
 829/* Record call setup. */
 830static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs)
 831{
 832  RecordIndex ix;
 833  TValue *functv = &J->L->base[func];
 834  TRef kfunc, *fbase = &J->base[func];
 835  ptrdiff_t i;
 836  (void)getslot(J, func); /* Ensure func has a reference. */
 837  for (i = 1; i <= nargs; i++)
 838    (void)getslot(J, func+LJ_FR2+i);  /* Ensure all args have a reference. */
 839  if (!tref_isfunc(fbase[0])) {  /* Resolve __call metamethod. */
 840    ix.tab = fbase[0];
 841    copyTV(J->L, &ix.tabv, functv);
 842    if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
 843      lj_trace_err(J, LJ_TRERR_NOMM);
 844    for (i = ++nargs; i > LJ_FR2; i--)  /* Shift arguments up. */
 845      fbase[i+LJ_FR2] = fbase[i+LJ_FR2-1];
 846#if LJ_FR2
 847    fbase[2] = fbase[0];
 848#endif
 849    fbase[0] = ix.mobj;  /* Replace function. */
 850    functv = &ix.mobjv;
 851  }
 852  kfunc = rec_call_specialize(J, funcV(functv), fbase[0]);
 853#if LJ_FR2
 854  fbase[0] = kfunc;
 855  fbase[1] = TREF_FRAME;
 856#else
 857  fbase[0] = kfunc | TREF_FRAME;
 858#endif
 859  J->maxslot = (BCReg)nargs;
 860}
 861
 862/* Record call. */
 863void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs)
 864{
 865  rec_call_setup(J, func, nargs);
 866  /* Bump frame. */
 867  J->framedepth++;
 868  J->base += func+1+LJ_FR2;
 869  J->baseslot += func+1+LJ_FR2;
 870  if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
 871    lj_trace_err(J, LJ_TRERR_STACKOV);
 872}
 873
 874/* Record tail call. */
 875void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
 876{
 877  rec_call_setup(J, func, nargs);
 878  if (frame_isvarg(J->L->base - 1)) {
 879    BCReg cbase = (BCReg)frame_delta(J->L->base - 1);
 880    if (--J->framedepth < 0)
 881      lj_trace_err(J, LJ_TRERR_NYIRETL);
 882    J->baseslot -= (BCReg)cbase;
 883    J->base -= cbase;
 884    func += cbase;
 885  }
 886  /* Move func + args down. */
 887  if (LJ_FR2 && J->baseslot == 2)
 888    J->base[func+1] = TREF_FRAME;
 889  memmove(&J->base[-1-LJ_FR2], &J->base[func], sizeof(TRef)*(J->maxslot+1+LJ_FR2));
 890  /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
 891  /* Tailcalls can form a loop, so count towards the loop unroll limit. */
 892  if (++J->tailcalled > J->loopunroll)
 893    lj_trace_err(J, LJ_TRERR_LUNROLL);
 894}
 895
 896/* Check unroll limits for down-recursion. */
 897static int check_downrec_unroll(jit_State *J, GCproto *pt)
 898{
 899  IRRef ptref;
 900  for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev)
 901    if (ir_kgc(IR(ptref)) == obj2gco(pt)) {
 902      int count = 0;
 903      IRRef ref;
 904      for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev)
 905	if (IR(ref)->op1 == ptref)
 906	  count++;
 907      if (count) {
 908	if (J->pc == J->startpc) {
 909	  if (count + J->tailcalled > J->param[JIT_P_recunroll])
 910	    return 1;
 911	} else {
 912	  lj_trace_err(J, LJ_TRERR_DOWNREC);
 913	}
 914      }
 915    }
 916  return 0;
 917}
 918
 919static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);
 920
 921/* Record return. */
 922void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
 923{
 924  TValue *frame = J->L->base - 1;
 925  ptrdiff_t i;
 926  BCReg baseadj = 0;
 927  for (i = 0; i < gotresults; i++)
 928    (void)getslot(J, rbase+i);  /* Ensure all results have a reference. */
 929  while (frame_ispcall(frame)) {  /* Immediately resolve pcall() returns. */
 930    BCReg cbase = (BCReg)frame_delta(frame);
 931    if (--J->framedepth <= 0)
 932      lj_trace_err(J, LJ_TRERR_NYIRETL);
 933    lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
 934    gotresults++;
 935    baseadj += cbase;
 936    rbase += cbase;
 937    J->baseslot -= (BCReg)cbase;
 938    J->base -= cbase;
 939    J->base[--rbase] = TREF_TRUE;  /* Prepend true to results. */
 940    frame = frame_prevd(frame);
 941    J->needsnap = 1;  /* Stop catching on-trace errors. */
 942  }
 943  /* Return to lower frame via interpreter for unhandled cases. */
 944  if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) &&
 945       (!frame_islua(frame) ||
 946	(J->parent == 0 && J->exitno == 0 &&
 947	 !bc_isret(bc_op(J->cur.startins))))) {
 948    /* NYI: specialize to frame type and return directly, not via RET*. */
 949    for (i = 0; i < (ptrdiff_t)rbase; i++)
 950      J->base[i] = 0;  /* Purge dead slots. */
 951    J->maxslot = rbase + (BCReg)gotresults;
 952    lj_record_stop(J, LJ_TRLINK_RETURN, 0);  /* Return to interpreter. */
 953    return;
 954  }
 955  if (frame_isvarg(frame)) {
 956    BCReg cbase = (BCReg)frame_delta(frame);
 957    if (--J->framedepth < 0)  /* NYI: return of vararg func to lower frame. */
 958      lj_trace_err(J, LJ_TRERR_NYIRETL);
 959    lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
 960    baseadj += cbase;
 961    rbase += cbase;
 962    J->baseslot -= (BCReg)cbase;
 963    J->base -= cbase;
 964    frame = frame_prevd(frame);
 965  }
 966  if (frame_islua(frame)) {  /* Return to Lua frame. */
 967    BCIns callins = *(frame_pc(frame)-1);
 968    ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
 969    BCReg cbase = bc_a(callins);
 970    GCproto *pt = funcproto(frame_func(frame - (cbase+1+LJ_FR2)));
 971    if ((pt->flags & PROTO_NOJIT))
 972      lj_trace_err(J, LJ_TRERR_CJITOFF);
 973    if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) {
 974      if (!J->cur.root && check_downrec_unroll(J, pt)) {
 975	J->maxslot = (BCReg)(rbase + gotresults);
 976	lj_snap_purge(J);
 977	lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno);  /* Down-rec. */
 978	return;
 979      }
 980      lj_snap_add(J);
 981    }
 982    for (i = 0; i < nresults; i++)  /* Adjust results. */
 983      J->base[i-1-LJ_FR2] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
 984    J->maxslot = cbase+(BCReg)nresults;
 985    if (J->framedepth > 0) {  /* Return to a frame that is part of the trace. */
 986      J->framedepth--;
 987      lj_assertJ(J->baseslot > cbase+1+LJ_FR2, "bad baseslot for return");
 988      J->baseslot -= cbase+1+LJ_FR2;
 989      J->base -= cbase+1+LJ_FR2;
 990    } else if (J->parent == 0 && J->exitno == 0 &&
 991	       !bc_isret(bc_op(J->cur.startins))) {
 992      /* Return to lower frame would leave the loop in a root trace. */
 993      lj_trace_err(J, LJ_TRERR_LLEAVE);
 994    } else if (J->needsnap) {  /* Tailcalled to ff with side-effects. */
 995      lj_trace_err(J, LJ_TRERR_NYIRETL);  /* No way to insert snapshot here. */
 996    } else if (1 + pt->framesize >= LJ_MAX_JSLOTS ||
 997	       J->baseslot + J->maxslot >= LJ_MAX_JSLOTS) {
 998      lj_trace_err(J, LJ_TRERR_STACKOV);
 999    } else {  /* Return to lower frame. Guard for the target we return to. */
1000      TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
1001      TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
1002      emitir(IRTG(IR_RETF, IRT_PGC), trpt, trpc);
1003      J->retdepth++;
1004      J->needsnap = 1;
1005      J->scev.idx = REF_NIL;
1006      lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot for return");
1007      /* Shift result slots up and clear the slots of the new frame below. */
1008      memmove(J->base + cbase, J->base-1-LJ_FR2, sizeof(TRef)*nresults);
1009      memset(J->base-1-LJ_FR2, 0, sizeof(TRef)*(cbase+1+LJ_FR2));
1010    }
1011  } else if (frame_iscont(frame)) {  /* Return to continuation frame. */
1012    ASMFunction cont = frame_contf(frame);
1013    BCReg cbase = (BCReg)frame_delta(frame);
1014    if ((J->framedepth -= 2) < 0)
1015      lj_trace_err(J, LJ_TRERR_NYIRETL);
1016    J->baseslot -= (BCReg)cbase;
1017    J->base -= cbase;
1018    J->maxslot = cbase-(2<<LJ_FR2);
1019    if (cont == lj_cont_ra) {
1020      /* Copy result to destination slot. */
1021      BCReg dst = bc_a(*(frame_contpc(frame)-1));
1022      J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
1023      if (dst >= J->maxslot) {
1024	J->maxslot = dst+1;
1025      }
1026    } else if (cont == lj_cont_nop) {
1027      /* Nothing to do here. */
1028    } else if (cont == lj_cont_cat) {
1029      BCReg bslot = bc_b(*(frame_contpc(frame)-1));
1030      TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
1031      if (bslot != J->maxslot) {  /* Concatenate the remainder. */
1032	/* Simulate lower frame and result. */
1033	TValue *b = J->L->base - baseadj, save;
1034	/* Can't handle MM_concat + CALLT + fast func side-effects. */
1035	if (J->postproc != LJ_POST_NONE)
1036	  lj_trace_err(J, LJ_TRERR_NYIRETL);
1037	J->base[J->maxslot] = tr;
1038	copyTV(J->L, &save, b-(2<<LJ_FR2));
1039	if (gotresults)
1040	  copyTV(J->L, b-(2<<LJ_FR2), b+rbase);
1041	else
1042	  setnilV(b-(2<<LJ_FR2));
1043	J->L->base = b - cbase;
1044	tr = rec_cat(J, bslot, cbase-(2<<LJ_FR2));
1045	b = J->L->base + cbase;  /* Undo. */
1046	J->L->base = b + baseadj;
1047	copyTV(J->L, b-(2<<LJ_FR2), &save);
1048      }
1049      if (tr >= 0xffffff00) {
1050	lj_err_throw(J->L, -(int32_t)tr);  /* Propagate errors. */
1051      } else if (tr) {  /* Store final result. */
1052	BCReg dst = bc_a(*(frame_contpc(frame)-1));
1053	J->base[dst] = tr;
1054	if (dst >= J->maxslot) {
1055	  J->maxslot = dst+1;
1056	}
1057      }  /* Otherwise continue with another __concat call. */
1058    } else {
1059      /* Result type already specialized. */
1060      lj_assertJ(cont == lj_cont_condf || cont == lj_cont_condt,
1061		 "bad continuation type");
1062    }
1063  } else {
1064    lj_trace_err(J, LJ_TRERR_NYIRETL);  /* NYI: handle return to C frame. */
1065  }
1066  lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot for return");
1067}
1068
1069/* -- Metamethod handling ------------------------------------------------- */
1070
1071/* Prepare to record call to metamethod. */
1072static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
1073{
1074  BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
1075#if LJ_FR2
1076  J->base[top] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
1077  J->base[top+1] = TREF_CONT;
1078#else
1079  J->base[top] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
1080#endif
1081  J->framedepth++;
1082  for (s = J->maxslot; s < top; s++)
1083    J->base[s] = 0;  /* Clear frame gap to avoid resurrecting previous refs. */
1084  return top+1+LJ_FR2;
1085}
1086
1087/* Record metamethod lookup. */
1088int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
1089{
1090  RecordIndex mix;
1091  GCtab *mt;
1092  if (tref_istab(ix->tab)) {
1093    mt = tabref(tabV(&ix->tabv)->metatable);
1094    mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
1095  } else if (tref_isudata(ix->tab)) {
1096    int udtype = udataV(&ix->tabv)->udtype;
1097    mt = tabref(udataV(&ix->tabv)->metatable);
1098    mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META);
1099    /* The metatables of special userdata objects are treated as immutable. */
1100    if (udtype != UDTYPE_USERDATA) {
1101      cTValue *mo;
1102      if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) {
1103	/* Specialize to the C library namespace object. */
1104	emitir(IRTG(IR_EQ, IRT_PGC), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv)));
1105      } else {
1106	/* Specialize to the type of userdata. */
1107	TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE);
1108	emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype));
1109      }
1110  immutable_mt:
1111      mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm));
1112      ix->mt = mix.tab;
1113      ix->mtv = mt;
1114      if (!mo || tvisnil(mo))
1115	return 0;  /* No metamethod. */
1116      /* Treat metamethod or index table as immutable, too. */
1117      if (!(tvisfunc(mo) || tvistab(mo)))
1118	lj_trace_err(J, LJ_TRERR_BADTYPE);
1119      copyTV(J->L, &ix->mobjv, mo);
1120      ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB);
1121      return 1;  /* Got metamethod or index table. */
1122    }
1123  } else {
1124    /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
1125    mt = tabref(basemt_obj(J2G(J), &ix->tabv));
1126    if (mt == NULL) {
1127      ix->mt = TREF_NIL;
1128      return 0;  /* No metamethod. */
1129    }
1130    /* The cdata metatable is treated as immutable. */
1131    if (LJ_HASFFI && tref_iscdata(ix->tab)) {
1132      mix.tab = TREF_NIL;
1133      goto immutable_mt;
1134    }
1135    ix->mt = mix.tab = lj_ir_ggfload(J, IRT_TAB,
1136      GG_OFS(g.gcroot[GCROOT_BASEMT+itypemap(&ix->tabv)]));
1137    goto nocheck;
1138  }
1139  ix->mt = mt ? mix.tab : TREF_NIL;
1140  emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB));
1141nocheck:
1142  if (mt) {
1143    GCstr *mmstr = mmname_str(J2G(J), mm);
1144    cTValue *mo = lj_tab_getstr(mt, mmstr);
1145    if (mo && !tvisnil(mo))
1146      copyTV(J->L, &ix->mobjv, mo);
1147    ix->mtv = mt;
1148    settabV(J->L, &mix.tabv, mt);
1149    setstrV(J->L, &mix.keyv, mmstr);
1150    mix.key = lj_ir_kstr(J, mmstr);
1151    mix.val = 0;
1152    mix.idxchain = 0;
1153    ix->mobj = lj_record_idx(J, &mix);
1154    return !tref_isnil(ix->mobj);  /* 1 if metamethod found, 0 if not. */
1155  }
1156  return 0;  /* No metamethod. */
1157}
1158
1159/* Record call to arithmetic metamethod. */
1160static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
1161{
1162  /* Set up metamethod call first to save ix->tab and ix->tabv. */
1163  BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
1164  TRef *base = J->base + func;
1165  TValue *basev = J->L->base + func;
1166  base[1+LJ_FR2] = ix->tab; base[2+LJ_FR2] = ix->key;
1167  copyTV(J->L, basev+1+LJ_FR2, &ix->tabv);
1168  copyTV(J->L, basev+2+LJ_FR2, &ix->keyv);
1169  if (!lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
1170    if (mm != MM_unm) {
1171      ix->tab = ix->key;
1172      copyTV(J->L, &ix->tabv, &ix->keyv);
1173      if (lj_record_mm_lookup(J, ix, mm))  /* Lookup mm on 2nd operand. */
1174	goto ok;
1175    }
1176    lj_trace_err(J, LJ_TRERR_NOMM);
1177  }
1178ok:
1179  base[0] = ix->mobj;
1180#if LJ_FR2
1181  base[1] = 0;
1182#endif
1183  copyTV(J->L, basev+0, &ix->mobjv);
1184  lj_record_call(J, func, 2);
1185  return 0;  /* No result yet. */
1186}
1187
1188/* Record call to __len metamethod. */
1189static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
1190{
1191  RecordIndex ix;
1192  ix.tab = tr;
1193  copyTV(J->L, &ix.tabv, tv);
1194  if (lj_record_mm_lookup(J, &ix, MM_len)) {
1195    BCReg func = rec_mm_prep(J, lj_cont_ra);
1196    TRef *base = J->base + func;
1197    TValue *basev = J->L->base + func;
1198    base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv);
1199    base += LJ_FR2;
1200    basev += LJ_FR2;
1201    base[1] = tr; copyTV(J->L, basev+1, tv);
1202#if LJ_52
1203    base[2] = tr; copyTV(J->L, basev+2, tv);
1204#else
1205    base[2] = TREF_NIL; setnilV(basev+2);
1206#endif
1207    lj_record_call(J, func, 2);
1208  } else {
1209    if (LJ_52 && tref_istab(tr))
1210      return emitir(IRTI(IR_ALEN), tr, TREF_NIL);
1211    lj_trace_err(J, LJ_TRERR_NOMM);
1212  }
1213  return 0;  /* No result yet. */
1214}
1215
1216/* Call a comparison metamethod. */
1217static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
1218{
1219  BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
1220  TRef *base = J->base + func + LJ_FR2;
1221  TValue *tv = J->L->base + func + LJ_FR2;
1222  base[-LJ_FR2] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
1223  copyTV(J->L, tv-LJ_FR2, &ix->mobjv);
1224  copyTV(J->L, tv+1, &ix->valv);
1225  copyTV(J->L, tv+2, &ix->keyv);
1226  lj_record_call(J, func, 2);
1227}
1228
1229/* Record call to equality comparison metamethod (for tab and udata only). */
1230static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op)
1231{
1232  ix->tab = ix->val;
1233  copyTV(J->L, &ix->tabv, &ix->valv);
1234  if (lj_record_mm_lookup(J, ix, MM_eq)) {  /* Lookup mm on 1st operand. */
1235    cTValue *bv;
1236    TRef mo1 = ix->mobj;
1237    TValue mo1v;
1238    copyTV(J->L, &mo1v, &ix->mobjv);
1239    /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1240    bv = &ix->keyv;
1241    if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1242      TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1243      emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1244    } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1245      TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1246      emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1247    } else {  /* Lookup metamethod on 2nd operand and compare both. */
1248      ix->tab = ix->key;
1249      copyTV(J->L, &ix->tabv, bv);
1250      if (!lj_record_mm_lookup(J, ix, MM_eq) ||
1251	  lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1252	return;
1253    }
1254    rec_mm_callcomp(J, ix, op);
1255  }
1256}
1257
1258/* Record call to ordered comparison metamethods (for arbitrary objects). */
1259static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
1260{
1261  ix->tab = ix->val;
1262  copyTV(J->L, &ix->tabv, &ix->valv);
1263  while (1) {
1264    MMS mm = (op & 2) ? MM_le : MM_lt;  /* Try __le + __lt or only __lt. */
1265#if LJ_52
1266    if (!lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
1267      ix->tab = ix->key;
1268      copyTV(J->L, &ix->tabv, &ix->keyv);
1269      if (!lj_record_mm_lookup(J, ix, mm))  /* Lookup mm on 2nd operand. */
1270	goto nomatch;
1271    }
1272    rec_mm_callcomp(J, ix, op);
1273    return;
1274#else
1275    if (lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
1276      cTValue *bv;
1277      TRef mo1 = ix->mobj;
1278      TValue mo1v;
1279      copyTV(J->L, &mo1v, &ix->mobjv);
1280      /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1281      bv = &ix->keyv;
1282      if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1283	TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1284	emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1285      } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1286	TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1287	emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1288      } else {  /* Lookup metamethod on 2nd operand and compare both. */
1289	ix->tab = ix->key;
1290	copyTV(J->L, &ix->tabv, bv);
1291	if (!lj_record_mm_lookup(J, ix, mm) ||
1292	    lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1293	  goto nomatch;
1294      }
1295      rec_mm_callcomp(J, ix, op);
1296      return;
1297    }
1298#endif
1299  nomatch:
1300    /* Lookup failed. Retry with  __lt and swapped operands. */
1301    if (!(op & 2)) break;  /* Already at __lt. Interpreter will throw. */
1302    ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab;
1303    copyTV(J->L, &ix->tabv, &ix->keyv);
1304    copyTV(J->L, &ix->keyv, &ix->valv);
1305    copyTV(J->L, &ix->valv, &ix->tabv);
1306    op ^= 3;
1307  }
1308}
1309
1310#if LJ_HASFFI
1311/* Setup call to cdata comparison metamethod. */
1312static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
1313{
1314  lj_snap_add(J);
1315  if (tref_iscdata(ix->val)) {
1316    ix->tab = ix->val;
1317    copyTV(J->L, &ix->tabv, &ix->valv);
1318  } else {
1319    lj_assertJ(tref_iscdata(ix->key), "cdata expected");
1320    ix->tab = ix->key;
1321    copyTV(J->L, &ix->tabv, &ix->keyv);
1322  }
1323  lj_record_mm_lookup(J, ix, mm);
1324  rec_mm_callcomp(J, ix, op);
1325}
1326#endif
1327
1328/* -- Indexed access ------------------------------------------------------ */
1329
1330#ifdef LUAJIT_ENABLE_TABLE_BUMP
1331/* Bump table allocations in bytecode when they grow during recording. */
1332static void rec_idx_bump(jit_State *J, RecordIndex *ix)
1333{
1334  RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))];
1335  if (tref_ref(ix->tab) == rbc->ref) {
1336    const BCIns *pc = mref(rbc->pc, const BCIns);
1337    GCtab *tb = tabV(&ix->tabv);
1338    uint32_t nhbits;
1339    IRIns *ir;
1340    if (!tvisnil(&ix->keyv))
1341      (void)lj_tab_set(J->L, tb, &ix->keyv);  /* Grow table right now. */
1342    nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0;
1343    ir = IR(tref_ref(ix->tab));
1344    if (ir->o == IR_TNEW) {
1345      uint32_t ah = bc_d(*pc);
1346      uint32_t asize = ah & 0x7ff, hbits = ah >> 11;
1347      if (nhbits > hbits) hbits = nhbits;
1348      if (tb->asize > asize) {
1349	asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff;
1350      }
1351      if ((asize | (hbits<<11)) != ah) {  /* Has the size changed? */
1352	/* Patch bytecode, but continue recording (for more patching). */
1353	setbc_d(pc, (asize | (hbits<<11)));
1354	/* Patching TNEW operands is only safe if the trace is aborted. */
1355	ir->op1 = asize; ir->op2 = hbits;
1356	J->retryrec = 1;  /* Abort the trace at the end of recording. */
1357      }
1358    } else if (ir->o == IR_TDUP) {
1359      GCtab *tpl = gco2tab(proto_kgc(&gcref(rbc->pt)->pt, ~(ptrdiff_t)bc_d(*pc)));
1360      /* Grow template table, but preserve keys with nil values. */
1361      if ((tb->asize > tpl->asize && (1u << nhbits)-1 == tpl->hmask) ||
1362	  (tb->asize == tpl->asize && (1u << nhbits)-1 > tpl->hmask)) {
1363	Node *node = noderef(tpl->node);
1364	uint32_t i, hmask = tpl->hmask, asize;
1365	TValue *array;
1366	for (i = 0; i <= hmask; i++) {
1367	  if (!tvisnil(&node[i].key) && tvisnil(&node[i].val))
1368	    settabV(J->L, &node[i].val, tpl);
1369	}
1370	if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) {
1371	  TValue *o = lj_tab_set(J->L, tpl, &ix->keyv);
1372	  if (tvisnil(o)) settabV(J->L, o, tpl);
1373	}
1374	lj_tab_resize(J->L, tpl, tb->asize, nhbits);
1375	node = noderef(tpl->node);
1376	hmask = tpl->hmask;
1377	for (i = 0; i <= hmask; i++) {
1378	  /* This is safe, since template tables only hold immutable values. */
1379	  if (tvistab(&node[i].val))
1380	    setnilV(&node[i].val);
1381	}
1382	/* The shape of the table may have changed. Clean up array part, too. */
1383	asize = tpl->asize;
1384	array = tvref(tpl->array);
1385	for (i = 0; i < asize; i++) {
1386	  if (tvistab(&array[i]))
1387	    setnilV(&array[i]);
1388	}
1389	J->retryrec = 1;  /* Abort the trace at the end of recording. */
1390      }
1391    }
1392  }
1393}
1394#endif
1395
1396/* Record bounds-check. */
1397static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1398{
1399  /* Try to emit invariant bounds checks. */
1400  if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) ==
1401      (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) {
1402    IRRef ref = tref_ref(ikey);
1403    IRIns *ir = IR(ref);
1404    int32_t ofs = 0;
1405    IRRef ofsref = 0;
1406    /* Handle constant offsets. */
1407    if (ir->o == IR_ADD && irref_isk(ir->op2)) {
1408      ofsref = ir->op2;
1409      ofs = IR(ofsref)->i;
1410      ref = ir->op1;
1411      ir = IR(ref);
1412    }
1413    /* Got scalar evolution analysis results for this reference? */
1414    if (ref == J->scev.idx) {
1415      int32_t stop;
1416      lj_assertJ(irt_isint(J->scev.t) && ir->o == IR_SLOAD,
1417		 "only int SCEV supported");
1418      stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]);
1419      /* Runtime value for stop of loop is within bounds? */
1420      if ((uint64_t)stop + ofs < (uint64_t)asize) {
1421	/* Emit invariant bounds check for stop. */
1422	uint32_t abc = IRTG(IR_ABC, tref_isk(asizeref) ? IRT_U32 : IRT_P32);
1423	emitir(abc, asizeref, ofs == 0 ? J->scev.stop :
1424	       emitir(IRTI(IR_ADD), J->scev.stop, ofsref));
1425	/* Emit invariant bounds check for start, if not const or negative. */
1426	if (!(J->scev.dir && J->scev.start &&
1427	      (int64_t)IR(J->scev.start)->i + ofs >= 0))
1428	  emitir(abc, asizeref, ikey);
1429	return;
1430      }
1431    }
1432  }
1433  emitir(IRTGI(IR_ABC), asizeref, ikey);  /* Emit regular bounds check. */
1434}
1435
1436/* Record indexed key lookup. */
1437static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref,
1438			IRType1 *rbguard)
1439{
1440  TRef key;
1441  GCtab *t = tabV(&ix->tabv);
1442  ix->oldv = lj_tab_get(J->L, t, &ix->keyv);  /* Lookup previous value. */
1443  *rbref = 0;
1444  rbguard->irt = 0;
1445
1446  /* Integer keys are looked up in the array part first. */
1447  key = ix->key;
1448  if (tref_isnumber(key)) {
1449    int32_t k;
1450    if (tvisint(&ix->keyv)) {
1451      k = intV(&ix->keyv);
1452    } else {
1453      int64_t i64;
1454      if (!lj_num2int_check(numV(&ix->keyv), i64, k)) k = LJ_MAX_ASIZE;
1455    }
1456    if ((MSize)k < LJ_MAX_ASIZE) {  /* Potential array key? */
1457      TRef ikey = lj_opt_narrow_index(J, key);
1458      TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1459      if ((MSize)k < t->asize) {  /* Currently an array key? */
1460	TRef arrayref;
1461	rec_idx_abc(J, asizeref, ikey, t->asize);
1462	arrayref = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_ARRAY);
1463	return emitir(IRT(IR_AREF, IRT_PGC), arrayref, ikey);
1464      } else {  /* Currently not in array (may be an array extension)? */
1465	emitir(IRTGI(IR_ULE), asizeref, ikey);  /* Inv. bounds check. */
1466	if (k == 0 && tref_isk(key))
1467	  key = lj_ir_knum_zero(J);  /* Canonicalize 0 or +-0.0 to +0.0. */
1468	/* And continue with the hash lookup. */
1469      }
1470    } else if (!tref_isk(key)) {
1471      /* We can rule out const numbers which failed the integerness test
1472      ** above. But all other numbers are potential array keys.
1473      */
1474      if (t->asize == 0) {  /* True sparse tables have an empty array part. */
1475	/* Guard that the array part stays empty. */
1476	TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1477	emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1478      } else {
1479	lj_trace_err(J, LJ_TRERR_NYITMIX);
1480      }
1481    }
1482  }
1483
1484  /* Otherwise the key is located in the hash part. */
1485  if (t->hmask == 0) {  /* Shortcut for empty hash part. */
1486    /* Guard that the hash part stays empty. */
1487    TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1488    emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1489    return lj_ir_kkptr(J, niltvg(J2G(J)));
1490  }
1491  if (tref_isinteger(key))  /* Hash keys are based on numbers, not ints. */
1492    key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1493  if (tref_isk(key)) {
1494    /* Optimize lookup of constant hash keys. */
1495    GCSize hslot = (GCSize)((char *)ix->oldv-(char *)&noderef(t->node)[0].val);
1496    if (hslot <= t->hmask*(GCSize)sizeof(Node) &&
1497	hslot <= 65535*(GCSize)sizeof(Node)) {
1498      TRef node, kslot, hm;
1499      *rbref = J->cur.nins;  /* Mark possible rollback point. */
1500      *rbguard = J->guardemit;
1501      hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1502      emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
1503      node = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_NODE);
1504      kslot = lj_ir_kslot(J, key, (IRRef)(hslot / sizeof(Node)));
1505      return emitir(IRTG(IR_HREFK, IRT_PGC), node, kslot);
1506    }
1507  }
1508  /* Fall back to a regular hash lookup. */
1509  return emitir(IRT(IR_HREF, IRT_PGC), ix->tab, key);
1510}
1511
1512/* Determine whether a key is NOT one of the fast metamethod names. */
1513static int nommstr(jit_State *J, TRef key)
1514{
1515  if (tref_isstr(key)) {
1516    if (tref_isk(key)) {
1517      GCstr *str = ir_kstr(IR(tref_ref(key)));
1518      uint32_t mm;
1519      for (mm = 0; mm <= MM_FAST; mm++)
1520	if (mmname_str(J2G(J), mm) == str)
1521	  return 0;  /* MUST be one the fast metamethod names. */
1522    } else {
1523      return 0;  /* Variable string key MAY be a metamethod name. */
1524    }
1525  }
1526  return 1;  /* CANNOT be a metamethod name. */
1527}
1528
1529/* Record indexed load/store. */
1530TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1531{
1532  TRef xref;
1533  IROp xrefop, loadop;
1534  IRRef rbref;
1535  IRType1 rbguard;
1536  cTValue *oldv;
1537
1538  while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
1539    /* Never call raw lj_record_idx() on non-table. */
1540    lj_assertJ(ix->idxchain != 0, "bad usage");
1541    if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
1542      lj_trace_err(J, LJ_TRERR_NOMM);
1543  handlemm:
1544    if (tref_isfunc(ix->mobj)) {  /* Handle metamethod call. */
1545      BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
1546      TRef *base = J->base + func + LJ_FR2;
1547      TValue *tv = J->L->base + func + LJ_FR2;
1548      base[-LJ_FR2] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
1549      setfuncV(J->L, tv-LJ_FR2, funcV(&ix->mobjv));
1550      copyTV(J->L, tv+1, &ix->tabv);
1551      copyTV(J->L, tv+2, &ix->keyv);
1552      if (ix->val) {
1553	base[3] = ix->val;
1554	copyTV(J->L, tv+3, &ix->valv);
1555	lj_record_call(J, func, 3);  /* mobj(tab, key, val) */
1556	return 0;
1557      } else {
1558	lj_record_call(J, func, 2);  /* res = mobj(tab, key) */
1559	return 0;  /* No result yet. */
1560      }
1561    }
1562#if LJ_HASBUFFER
1563    /* The index table of buffer objects is treated as immutable. */
1564    if (ix->mt == TREF_NIL && !ix->val &&
1565	tref_isudata(ix->tab) && udataV(&ix->tabv)->udtype == UDTYPE_BUFFER &&
1566	tref_istab(ix->mobj) && tref_isstr(ix->key) && tref_isk(ix->key)) {
1567      cTValue *val = lj_tab_getstr(tabV(&ix->mobjv), strV(&ix->keyv));
1568      TRef tr = lj_record_constify(J, val);
1569      if (tr) return tr;  /* Specialize to the value, i.e. a method. */
1570    }
1571#endif
1572    /* Otherwise retry lookup with metaobject. */
1573    ix->tab = ix->mobj;
1574    copyTV(J->L, &ix->tabv, &ix->mobjv);
1575    if (--ix->idxchain == 0)
1576      lj_trace_err(J, LJ_TRERR_IDXLOOP);
1577  }
1578
1579  /* First catch nil and NaN keys for tables. */
1580  if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) {
1581    if (ix->val)  /* Better fail early. */
1582      lj_trace_err(J, LJ_TRERR_STORENN);
1583    if (tref_isk(ix->key)) {
1584      if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1585	goto handlemm;
1586      return TREF_NIL;
1587    }
1588  }
1589
1590  /* Record the key lookup. */
1591  xref = rec_idx_key(J, ix, &rbref, &rbguard);
1592  xrefop = IR(tref_ref(xref))->o;
1593  loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
1594  /* The lj_meta_tset() inconsistency is gone, but better play safe. */
1595  oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv;
1596
1597  if (ix->val == 0) {  /* Indexed load */
1598    IRType t = itype2irt(oldv);
1599    TRef res;
1600    if (oldv == niltvg(J2G(J))) {
1601      emitir(IRTG(IR_EQ, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1602      res = TREF_NIL;
1603    } else {
1604      res = emitir(IRTG(loadop, t), xref, 0);
1605    }
1606    if (tref_ref(res) < rbref) {  /* HREFK + load forwarded? */
1607      lj_ir_rollback(J, rbref);  /* Rollback to eliminate hmask guard. */
1608      J->guardemit = rbguard;
1609    }
1610    if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1611      goto handlemm;
1612    if (irtype_ispri(t)) res = TREF_PRI(t);  /* Canonicalize primitives. */
1613    return res;
1614  } else {  /* Indexed store. */
1615    GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
1616    int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val);
1617    if (tref_ref(xref) < rbref) {  /* HREFK forwarded? */
1618      lj_ir_rollback(J, rbref);  /* Rollback to eliminate hmask guard. */
1619      J->guardemit = rbguard;
1620    }
1621    if (tvisnil(oldv)) {  /* Previous value was nil? */
1622      /* Need to duplicate the hasmm check for the early guards. */
1623      int hasmm = 0;
1624      if (ix->idxchain && mt) {
1625	cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex));
1626	hasmm = mo && !tvisnil(mo);
1627      }
1628      if (hasmm)
1629	emitir(IRTG(loadop, IRT_NIL), xref, 0);  /* Guard for nil value. */
1630      else if (xrefop == IR_HREF)
1631	emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_PGC),
1632	       xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1633      if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) {
1634	lj_assertJ(hasmm, "inconsistent metamethod handling");
1635	goto handlemm;
1636      }
1637      lj_assertJ(!hasmm, "inconsistent metamethod handling");
1638      if (oldv == niltvg(J2G(J))) {  /* Need to insert a new key. */
1639	TRef key = ix->key;
1640	if (tref_isinteger(key)) {  /* NEWREF needs a TValue as a key. */
1641	  key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1642	} else if (tref_isnum(key)) {
1643	  if (tref_isk(key)) {
1644	    if (tvismzero(&ix->keyv))
1645	      key = lj_ir_knum_zero(J);  /* Canonicalize -0.0 to +0.0. */
1646	  } else {
1647	    emitir(IRTG(IR_EQ, IRT_NUM), key, key);  /* Check for !NaN. */
1648	  }
1649	}
1650	xref = emitir(IRT(IR_NEWREF, IRT_PGC), ix->tab, key);
1651	keybarrier = 0;  /* NEWREF already takes care of the key barrier. */
1652#ifdef LUAJIT_ENABLE_TABLE_BUMP
1653	if ((J->flags & JIT_F_OPT_SINK))  /* Avoid a separate flag. */
1654	  rec_idx_bump(J, ix);
1655#endif
1656      }
1657    } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
1658      /* Cannot derive that the previous value was non-nil, must do checks. */
1659      if (xrefop == IR_HREF)  /* Guard against store to niltv. */
1660	emitir(IRTG(IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1661      if (ix->idxchain) {  /* Metamethod lookup required? */
1662	/* A check for NULL metatable is cheaper (hoistable) than a load. */
1663	if (!mt) {
1664	  TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
1665	  emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
1666	} else {
1667	  IRType t = itype2irt(oldv);
1668	  emitir(IRTG(loadop, t), xref, 0);  /* Guard for non-nil value. */
1669	}
1670      }
1671    } else {
1672      keybarrier = 0;  /* Previous non-nil value kept the key alive. */
1673    }
1674    /* Convert int to number before storing. */
1675    if (!LJ_DUALNUM && tref_isinteger(ix->val))
1676      ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT);
1677    emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val);
1678    if (keybarrier || tref_isgcv(ix->val))
1679      emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
1680    /* Invalidate neg. metamethod cache for stores with certain string keys. */
1681    if (!nommstr(J, ix->key)) {
1682      TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ix->tab, IRFL_TAB_NOMM);
1683      emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
1684    }
1685    J->needsnap = 1;
1686    return 0;
1687  }
1688}
1689
1690/* Determine result type of table traversal. */
1691static IRType rec_next_types(GCtab *t, uint32_t idx)
1692{
1693  for (; idx < t->asize; idx++) {
1694    cTValue *a = arrayslot(t, idx);
1695    if (LJ_LIKELY(!tvisnil(a)))
1696      return (LJ_DUALNUM ? IRT_INT : IRT_NUM) + (itype2irt(a) << 8);
1697  }
1698  idx -= t->asize;
1699  for (; idx <= t->hmask; idx++) {
1700    Node *n = &noderef(t->node)[idx];
1701    if (!tvisnil(&n->val))
1702      return itype2irt(&n->key) + (itype2irt(&n->val) << 8);
1703  }
1704  return IRT_NIL + (IRT_NIL << 8);
1705}
1706
1707/* Record a table traversal step aka next(). */
1708int lj_record_next(jit_State *J, RecordIndex *ix)
1709{
1710  IRType t, tkey, tval;
1711  TRef trvk;
1712  t = rec_next_types(tabV(&ix->tabv), ix->keyv.u32.lo);
1713  tkey = (t & 0xff); tval = (t >> 8);
1714  trvk = lj_ir_call(J, IRCALL_lj_vm_next, ix->tab, ix->key);
1715  if (ix->mobj || tkey == IRT_NIL) {
1716    TRef idx = emitir(IRTI(IR_HIOP), trvk, trvk);
1717    /* Always check for invalid key from next() for nil result. */
1718    if (!ix->mobj) emitir(IRTGI(IR_NE), idx, lj_ir_kint(J, -1));
1719    ix->mobj = idx;
1720  }
1721  ix->key = lj_record_vload(J, trvk, 1, tkey);
1722  if (tkey == IRT_NIL || ix->idxchain) {  /* Omit value type check. */
1723    ix->val = TREF_NIL;
1724    return 1;
1725  } else {  /* Need value. */
1726    ix->val = lj_record_vload(J, trvk, 0, tval);
1727    return 2;
1728  }
1729}
1730
1731static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
1732{
1733  RecordIndex ix;
1734  cTValue *basev = J->L->base;
1735  GCtab *t = tabV(&basev[ra-1]);
1736  settabV(J->L, &ix.tabv, t);
1737  ix.tab = getslot(J, ra-1);
1738  ix.idxchain = 0;
1739#ifdef LUAJIT_ENABLE_TABLE_BUMP
1740  if ((J->flags & JIT_F_OPT_SINK)) {
1741    if (t->asize < i+rn-ra)
1742      lj_tab_reasize(J->L, t, i+rn-ra);
1743    setnilV(&ix.keyv);
1744    rec_idx_bump(J, &ix);
1745  }
1746#endif
1747  for (; ra < rn; i++, ra++) {
1748    setintV(&ix.keyv, i);
1749    ix.key = lj_ir_kint(J, i);
1750    copyTV(J->L, &ix.valv, &basev[ra]);
1751    ix.val = getslot(J, ra);
1752    lj_record_idx(J, &ix);
1753  }
1754}
1755
1756/* -- Upvalue access ------------------------------------------------------ */
1757
1758/* Check whether upvalue is immutable and ok to constify. */
1759static int rec_upvalue_constify(jit_State *J, GCupval *uvp)
1760{
1761  if (uvp->immutable) {
1762    cTValue *o = uvval(uvp);
1763    /* Don't constify objects that may retain large amounts of memory. */
1764#if LJ_HASFFI
1765    if (tviscdata(o)) {
1766      GCcdata *cd = cdataV(o);
1767      if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) {
1768	CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid);
1769	if (!ctype_hassize(ct->info) || ct->size <= 16)
1770	  return 1;
1771      }
1772      return 0;
1773    }
1774#else
1775    UNUSED(J);
1776#endif
1777    if (!(tvistab(o) || tvisudata(o) || tvisthread(o)))
1778      return 1;
1779  }
1780  return 0;
1781}
1782
1783/* Record upvalue load/store. */
1784static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
1785{
1786  GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv;
1787  TRef fn = getcurrf(J);
1788  IRRef uref;
1789  int needbarrier = 0;
1790  if (rec_upvalue_constify(J, uvp)) {  /* Try to constify immutable upvalue. */
1791    TRef tr, kfunc;
1792    lj_assertJ(val == 0, "bad usage");
1793    if (!tref_isk(fn)) {  /* Late specialization of current function. */
1794      if (J->pt->flags >= PROTO_CLC_POLY)
1795	goto noconstify;
1796      kfunc = lj_ir_kfunc(J, J->fn);
1797      emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
1798#if LJ_FR2
1799      J->base[-2] = kfunc;
1800#else
1801      J->base[-1] = kfunc | TREF_FRAME;
1802#endif
1803      fn = kfunc;
1804    }
1805    tr = lj_record_constify(J, uvval(uvp));
1806    if (tr)
1807      return tr;
1808  }
1809noconstify:
1810  /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1811  uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
1812  if (!uvp->closed) {
1813    /* In current stack? */
1814    if (uvval(uvp) >= tvref(J->L->stack) &&
1815	uvval(uvp) < tvref(J->L->maxstack)) {
1816      int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot));
1817      if (slot >= 0) {  /* Aliases an SSA slot? */
1818	uref = tref_ref(emitir(IRT(IR_UREFO, IRT_PGC), fn, uv));
1819	emitir(IRTG(IR_EQ, IRT_PGC),
1820	       REF_BASE,
1821	       emitir(IRT(IR_ADD, IRT_PGC), uref,
1822		      lj_ir_kintpgc(J, (slot - 1 - LJ_FR2) * -8)));
1823	slot -= (int32_t)J->baseslot;  /* Note: slot number may be negative! */
1824	if (val == 0) {
1825	  return getslot(J, slot);
1826	} else {
1827	  J->base[slot] = val;
1828	  if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1);
1829	  return 0;
1830	}
1831      }
1832    }
1833    /* IR_UREFO+IRT_IGC is not checked for open-ness at runtime.
1834    ** Always marked as a guard, since it might get promoted to IRT_PGC later.
1835    */
1836    uref = emitir(IRTG(IR_UREFO, tref_isgcv(val) ? IRT_PGC : IRT_IGC), fn, uv);
1837    uref = tref_ref(uref);
1838    emitir(IRTG(IR_UGT, IRT_PGC),
1839	   emitir(IRT(IR_SUB, IRT_PGC), uref, REF_BASE),
1840	   lj_ir_kintpgc(J, (J->baseslot + J->maxslot) * 8));
1841  } else {
1842    /* If fn is constant, then so is the GCupval*, and the upvalue cannot
1843    ** transition back to open, so no guard is required in this case.
1844    */
1845    IRType t = (tref_isk(fn) ? 0 : IRT_GUARD) | IRT_PGC;
1846    uref = tref_ref(emitir(IRT(IR_UREFC, t), fn, uv));
1847    needbarrier = 1;
1848  }
1849  if (val == 0) {  /* Upvalue load */
1850    IRType t = itype2irt(uvval(uvp));
1851    TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0);
1852    if (irtype_ispri(t)) res = TREF_PRI(t);  /* Canonicalize primitive refs. */
1853    return res;
1854  } else {  /* Upvalue store. */
1855    /* Convert int to number before storing. */
1856    if (!LJ_DUALNUM && tref_isinteger(val))
1857      val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
1858    emitir(IRT(IR_USTORE, tref_type(val)), uref, val);
1859    if (needbarrier && tref_isgcv(val))
1860      emitir(IRT(IR_OBAR, IRT_NIL), uref, val);
1861    J->needsnap = 1;
1862    return 0;
1863  }
1864}
1865
1866/* -- Record calls to Lua functions --------------------------------------- */
1867
1868/* Check unroll limits for calls. */
1869static void check_call_unroll(jit_State *J, TraceNo lnk)
1870{
1871  cTValue *frame = J->L->base - 1;
1872  void *pc = mref(frame_func(frame)->l.pc, void);
1873  int32_t depth = J->framedepth;
1874  int32_t count = 0;
1875  if ((J->pt->flags & PROTO_VARARG)) depth--;  /* Vararg frame still missing. */
1876  for (; depth > 0; depth--) {  /* Count frames with same prototype. */
1877    if (frame_iscont(frame)) depth--;
1878    frame = frame_prev(frame);
1879    if (mref(frame_func(frame)->l.pc, void) == pc)
1880      count++;
1881  }
1882  if (J->pc == J->startpc) {
1883    if (count + J->tailcalled > J->param[JIT_P_recunroll]) {
1884      J->pc++;
1885      if (J->framedepth + J->retdepth == 0)
1886	lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno);  /* Tail-rec. */
1887      else
1888	lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno);  /* Up-recursion. */
1889    }
1890  } else {
1891    if (count > J->param[JIT_P_callunroll]) {
1892      if (lnk) {  /* Possible tail- or up-recursion. */
1893	lj_trace_flush(J, lnk);  /* Flush trace that only returns. */
1894	/* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
1895	hotcount_set(J2GG(J), J->pc+1, lj_prng_u64(&J2G(J)->prng) & 15u);
1896      }
1897      lj_trace_err(J, LJ_TRERR_CUNROLL);
1898    }
1899  }
1900}
1901
1902/* Record Lua function setup. */
1903static void rec_func_setup(jit_State *J)
1904{
1905  GCproto *pt = J->pt;
1906  BCReg s, numparams = pt->numparams;
1907  if ((pt->flags & PROTO_NOJIT))
1908    lj_trace_err(J, LJ_TRERR_CJITOFF);
1909  if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
1910    lj_trace_err(J, LJ_TRERR_STACKOV);
1911  /* Fill up missing parameters with nil. */
1912  for (s = J->maxslot; s < numparams; s++)
1913    J->base[s] = TREF_NIL;
1914  /* The remaining slots should never be read before they are written. */
1915  J->maxslot = numparams;
1916}
1917
1918/* Record Lua vararg function setup. */
1919static void rec_func_vararg(jit_State *J)
1920{
1921  GCproto *pt = J->pt;
1922  BCReg s, fixargs, vframe = J->maxslot+1+LJ_FR2;
1923  lj_assertJ((pt->flags & PROTO_VARARG), "FUNCV in non-vararg function");
1924  if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS)
1925    lj_trace_err(J, LJ_TRERR_STACKOV);
1926  J->base[vframe-1-LJ_FR2] = J->base[-1-LJ_FR2];  /* Copy function up. */
1927#if LJ_FR2
1928  J->base[vframe-1] = TREF_FRAME;
1929#endif
1930  /* Copy fixarg slots up and set their original slots to nil. */
1931  fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot;
1932  for (s = 0; s < fixargs; s++) {
1933    J->base[vframe+s] = J->base[s];
1934    J->base[s] = TREF_NIL;
1935  }
1936  J->maxslot = fixargs;
1937  J->framedepth++;
1938  J->base += vframe;
1939  J->baseslot += vframe;
1940}
1941
1942/* Record entry to a Lua function. */
1943static void rec_func_lua(jit_State *J)
1944{
1945  rec_func_setup(J);
1946  check_call_unroll(J, 0);
1947}
1948
1949/* Record entry to an already compiled function. */
1950static void rec_func_jit(jit_State *J, TraceNo lnk)
1951{
1952  GCtrace *T;
1953  rec_func_setup(J);
1954  T = traceref(J, lnk);
1955  if (T->linktype == LJ_TRLINK_RETURN) {  /* Trace returns to interpreter? */
1956    check_call_unroll(J, lnk);
1957    /* Temporarily unpatch JFUNC* to continue recording across function. */
1958    J->patchins = *J->pc;
1959    J->patchpc = (BCIns *)J->pc;
1960    *J->patchpc = T->startins;
1961    return;
1962  }
1963  J->instunroll = 0;  /* Cannot continue across a compiled function. */
1964  if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
1965    lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno);  /* Extra tail-rec. */
1966  else
1967    lj_record_stop(J, LJ_TRLINK_ROOT, lnk);  /* Link to the function. */
1968}
1969
1970/* -- Vararg handling ----------------------------------------------------- */
1971
1972/* Detect y = select(x, ...) idiom. */
1973static int select_detect(jit_State *J)
1974{
1975  BCIns ins = J->pc[1];
1976  if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) {
1977    cTValue *func = &J->L->base[bc_a(ins)];
1978    if (tvisfunc(func) && funcV(func)->c.ffid == FF_select) {
1979      TRef kfunc = lj_ir_kfunc(J, funcV(func));
1980      emitir(IRTG(IR_EQ, IRT_FUNC), getslot(J, bc_a(ins)), kfunc);
1981      return 1;
1982    }
1983  }
1984  return 0;
1985}
1986
1987/* Record vararg instruction. */
1988static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1989{
1990  int32_t numparams = J->pt->numparams;
1991  ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1 - LJ_FR2;
1992  lj_assertJ(frame_isvarg(J->L->base-1), "VARG in non-vararg frame");
1993  if (LJ_FR2 && dst > J->maxslot)
1994    J->base[dst-1] = 0;  /* Prevent resurrection of unrelated slot. */
1995  if (J->framedepth > 0) {  /* Simple case: varargs defined on-trace. */
1996    ptrdiff_t i;
1997    if (nvararg < 0) nvararg = 0;
1998    if (nresults != 1) {
1999      if (nresults == -1) nresults = nvararg;
2000      J->maxslot = dst + (BCReg)nresults;
2001    } else if (dst >= J->maxslot) {
2002      J->maxslot = dst + 1;
2003    }
2004    if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
2005      lj_trace_err(J, LJ_TRERR_STACKOV);
2006    for (i = 0; i < nresults; i++)
2007      J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL;
2008  } else {  /* Unknown number of varargs passed to trace. */
2009    TRef fr = emitir(IRTI(IR_SLOAD), LJ_FR2, IRSLOAD_READONLY|IRSLOAD_FRAME);
2010    int32_t frofs = 8*(1+LJ_FR2+numparams)+FRAME_VARG;
2011    if (nresults >= 0) {  /* Known fixed number of results. */
2012      ptrdiff_t i;
2013      if (nvararg > 0) {
2014	ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg;
2015	TRef vbase;
2016	if (nvararg >= nresults)
2017	  emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults));
2018	else
2019	  emitir(IRTGI(IR_EQ), fr,
2020		 lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1)));
2021	vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
2022	vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
2023		       lj_ir_kintpgc(J, frofs-8*(1+LJ_FR2)));
2024	for (i = 0; i < nload; i++) {
2025	  IRType t = itype2irt(&J->L->base[i-1-LJ_FR2-nvararg]);
2026	  J->base[dst+i] = lj_record_vload(J, vbase, (MSize)i, t);
2027	}
2028      } else {
2029	emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs));
2030	nvararg = 0;
2031      }
2032      for (i = nvararg; i < nresults; i++)
2033	J->base[dst+i] = TREF_NIL;
2034      if (nresults != 1 || dst >= J->maxslot) {
2035	J->maxslot = dst + (BCReg)nresults;
2036      }
2037    } else if (select_detect(J)) {  /* y = select(x, ...) */
2038      TRef tridx = getslot(J, dst-1);
2039      TRef tr = TREF_NIL;
2040      ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]);
2041      if (idx < 0) goto nyivarg;
2042      if (idx != 0 && !tref_isinteger(tridx)) {
2043	if (tref_isstr(tridx))
2044	  tridx = emitir(IRTG(IR_STRTO, IRT_NUM), tridx, 0);
2045	tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX);
2046      }
2047      if (idx != 0 && tref_isk(tridx)) {
2048	emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT),
2049	       fr, lj_ir_kint(J, frofs+8*(int32_t)idx));
2050	frofs -= 8;  /* Bias for 1-based index. */
2051      } else if (idx <= nvararg) {  /* Compute size. */
2052	TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs));
2053	if (numparams)
2054	  emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0));
2055	tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3));
2056	if (idx != 0) {
2057	  tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1));
2058	  rec_idx_abc(J, tr, tridx, (uint32_t)nvararg);
2059	}
2060      } else {
2061	TRef tmp = lj_ir_kint(J, frofs);
2062	if (idx != 0) {
2063	  TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3));
2064	  tmp = emitir(IRTI(IR_ADD), tmp2, tmp);
2065	} else {
2066	  tr = lj_ir_kint(J, 0);
2067	}
2068	emitir(IRTGI(IR_LT), fr, tmp);
2069      }
2070      if (idx != 0 && idx <= nvararg) {
2071	IRType t;
2072	TRef aref, vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
2073	vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
2074		       lj_ir_kintpgc(J, frofs-(8<<LJ_FR2)));
2075	t = itype2irt(&J->L->base[idx-2-LJ_FR2-nvararg]);
2076	aref = emitir(IRT(IR_AREF, IRT_PGC), vbase, tridx);
2077	tr = lj_record_vload(J, aref, 0, t);
2078      }
2079      J->base[dst-2-LJ_FR2] = tr;
2080      J->maxslot = dst-1-LJ_FR2;
2081      J->bcskip = 2;  /* Skip CALLM + select. */
2082    } else {
2083    nyivarg:
2084      setintV(&J->errinfo, BC_VARG);
2085      lj_trace_err_info(J, LJ_TRERR_NYIBC);
2086    }
2087  }
2088}
2089
2090/* -- Record allocations -------------------------------------------------- */
2091
2092static TRef rec_tnew(jit_State *J, uint32_t ah)
2093{
2094  uint32_t asize = ah & 0x7ff;
2095  uint32_t hbits = ah >> 11;
2096  TRef tr;
2097  if (asize == 0x7ff) asize = 0x801;
2098  tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
2099#ifdef LUAJIT_ENABLE_TABLE_BUMP
2100  J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr);
2101  setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc);
2102  setgcref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2103#endif
2104  return tr;
2105}
2106
2107/* -- Concatenation ------------------------------------------------------- */
2108
2109typedef struct RecCatDataCP {
2110  TValue savetv[5+LJ_FR2];
2111  jit_State *J;
2112  BCReg baseslot, topslot;
2113  TRef tr;
2114} RecCatDataCP;
2115
2116static TValue *rec_mm_concat_cp(lua_State *L, lua_CFunction dummy, void *ud)
2117{
2118  RecCatDataCP *rcd = (RecCatDataCP *)ud;
2119  jit_State *J = rcd->J;
2120  BCReg baseslot = rcd->baseslot, topslot = rcd->topslot;
2121  TRef *top = &J->base[topslot];
2122  BCReg s;
2123  RecordIndex ix;
2124  UNUSED(L); UNUSED(dummy);
2125  lj_assertJ(baseslot < topslot, "bad CAT arg");
2126  for (s = baseslot; s <= topslot; s++)
2127    (void)getslot(J, s);  /* Ensure all arguments have a reference. */
2128  if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
2129    TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
2130    /* First convert numbers to strings. */
2131    for (trp = top; trp >= base; trp--) {
2132      if (tref_isnumber(*trp))
2133	*trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
2134		      tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
2135      else if (!tref_isstr(*trp))
2136	break;
2137    }
2138    xbase = ++trp;
2139    tr = hdr = emitir(IRT(IR_BUFHDR, IRT_PGC),
2140		      lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
2141    do {
2142      tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, *trp++);
2143    } while (trp <= top);
2144    tr = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
2145    J->maxslot = (BCReg)(xbase - J->base);
2146    if (xbase == base) {
2147      rcd->tr = tr;  /* Return simple concatenation result. */
2148      return NULL;
2149    }
2150    /* Pass partial result. */
2151    rcd->topslot = topslot = J->maxslot--;
2152    /* Save updated range of slots. */
2153    memcpy(rcd->savetv, &L->base[topslot-1], sizeof(rcd->savetv));
2154    *xbase = tr;
2155    top = xbase;
2156    setstrV(J->L, &ix.keyv, &J2G(J)->strempty);  /* Simulate string result. */
2157  } else {
2158    J->maxslot = topslot-1;
2159    copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
2160  }
2161  copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
2162  ix.tab = top[-1];
2163  ix.key = top[0];
2164  rec_mm_arith(J, &ix, MM_concat);  /* Call __concat metamethod. */
2165  rcd->tr = 0;  /* No result yet. */
2166  return NULL;
2167}
2168
2169static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
2170{
2171  lua_State *L = J->L;
2172  ptrdiff_t delta = L->top - L->base;
2173  TValue errobj;
2174  RecCatDataCP rcd;
2175  int errcode;
2176  rcd.J = J;
2177  rcd.baseslot = baseslot;
2178  rcd.topslot = topslot;
2179  /* Save slots. */
2180  memcpy(rcd.savetv, &L->base[topslot-1], sizeof(rcd.savetv));
2181  errcode = lj_vm_cpcall(L, NULL, &rcd, rec_mm_concat_cp);
2182  if (errcode) copyTV(L, &errobj, L->top-1);
2183  /* Restore slots. */
2184  memcpy(&L->base[rcd.topslot-1], rcd.savetv, sizeof(rcd.savetv));
2185  if (errcode) {
2186    L->top = L->base + delta;
2187    copyTV(L, L->top++, &errobj);
2188    return (TRef)(-errcode);
2189  }
2190  return rcd.tr;
2191}
2192
2193/* -- Record bytecode ops ------------------------------------------------- */
2194
2195/* Prepare for comparison. */
2196static void rec_comp_prep(jit_State *J)
2197{
2198  /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
2199  if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins)
2200    emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
2201  lj_snap_add(J);
2202}
2203
2204/* Fixup comparison. */
2205static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond)
2206{
2207  BCIns jmpins = pc[1];
2208  const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0);
2209  SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
2210  /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
2211#if LJ_FR2
2212  SnapEntry *flink = &J->cur.snapmap[snap->mapofs + snap->nent];
2213  uint64_t pcbase;
2214  memcpy(&pcbase, flink, sizeof(uint64_t));
2215  pcbase = (pcbase & 0xff) | (u64ptr(npc) << 8);
2216  memcpy(flink, &pcbase, sizeof(uint64_t));
2217#else
2218  J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc);
2219#endif
2220  J->needsnap = 1;
2221  if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins);
2222  lj_snap_shrink(J);  /* Shrink last snapshot if possible. */
2223}
2224
2225/* Record the next bytecode instruction (_before_ it's executed). */
2226void lj_record_ins(jit_State *J)
2227{
2228  cTValue *lbase;
2229  RecordIndex ix;
2230  const BCIns *pc;
2231  BCIns ins;
2232  BCOp op;
2233  TRef ra, rb, rc;
2234
2235  /* Perform post-processing action before recording the next instruction. */
2236  if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) {
2237    switch (J->postproc) {
2238    case LJ_POST_FIXCOMP:  /* Fixup comparison. */
2239      pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64;
2240      rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1)));
2241      /* fallthrough */
2242    case LJ_POST_FIXGUARD:  /* Fixup and emit pending guard. */
2243    case LJ_POST_FIXGUARDSNAP:  /* Fixup and emit pending guard and snapshot. */
2244      if (!tvistruecond(&J2G(J)->tmptv2)) {
2245	J->fold.ins.o ^= 1;  /* Flip guard to opposite. */
2246	if (J->postproc == LJ_POST_FIXGUARDSNAP) {
2247	  SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
2248	  J->cur.snapmap[snap->mapofs+snap->nent-1]--;  /* False -> true. */
2249	}
2250      }
2251      lj_opt_fold(J);  /* Emit pending guard. */
2252      /* fallthrough */
2253    case LJ_POST_FIXBOOL:
2254      if (!tvistruecond(&J2G(J)->tmptv2)) {
2255	BCReg s;
2256	TValue *tv = J->L->base;
2257	for (s = 0; s < J->maxslot; s++)  /* Fixup stack slot (if any). */
2258	  if (J->base[s] == TREF_TRUE && tvisfalse(&tv[s])) {
2259	    J->base[s] = TREF_FALSE;
2260	    break;
2261	  }
2262      }
2263      break;
2264    case LJ_POST_FIXCONST:
2265      {
2266	BCReg s;
2267	TValue *tv = J->L->base;
2268	for (s = 0; s < J->maxslot; s++)  /* Constify stack slots (if any). */
2269	  if (J->base[s] == TREF_NIL && !tvisnil(&tv[s]))
2270	    J->base[s] = lj_record_constify(J, &tv[s]);
2271      }
2272      break;
2273    case LJ_POST_FFRETRY:  /* Suppress recording of retried fast function. */
2274      if (bc_op(*J->pc) >= BC__MAX)
2275	return;
2276      break;
2277    default: lj_assertJ(0, "bad post-processing mode"); break;
2278    }
2279    J->postproc = LJ_POST_NONE;
2280  }
2281
2282  /* Need snapshot before recording next bytecode (e.g. after a store). */
2283  if (J->needsnap) {
2284    J->needsnap = 0;
2285    if (J->pt && bc_op(*J->pc) < BC_FUNCF) lj_snap_purge(J);
2286    lj_snap_add(J);
2287    J->mergesnap = 1;
2288  }
2289
2290  /* Skip some bytecodes. */
2291  if (LJ_UNLIKELY(J->bcskip > 0)) {
2292    J->bcskip--;
2293    return;
2294  }
2295
2296  /* Record only closed loops for root traces. */
2297  pc = J->pc;
2298  if (J->framedepth == 0 &&
2299     (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent)
2300    lj_trace_err(J, LJ_TRERR_LLEAVE);
2301
2302#ifdef LUA_USE_ASSERT
2303  rec_check_slots(J);
2304  rec_check_ir(J);
2305#endif
2306
2307#if LJ_HASPROFILE
2308  rec_profile_ins(J, pc);
2309#endif
2310
2311  /* Keep a copy of the runtime values of var/num/str operands. */
2312#define rav	(&ix.valv)
2313#define rbv	(&ix.tabv)
2314#define rcv	(&ix.keyv)
2315
2316  lbase = J->L->base;
2317  ins = *pc;
2318  op = bc_op(ins);
2319  ra = bc_a(ins);
2320  ix.val = 0;
2321  switch (bcmode_a(op)) {
2322  case BCMvar:
2323    copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break;
2324  default: break;  /* Handled later. */
2325  }
2326  rb = bc_b(ins);
2327  rc = bc_c(ins);
2328  switch (bcmode_b(op)) {
2329  case BCMnone: rb = 0; rc = bc_d(ins); break;  /* Upgrade rc to 'rd'. */
2330  case BCMvar:
2331    copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break;
2332  default: break;  /* Handled later. */
2333  }
2334  switch (bcmode_c(op)) {
2335  case BCMvar:
2336    copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
2337  case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break;
2338  case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc);
2339    copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) :
2340    tv->u32.hi == LJ_KEYINDEX ? (lj_ir_kint(J, 0) | TREF_KEYINDEX) :
2341    lj_ir_knumint(J, numV(tv)); } break;
2342  case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc));
2343    setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
2344  default: break;  /* Handled later. */
2345  }
2346
2347  switch (op) {
2348
2349  /* -- Comparison ops ---------------------------------------------------- */
2350
2351  case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2352#if LJ_HASFFI
2353    if (tref_iscdata(ra) || tref_iscdata(rc)) {
2354      rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt);
2355      break;
2356    }
2357#endif
2358    /* Emit nothing for two numeric or string consts. */
2359    if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
2360      IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra);
2361      IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc);
2362      int irop;
2363      if (ta != tc) {
2364	/* Widen mixed number/int comparisons to number/number comparison. */
2365	if (ta == IRT_INT && tc == IRT_NUM) {
2366	  ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT);
2367	  ta = IRT_NUM;
2368	} else if (ta == IRT_NUM && tc == IRT_INT) {
2369	  rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT);
2370	} else if (LJ_52) {
2371	  ta = IRT_NIL;  /* Force metamethod for different types. */
2372	} else if (!((ta == IRT_FALSE || ta == IRT_TRUE) &&
2373		     (tc == IRT_FALSE || tc == IRT_TRUE))) {
2374	  break;  /* Interpreter will throw for two different types. */
2375	}
2376      }
2377      rec_comp_prep(J);
2378      irop = (int)op - (int)BC_ISLT + (int)IR_LT;
2379      if (ta == IRT_NUM) {
2380	if ((irop & 1)) irop ^= 4;  /* ISGE/ISGT are unordered. */
2381	if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2382	  irop ^= 5;
2383      } else if (ta == IRT_INT) {
2384	if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2385	  irop ^= 1;
2386      } else if (ta == IRT_STR) {
2387	if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1;
2388	ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc);
2389	rc = lj_ir_kint(J, 0);
2390	ta = IRT_INT;
2391      } else {
2392	rec_mm_comp(J, &ix, (int)op);
2393	break;
2394      }
2395      emitir(IRTG(irop, ta), ra, rc);
2396      rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1);
2397    }
2398    break;
2399
2400  case BC_ISEQV: case BC_ISNEV:
2401  case BC_ISEQS: case BC_ISNES:
2402  case BC_ISEQN: case BC_ISNEN:
2403  case BC_ISEQP: case BC_ISNEP:
2404#if LJ_HASFFI
2405    if (tref_iscdata(ra) || tref_iscdata(rc)) {
2406      rec_mm_comp_cdata(J, &ix, op, MM_eq);
2407      break;
2408    }
2409#endif
2410    /* Emit nothing for two non-table, non-udata consts. */
2411    if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
2412      int diff;
2413      rec_comp_prep(J);
2414      diff = lj_record_objcmp(J, ra, rc, rav, rcv);
2415      if (diff == 2 || !(tref_istab(ra) || tref_isudata(ra)))
2416	rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff);
2417      else if (diff == 1)  /* Only check __eq if different, but same type. */
2418	rec_mm_equal(J, &ix, (int)op);
2419    }
2420    break;
2421
2422  /* -- Unary test and copy ops ------------------------------------------- */
2423
2424  case BC_ISTC: case BC_ISFC:
2425    if ((op & 1) == tref_istruecond(rc))
2426      rc = 0;  /* Don't store if condition is not true. */
2427    /* fallthrough */
2428  case BC_IST: case BC_ISF:  /* Type specialization suffices. */
2429    if (bc_a(pc[1]) < J->maxslot)
2430      J->maxslot = bc_a(pc[1]);  /* Shrink used slots. */
2431    break;
2432
2433  case BC_ISTYPE: case BC_ISNUM:
2434    /* These coercions need to correspond with lj_meta_istype(). */
2435    if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
2436      ra = lj_opt_narrow_toint(J, ra);
2437    else if (rc == ~LJ_TNUMX+2)
2438      ra = lj_ir_tonum(J, ra);
2439    else if (rc == ~LJ_TSTR+1)
2440      ra = lj_ir_tostr(J, ra);
2441    /* else: type specialization suffices. */
2442    J->base[bc_a(ins)] = ra;
2443    break;
2444
2445  /* -- Unary ops --------------------------------------------------------- */
2446
2447  case BC_NOT:
2448    /* Type specialization already forces const result. */
2449    rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE;
2450    break;
2451
2452  case BC_LEN:
2453    if (tref_isstr(rc))
2454      rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
2455    else if (!LJ_52 && tref_istab(rc))
2456      rc = emitir(IRTI(IR_ALEN), rc, TREF_NIL);
2457    else
2458      rc = rec_mm_len(J, rc, rcv);
2459    break;
2460
2461  /* -- Arithmetic ops ---------------------------------------------------- */
2462
2463  case BC_UNM:
2464    if (tref_isnumber_str(rc)) {
2465      rc = lj_opt_narrow_unm(J, rc, rcv);
2466    } else {
2467      ix.tab = rc;
2468      copyTV(J->L, &ix.tabv, rcv);
2469      rc = rec_mm_arith(J, &ix, MM_unm);
2470    }
2471    break;
2472
2473  case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV:
2474    /* Swap rb/rc and rbv/rcv. rav is temp. */
2475    ix.tab = rc; ix.key = rc = rb; rb = ix.tab;
2476    copyTV(J->L, rav, rbv);
2477    copyTV(J->L, rbv, rcv);
2478    copyTV(J->L, rcv, rav);
2479    if (op == BC_MODNV)
2480      goto recmod;
2481    /* fallthrough */
2482  case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN:
2483  case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: {
2484    MMS mm = bcmode_mm(op);
2485    if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2486      rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv,
2487			       (int)mm - (int)MM_add + (int)IR_ADD);
2488    else
2489      rc = rec_mm_arith(J, &ix, mm);
2490    break;
2491    }
2492
2493  case BC_MODVN: case BC_MODVV:
2494  recmod:
2495    if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2496      rc = lj_opt_narrow_mod(J, rb, rc, rbv, rcv);
2497    else
2498      rc = rec_mm_arith(J, &ix, MM_mod);
2499    break;
2500
2501  case BC_POW:
2502    if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2503      rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv, IR_POW);
2504    else
2505      rc = rec_mm_arith(J, &ix, MM_pow);
2506    break;
2507
2508  /* -- Miscellaneous ops ------------------------------------------------- */
2509
2510  case BC_CAT:
2511    rc = rec_cat(J, rb, rc);
2512    if (rc >= 0xffffff00)
2513      lj_err_throw(J->L, -(int32_t)rc);  /* Propagate errors. */
2514    break;
2515
2516  /* -- Constant and move ops --------------------------------------------- */
2517
2518  case BC_MOV:
2519    /* Clear gap of method call to avoid resurrecting previous refs. */
2520    if (ra > J->maxslot) {
2521#if LJ_FR2
2522      memset(J->base + J->maxslot, 0, (ra - J->maxslot) * sizeof(TRef));
2523#else
2524      J->base[ra-1] = 0;
2525#endif
2526    }
2527    break;
2528  case BC_KSTR: case BC_KNUM: case BC_KPRI:
2529    break;
2530  case BC_KSHORT:
2531    rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
2532    break;
2533  case BC_KNIL:
2534    if (LJ_FR2 && ra > J->maxslot)
2535      J->base[ra-1] = 0;
2536    while (ra <= rc)
2537      J->base[ra++] = TREF_NIL;
2538    if (rc >= J->maxslot) J->maxslot = rc+1;
2539    break;
2540#if LJ_HASFFI
2541  case BC_KCDATA:
2542    rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA);
2543    break;
2544#endif
2545
2546  /* -- Upvalue and function ops ------------------------------------------ */
2547
2548  case BC_UGET:
2549    rc = rec_upvalue(J, rc, 0);
2550    break;
2551  case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP:
2552    rec_upvalue(J, ra, rc);
2553    break;
2554
2555  /* -- Table ops --------------------------------------------------------- */
2556
2557  case BC_GGET: case BC_GSET:
2558    settabV(J->L, &ix.tabv, tabref(J->fn->l.env));
2559    ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV);
2560    ix.idxchain = LJ_MAX_IDXCHAIN;
2561    rc = lj_record_idx(J, &ix);
2562    break;
2563
2564  case BC_TGETB: case BC_TSETB:
2565    setintV(&ix.keyv, (int32_t)rc);
2566    ix.key = lj_ir_kint(J, (int32_t)rc);
2567    /* fallthrough */
2568  case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS:
2569    ix.idxchain = LJ_MAX_IDXCHAIN;
2570    rc = lj_record_idx(J, &ix);
2571    break;
2572  case BC_TGETR: case BC_TSETR:
2573    ix.idxchain = 0;
2574    rc = lj_record_idx(J, &ix);
2575    break;
2576
2577  case BC_TSETM:
2578    rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo);
2579    J->maxslot = ra;  /* The table slot at ra-1 is the highest used slot. */
2580    break;
2581
2582  case BC_TNEW:
2583    rc = rec_tnew(J, rc);
2584    break;
2585  case BC_TDUP:
2586    rc = emitir(IRTG(IR_TDUP, IRT_TAB),
2587		lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
2588#ifdef LUAJIT_ENABLE_TABLE_BUMP
2589    J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc);
2590    setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc);
2591    setgcref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2592#endif
2593    break;
2594
2595  /* -- Calls and vararg handling ----------------------------------------- */
2596
2597  case BC_ITERC:
2598    J->base[ra] = getslot(J, ra-3);
2599    J->base[ra+1+LJ_FR2] = getslot(J, ra-2);
2600    J->base[ra+2+LJ_FR2] = getslot(J, ra-1);
2601    { /* Do the actual copy now because lj_record_call needs the values. */
2602      TValue *b = &J->L->base[ra];
2603      copyTV(J->L, b, b-3);
2604      copyTV(J->L, b+1+LJ_FR2, b-2);
2605      copyTV(J->L, b+2+LJ_FR2, b-1);
2606    }
2607    lj_record_call(J, ra, (ptrdiff_t)rc-1);
2608    break;
2609
2610  /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
2611  case BC_CALLM:
2612    rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2613    /* fallthrough */
2614  case BC_CALL:
2615    lj_record_call(J, ra, (ptrdiff_t)rc-1);
2616    break;
2617
2618  case BC_CALLMT:
2619    rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2620    /* fallthrough */
2621  case BC_CALLT:
2622    lj_record_tailcall(J, ra, (ptrdiff_t)rc-1);
2623    break;
2624
2625  case BC_VARG:
2626    rec_varg(J, ra, (ptrdiff_t)rb-1);
2627    break;
2628
2629  /* -- Returns ----------------------------------------------------------- */
2630
2631  case BC_RETM:
2632    /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
2633    rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
2634    /* fallthrough */
2635  case BC_RET: case BC_RET0: case BC_RET1:
2636#if LJ_HASPROFILE
2637    rec_profile_ret(J);
2638#endif
2639    lj_record_ret(J, ra, (ptrdiff_t)rc-1);
2640    break;
2641
2642  /* -- Loops and branches ------------------------------------------------ */
2643
2644  case BC_FORI:
2645    if (rec_for(J, pc, 0) != LOOPEV_LEAVE)
2646      J->loopref = J->cur.nins;
2647    break;
2648  case BC_JFORI:
2649    lj_assertJ(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL,
2650	       "JFORI does not point to JFORL");
2651    if (rec_for(J, pc, 0) != LOOPEV_LEAVE)  /* Link to existing loop. */
2652      lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
2653    /* Continue tracing if the loop is not entered. */
2654    break;
2655
2656  case BC_FORL:
2657    rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1));
2658    break;
2659  case BC_ITERL:
2660    rec_loop_interp(J, pc, rec_iterl(J, *pc));
2661    break;
2662  case BC_ITERN:
2663    rec_loop_interp(J, pc, rec_itern(J, ra, rb));
2664    break;
2665  case BC_LOOP:
2666    rec_loop_interp(J, pc, rec_loop(J, ra, 1));
2667    break;
2668
2669  case BC_JFORL:
2670    rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1));
2671    break;
2672  case BC_JITERL:
2673    rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins));
2674    break;
2675  case BC_JLOOP:
2676    rec_loop_jit(J, rc, rec_loop(J, ra,
2677				 !bc_isret(bc_op(traceref(J, rc)->startins)) &&
2678				 bc_op(traceref(J, rc)->startins) != BC_ITERN));
2679    break;
2680
2681  case BC_IFORL:
2682  case BC_IITERL:
2683  case BC_ILOOP:
2684  case BC_IFUNCF:
2685  case BC_IFUNCV:
2686    lj_trace_err(J, LJ_TRERR_BLACKL);
2687    break;
2688
2689  case BC_JMP:
2690    if (ra < J->maxslot)
2691      J->maxslot = ra;  /* Shrink used slots. */
2692    break;
2693
2694  case BC_ISNEXT:
2695    rec_isnext(J, ra);
2696    break;
2697
2698  /* -- Function headers -------------------------------------------------- */
2699
2700  case BC_FUNCF:
2701    rec_func_lua(J);
2702    break;
2703  case BC_JFUNCF:
2704    rec_func_jit(J, rc);
2705    break;
2706
2707  case BC_FUNCV:
2708    rec_func_vararg(J);
2709    rec_func_lua(J);
2710    break;
2711  case BC_JFUNCV:
2712    /* Cannot happen. No hotcall counting for varag funcs. */
2713    lj_assertJ(0, "unsupported vararg hotcall");
2714    break;
2715
2716  case BC_FUNCC:
2717  case BC_FUNCCW:
2718    lj_ffrecord_func(J);
2719    break;
2720
2721  default:
2722    if (op >= BC__MAX) {
2723      lj_ffrecord_func(J);
2724      break;
2725    }
2726    /* fallthrough */
2727  case BC_UCLO:
2728  case BC_FNEW:
2729    setintV(&J->errinfo, (int32_t)op);
2730    lj_trace_err_info(J, LJ_TRERR_NYIBC);
2731    break;
2732  }
2733
2734  /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
2735  if (bcmode_a(op) == BCMdst && rc) {
2736    J->base[ra] = rc;
2737    if (ra >= J->maxslot) {
2738#if LJ_FR2
2739      if (ra > J->maxslot) J->base[ra-1] = 0;
2740#endif
2741      J->maxslot = ra+1;
2742    }
2743  }
2744
2745#undef rav
2746#undef rbv
2747#undef rcv
2748
2749  /* Limit the number of recorded IR instructions and constants. */
2750  if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord] ||
2751      J->cur.nk < REF_BIAS-(IRRef)J->param[JIT_P_maxirconst])
2752    lj_trace_err(J, LJ_TRERR_TRACEOV);
2753}
2754
2755/* -- Recording setup ----------------------------------------------------- */
2756
2757/* Setup recording for a root trace started by a hot loop. */
2758static const BCIns *rec_setup_root(jit_State *J)
2759{
2760  /* Determine the next PC and the bytecode range for the loop. */
2761  const BCIns *pcj, *pc = J->pc;
2762  BCIns ins = *pc;
2763  BCReg ra = bc_a(ins);
2764  switch (bc_op(ins)) {
2765  case BC_FORL:
2766    J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2767    pc += 1+bc_j(ins);
2768    J->bc_min = pc;
2769    break;
2770  case BC_ITERL:
2771    if (bc_op(pc[-1]) == BC_JLOOP)
2772      lj_trace_err(J, LJ_TRERR_LINNER);
2773    lj_assertJ(bc_op(pc[-1]) == BC_ITERC, "no ITERC before ITERL");
2774    J->maxslot = ra + bc_b(pc[-1]) - 1;
2775    J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2776    pc += 1+bc_j(ins);
2777    lj_assertJ(bc_op(pc[-1]) == BC_JMP, "ITERL does not point to JMP+1");
2778    J->bc_min = pc;
2779    break;
2780  case BC_ITERN:
2781    lj_assertJ(bc_op(pc[1]) == BC_ITERL, "no ITERL after ITERN");
2782    J->maxslot = ra;
2783    J->bc_extent = (MSize)(-bc_j(pc[1]))*sizeof(BCIns);
2784    J->bc_min = pc+2 + bc_j(pc[1]);
2785    J->state = LJ_TRACE_RECORD_1ST;  /* Record the first ITERN, too. */
2786    break;
2787  case BC_LOOP:
2788    /* Only check BC range for real loops, but not for "repeat until true". */
2789    pcj = pc + bc_j(ins);
2790    ins = *pcj;
2791    if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) {
2792      J->bc_min = pcj+1 + bc_j(ins);
2793      J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2794    }
2795    J->maxslot = ra;
2796    pc++;
2797    break;
2798  case BC_RET:
2799  case BC_RET0:
2800  case BC_RET1:
2801    /* No bytecode range check for down-recursive root traces. */
2802    J->maxslot = ra + bc_d(ins) - 1;
2803    break;
2804  case BC_FUNCF:
2805    /* No bytecode range check for root traces started by a hot call. */
2806    J->maxslot = J->pt->numparams;
2807    pc++;
2808    break;
2809  case BC_CALLM:
2810  case BC_CALL:
2811  case BC_ITERC:
2812    /* No bytecode range check for stitched traces. */
2813    pc++;
2814    break;
2815  default:
2816    lj_assertJ(0, "bad root trace start bytecode %d", bc_op(ins));
2817    break;
2818  }
2819  return pc;
2820}
2821
2822/* Setup for recording a new trace. */
2823void lj_record_setup(jit_State *J)
2824{
2825  uint32_t i;
2826
2827  /* Initialize state related to current trace. */
2828  memset(J->slot, 0, sizeof(J->slot));
2829  memset(J->chain, 0, sizeof(J->chain));
2830#ifdef LUAJIT_ENABLE_TABLE_BUMP
2831  memset(J->rbchash, 0, sizeof(J->rbchash));
2832#endif
2833  memset(J->bpropcache, 0, sizeof(J->bpropcache));
2834  J->scev.idx = REF_NIL;
2835  setmref(J->scev.pc, NULL);
2836
2837  J->baseslot = 1+LJ_FR2;  /* Invoking function is at base[-1-LJ_FR2]. */
2838  J->base = J->slot + J->baseslot;
2839  J->maxslot = 0;
2840  J->framedepth = 0;
2841  J->retdepth = 0;
2842
2843  J->instunroll = J->param[JIT_P_instunroll];
2844  J->loopunroll = J->param[JIT_P_loopunroll];
2845  J->tailcalled = 0;
2846  J->loopref = 0;
2847
2848  J->bc_min = NULL;  /* Means no limit. */
2849  J->bc_extent = ~(MSize)0;
2850
2851  /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2852  emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno);
2853  for (i = 0; i <= 2; i++) {
2854    IRIns *ir = IR(REF_NIL-i);
2855    ir->i = 0;
2856    ir->t.irt = (uint8_t)(IRT_NIL+i);
2857    ir->o = IR_KPRI;
2858    ir->prev = 0;
2859  }
2860  J->cur.nk = REF_TRUE;
2861
2862  J->startpc = J->pc;
2863  setmref(J->cur.startpc, J->pc);
2864  if (J->parent) {  /* Side trace. */
2865    GCtrace *T = traceref(J, J->parent);
2866    TraceNo root = T->root ? T->root : J->parent;
2867    J->cur.root = (uint16_t)root;
2868    J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
2869    /* Check whether we could at least potentially form an extra loop. */
2870    if (J->exitno == 0 && T->snap[0].nent == 0) {
2871      /* We can narrow a FORL for some side traces, too. */
2872      if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI &&
2873	  bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) {
2874	lj_snap_add(J);
2875	rec_for_loop(J, J->pc-1, &J->scev, 1);
2876	goto sidecheck;
2877      }
2878    } else {
2879      J->startpc = NULL;  /* Prevent forming an extra loop. */
2880    }
2881    lj_snap_replay(J, T);
2882  sidecheck:
2883    if ((traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
2884	 T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
2885				     J->param[JIT_P_tryside])) {
2886      if (bc_op(*J->pc) == BC_JLOOP) {
2887	BCIns startins = traceref(J, bc_d(*J->pc))->startins;
2888	if (bc_op(startins) == BC_ITERN)
2889	  rec_itern(J, bc_a(startins), bc_b(startins));
2890      }
2891      lj_record_stop(J, LJ_TRLINK_INTERP, 0);
2892    }
2893  } else {  /* Root trace. */
2894    J->cur.root = 0;
2895    J->cur.startins = *J->pc;
2896    J->pc = rec_setup_root(J);
2897    /* Note: the loop instruction itself is recorded at the end and not
2898    ** at the start! So snapshot #0 needs to point to the *next* instruction.
2899    ** The one exception is BC_ITERN, which sets LJ_TRACE_RECORD_1ST.
2900    */
2901    lj_snap_add(J);
2902    if (bc_op(J->cur.startins) == BC_FORL)
2903      rec_for_loop(J, J->pc-1, &J->scev, 1);
2904    else if (bc_op(J->cur.startins) == BC_ITERC)
2905      J->startpc = NULL;
2906    if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
2907      lj_trace_err(J, LJ_TRERR_STACKOV);
2908  }
2909#if LJ_HASPROFILE
2910  J->prev_pt = NULL;
2911  J->prev_line = -1;
2912#endif
2913#ifdef LUAJIT_ENABLE_CHECKHOOK
2914  /* Regularly check for instruction/line hooks from compiled code and
2915  ** exit to the interpreter if the hooks are set.
2916  **
2917  ** This is a compile-time option and disabled by default, since the
2918  ** hook checks may be quite expensive in tight loops.
2919  **
2920  ** Note this is only useful if hooks are *not* set most of the time.
2921  ** Use this only if you want to *asynchronously* interrupt the execution.
2922  **
2923  ** You can set the instruction hook via lua_sethook() with a count of 1
2924  ** from a signal handler or another native thread. Please have a look
2925  ** at the first few functions in luajit.c for an example (Ctrl-C handler).
2926  */
2927  {
2928    TRef tr = emitir(IRT(IR_XLOAD, IRT_U8),
2929		     lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE);
2930    tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT)));
2931    emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
2932  }
2933#endif
2934}
2935
2936#undef IR
2937#undef emitir_raw
2938#undef emitir
2939
2940#endif