luna - luajit/src/lj

   1/*
   2** Snapshot handling.
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4*/
   5
   6#define lj_snap_c
   7#define LUA_CORE
   8
   9#include "lj_obj.h"
  10
  11#if LJ_HASJIT
  12
  13#include "lj_gc.h"
  14#include "lj_tab.h"
  15#include "lj_state.h"
  16#include "lj_frame.h"
  17#include "lj_bc.h"
  18#include "lj_ir.h"
  19#include "lj_jit.h"
  20#include "lj_iropt.h"
  21#include "lj_trace.h"
  22#include "lj_snap.h"
  23#include "lj_target.h"
  24#if LJ_HASFFI
  25#include "lj_ctype.h"
  26#include "lj_cdata.h"
  27#endif
  28
  29/* Pass IR on to next optimization in chain (FOLD). */
  30#define emitir(ot, a, b)	(lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  31
  32/* Emit raw IR without passing through optimizations. */
  33#define emitir_raw(ot, a, b)	(lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
  34
  35/* -- Snapshot buffer allocation ------------------------------------------ */
  36
  37/* Grow snapshot buffer. */
  38void lj_snap_grow_buf_(jit_State *J, MSize need)
  39{
  40  MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
  41  if (need > maxsnap)
  42    lj_trace_err(J, LJ_TRERR_SNAPOV);
  43  lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
  44  J->cur.snap = J->snapbuf;
  45}
  46
  47/* Grow snapshot map buffer. */
  48void lj_snap_grow_map_(jit_State *J, MSize need)
  49{
  50  if (need < 2*J->sizesnapmap)
  51    need = 2*J->sizesnapmap;
  52  else if (need < 64)
  53    need = 64;
  54  J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
  55		    J->sizesnapmap*sizeof(SnapEntry), need*sizeof(SnapEntry));
  56  J->cur.snapmap = J->snapmapbuf;
  57  J->sizesnapmap = need;
  58}
  59
  60/* -- Snapshot generation ------------------------------------------------- */
  61
  62/* Add all modified slots to the snapshot. */
  63static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
  64{
  65  IRRef retf = J->chain[IR_RETF];  /* Limits SLOAD restore elimination. */
  66  BCReg s;
  67  MSize n = 0;
  68  for (s = 0; s < nslots; s++) {
  69    TRef tr = J->slot[s];
  70    IRRef ref = tref_ref(tr);
  71#if LJ_FR2
  72    if (s == 1) {  /* Ignore slot 1 in LJ_FR2 mode, except if tailcalled. */
  73      if ((tr & TREF_FRAME))
  74	map[n++] = SNAP(1, SNAP_FRAME | SNAP_NORESTORE, REF_NIL);
  75      continue;
  76    }
  77    if ((tr & (TREF_FRAME | TREF_CONT)) && !ref) {
  78      cTValue *base = J->L->base - J->baseslot;
  79      tr = J->slot[s] = (tr & 0xff0000) | lj_ir_k64(J, IR_KNUM, base[s].u64);
  80      ref = tref_ref(tr);
  81    }
  82#endif
  83    if (ref) {
  84      SnapEntry sn = SNAP_TR(s, tr);
  85      IRIns *ir = &J->cur.ir[ref];
  86      if ((LJ_FR2 || !(sn & (SNAP_CONT|SNAP_FRAME))) &&
  87	  ir->o == IR_SLOAD && ir->op1 == s && ref > retf) {
  88	/*
  89	** No need to snapshot unmodified non-inherited slots.
  90	** But always snapshot the function below a frame in LJ_FR2 mode.
  91	*/
  92	if (!(ir->op2 & IRSLOAD_INHERIT) &&
  93	    (!LJ_FR2 || s == 0 || s+1 == nslots ||
  94	     !(J->slot[s+1] & (TREF_CONT|TREF_FRAME))))
  95	  continue;
  96	/* No need to restore readonly slots and unmodified non-parent slots. */
  97	if (!(LJ_DUALNUM && (ir->op2 & IRSLOAD_CONVERT)) &&
  98	    (ir->op2 & (IRSLOAD_READONLY|IRSLOAD_PARENT)) != IRSLOAD_PARENT)
  99	  sn |= SNAP_NORESTORE;
 100      }
 101      if (LJ_SOFTFP32 && irt_isnum(ir->t))
 102	sn |= SNAP_SOFTFPNUM;
 103      map[n++] = sn;
 104    }
 105  }
 106  return n;
 107}
 108
 109/* Add frame links at the end of the snapshot. */
 110static MSize snapshot_framelinks(jit_State *J, SnapEntry *map, uint8_t *topslot)
 111{
 112  cTValue *frame = J->L->base - 1;
 113  cTValue *lim = J->L->base - J->baseslot + LJ_FR2;
 114  GCfunc *fn = frame_func(frame);
 115  cTValue *ftop = isluafunc(fn) ? (frame+funcproto(fn)->framesize) : J->L->top;
 116#if LJ_FR2
 117  uint64_t pcbase = (u64ptr(J->pc) << 8) | (J->baseslot - 2);
 118  lj_assertJ(2 <= J->baseslot && J->baseslot <= 257, "bad baseslot");
 119  memcpy(map, &pcbase, sizeof(uint64_t));
 120#else
 121  MSize f = 0;
 122  map[f++] = SNAP_MKPC(J->pc);  /* The current PC is always the first entry. */
 123#endif
 124  lj_assertJ(!J->pt ||
 125	     (J->pc >= proto_bc(J->pt) &&
 126	      J->pc < proto_bc(J->pt) + J->pt->sizebc), "bad snapshot PC");
 127  while (frame > lim) {  /* Backwards traversal of all frames above base. */
 128    if (frame_islua(frame)) {
 129#if !LJ_FR2
 130      map[f++] = SNAP_MKPC(frame_pc(frame));
 131#endif
 132      frame = frame_prevl(frame);
 133    } else if (frame_iscont(frame)) {
 134#if !LJ_FR2
 135      map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
 136      map[f++] = SNAP_MKPC(frame_contpc(frame));
 137#endif
 138      frame = frame_prevd(frame);
 139    } else {
 140      lj_assertJ(!frame_isc(frame), "broken frame chain");
 141#if !LJ_FR2
 142      map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
 143#endif
 144      frame = frame_prevd(frame);
 145      continue;
 146    }
 147    if (frame + funcproto(frame_func(frame))->framesize > ftop)
 148      ftop = frame + funcproto(frame_func(frame))->framesize;
 149  }
 150  *topslot = (uint8_t)(ftop - lim);
 151#if LJ_FR2
 152  lj_assertJ(sizeof(SnapEntry) * 2 == sizeof(uint64_t), "bad SnapEntry def");
 153  return 2;
 154#else
 155  lj_assertJ(f == (MSize)(1 + J->framedepth), "miscalculated snapshot size");
 156  return f;
 157#endif
 158}
 159
 160/* Take a snapshot of the current stack. */
 161static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
 162{
 163  BCReg nslots = J->baseslot + J->maxslot;
 164  MSize nent;
 165  SnapEntry *p;
 166  /* Conservative estimate. */
 167  lj_snap_grow_map(J, nsnapmap + nslots + (MSize)(LJ_FR2?2:J->framedepth+1));
 168  p = &J->cur.snapmap[nsnapmap];
 169  nent = snapshot_slots(J, p, nslots);
 170  snap->nent = (uint8_t)nent;
 171  nent += snapshot_framelinks(J, p + nent, &snap->topslot);
 172  snap->mapofs = (uint32_t)nsnapmap;
 173  snap->ref = (IRRef1)J->cur.nins;
 174  snap->mcofs = 0;
 175  snap->nslots = (uint8_t)nslots;
 176  snap->count = 0;
 177  J->cur.nsnapmap = (uint32_t)(nsnapmap + nent);
 178}
 179
 180/* Add or merge a snapshot. */
 181void lj_snap_add(jit_State *J)
 182{
 183  MSize nsnap = J->cur.nsnap;
 184  MSize nsnapmap = J->cur.nsnapmap;
 185  /* Merge if no ins. inbetween or if requested and no guard inbetween. */
 186  if ((nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins) ||
 187      (J->mergesnap && !irt_isguard(J->guardemit))) {
 188    if (nsnap == 1) {  /* But preserve snap #0 PC. */
 189      emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
 190      goto nomerge;
 191    }
 192    nsnapmap = J->cur.snap[--nsnap].mapofs;
 193  } else {
 194  nomerge:
 195    lj_snap_grow_buf(J, nsnap+1);
 196    J->cur.nsnap = (uint16_t)(nsnap+1);
 197  }
 198  J->mergesnap = 0;
 199  J->guardemit.irt = 0;
 200  snapshot_stack(J, &J->cur.snap[nsnap], nsnapmap);
 201}
 202
 203/* -- Snapshot modification ----------------------------------------------- */
 204
 205#define SNAP_USEDEF_SLOTS	(LJ_MAX_JSLOTS+LJ_STACK_EXTRA)
 206
 207/* Find unused slots with reaching-definitions bytecode data-flow analysis. */
 208static BCReg snap_usedef(jit_State *J, uint8_t *udf,
 209			 const BCIns *pc, BCReg maxslot)
 210{
 211  BCReg s;
 212  GCobj *o;
 213
 214  if (maxslot == 0) return 0;
 215#ifdef LUAJIT_USE_VALGRIND
 216  /* Avoid errors for harmless reads beyond maxslot. */
 217  memset(udf, 1, SNAP_USEDEF_SLOTS);
 218#else
 219  memset(udf, 1, maxslot);
 220#endif
 221
 222  /* Treat open upvalues as used. */
 223  o = gcref(J->L->openupval);
 224  while (o) {
 225    if (uvval(gco2uv(o)) < J->L->base) break;
 226    udf[uvval(gco2uv(o)) - J->L->base] = 0;
 227    o = gcref(o->gch.nextgc);
 228  }
 229
 230#define USE_SLOT(s)		udf[(s)] &= ~1
 231#define DEF_SLOT(s)		udf[(s)] *= 3
 232
 233  /* Scan through following bytecode and check for uses/defs. */
 234  lj_assertJ(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc,
 235	     "snapshot PC out of range");
 236  for (;;) {
 237    BCIns ins = *pc++;
 238    BCOp op = bc_op(ins);
 239    switch (bcmode_b(op)) {
 240    case BCMvar: USE_SLOT(bc_b(ins)); break;
 241    default: break;
 242    }
 243    switch (bcmode_c(op)) {
 244    case BCMvar: USE_SLOT(bc_c(ins)); break;
 245    case BCMrbase:
 246      lj_assertJ(op == BC_CAT, "unhandled op %d with RC rbase", op);
 247      for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s);
 248      for (; s < maxslot; s++) DEF_SLOT(s);
 249      break;
 250    case BCMjump:
 251    handle_jump: {
 252      BCReg minslot = bc_a(ins);
 253      if (op >= BC_FORI && op <= BC_JFORL) minslot += FORL_EXT;
 254      else if (op >= BC_ITERL && op <= BC_JITERL) minslot += bc_b(pc[-2])-1;
 255      else if (op == BC_UCLO) {
 256	ptrdiff_t delta = bc_j(ins);
 257	if (delta < 0) return maxslot;  /* Prevent loop. */
 258	pc += delta;
 259	break;
 260      }
 261      for (s = minslot; s < maxslot; s++) DEF_SLOT(s);
 262      return minslot < maxslot ? minslot : maxslot;
 263      }
 264    case BCMlit:
 265      if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) {
 266	goto handle_jump;
 267      } else if (bc_isret(op)) {
 268	BCReg top = op == BC_RETM ? maxslot : (bc_a(ins) + bc_d(ins)-1);
 269	for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
 270	for (; s < top; s++) USE_SLOT(s);
 271	for (; s < maxslot; s++) DEF_SLOT(s);
 272	return 0;
 273      }
 274      break;
 275    case BCMfunc: return maxslot;  /* NYI: will abort, anyway. */
 276    default: break;
 277    }
 278    switch (bcmode_a(op)) {
 279    case BCMvar: USE_SLOT(bc_a(ins)); break;
 280    case BCMdst:
 281       if (!(op == BC_ISTC || op == BC_ISFC)) DEF_SLOT(bc_a(ins));
 282       break;
 283    case BCMbase:
 284      if (op >= BC_CALLM && op <= BC_ITERN) {
 285	BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ?
 286		    maxslot : (bc_a(ins) + bc_c(ins)+LJ_FR2);
 287	if (LJ_FR2) DEF_SLOT(bc_a(ins)+1);
 288	s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0);
 289	for (; s < top; s++) USE_SLOT(s);
 290	for (; s < maxslot; s++) DEF_SLOT(s);
 291	if (op == BC_CALLT || op == BC_CALLMT) {
 292	  for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
 293	  return 0;
 294	}
 295      } else if (op == BC_VARG) {
 296	return maxslot;  /* NYI: punt. */
 297      } else if (op == BC_KNIL) {
 298	for (s = bc_a(ins); s <= bc_d(ins); s++) DEF_SLOT(s);
 299      } else if (op == BC_TSETM) {
 300	for (s = bc_a(ins)-1; s < maxslot; s++) USE_SLOT(s);
 301      }
 302      break;
 303    default: break;
 304    }
 305    lj_assertJ(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc,
 306	       "use/def analysis PC out of range");
 307  }
 308
 309#undef USE_SLOT
 310#undef DEF_SLOT
 311
 312  return 0;  /* unreachable */
 313}
 314
 315/* Mark slots used by upvalues of child prototypes as used. */
 316static void snap_useuv(GCproto *pt, uint8_t *udf)
 317{
 318  /* This is a coarse check, because it's difficult to correlate the lifetime
 319  ** of slots and closures. But the number of false positives is quite low.
 320  ** A false positive may cause a slot not to be purged, which is just
 321  ** a missed optimization.
 322  */
 323  if ((pt->flags & PROTO_CHILD)) {
 324    ptrdiff_t i, j, n = pt->sizekgc;
 325    GCRef *kr = mref(pt->k, GCRef) - 1;
 326    for (i = 0; i < n; i++, kr--) {
 327      GCobj *o = gcref(*kr);
 328      if (o->gch.gct == ~LJ_TPROTO) {
 329	for (j = 0; j < gco2pt(o)->sizeuv; j++) {
 330	  uint32_t v = proto_uv(gco2pt(o))[j];
 331	  if ((v & PROTO_UV_LOCAL)) {
 332	    udf[(v & 0xff)] = 0;
 333	  }
 334	}
 335      }
 336    }
 337  }
 338}
 339
 340/* Purge dead slots before the next snapshot. */
 341void lj_snap_purge(jit_State *J)
 342{
 343  uint8_t udf[SNAP_USEDEF_SLOTS];
 344  BCReg s, maxslot = J->maxslot;
 345  if (bc_op(*J->pc) == BC_FUNCV && maxslot > J->pt->numparams)
 346    maxslot = J->pt->numparams;
 347  s = snap_usedef(J, udf, J->pc, maxslot);
 348  if (s < maxslot) {
 349    snap_useuv(J->pt, udf);
 350    for (; s < maxslot; s++)
 351      if (udf[s] != 0)
 352	J->base[s] = 0;  /* Purge dead slots. */
 353  }
 354}
 355
 356/* Shrink last snapshot. */
 357void lj_snap_shrink(jit_State *J)
 358{
 359  SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
 360  SnapEntry *map = &J->cur.snapmap[snap->mapofs];
 361  MSize n, m, nlim, nent = snap->nent;
 362  uint8_t udf[SNAP_USEDEF_SLOTS];
 363  BCReg maxslot = J->maxslot;
 364  BCReg baseslot = J->baseslot;
 365  BCReg minslot = snap_usedef(J, udf, snap_pc(&map[nent]), maxslot);
 366  if (minslot < maxslot) snap_useuv(J->pt, udf);
 367  maxslot += baseslot;
 368  minslot += baseslot;
 369  snap->nslots = (uint8_t)maxslot;
 370  for (n = m = 0; n < nent; n++) {  /* Remove unused slots from snapshot. */
 371    BCReg s = snap_slot(map[n]);
 372    if (s < minslot || (s < maxslot && udf[s-baseslot] == 0))
 373      map[m++] = map[n];  /* Only copy used slots. */
 374  }
 375  snap->nent = (uint8_t)m;
 376  nlim = J->cur.nsnapmap - snap->mapofs - 1;
 377  while (n <= nlim) map[m++] = map[n++];  /* Move PC + frame links down. */
 378  J->cur.nsnapmap = (uint32_t)(snap->mapofs + m);  /* Free up space in map. */
 379}
 380
 381/* -- Snapshot access ----------------------------------------------------- */
 382
 383/* Initialize a Bloom Filter with all renamed refs.
 384** There are very few renames (often none), so the filter has
 385** very few bits set. This makes it suitable for negative filtering.
 386*/
 387static BloomFilter snap_renamefilter(GCtrace *T, SnapNo lim)
 388{
 389  BloomFilter rfilt = 0;
 390  IRIns *ir;
 391  for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
 392    if (ir->op2 <= lim)
 393      bloomset(rfilt, ir->op1);
 394  return rfilt;
 395}
 396
 397/* Process matching renames to find the original RegSP. */
 398static RegSP snap_renameref(GCtrace *T, SnapNo lim, IRRef ref, RegSP rs)
 399{
 400  IRIns *ir;
 401  for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
 402    if (ir->op1 == ref && ir->op2 <= lim)
 403      rs = ir->prev;
 404  return rs;
 405}
 406
 407/* Copy RegSP from parent snapshot to the parent links of the IR. */
 408IRIns *lj_snap_regspmap(jit_State *J, GCtrace *T, SnapNo snapno, IRIns *ir)
 409{
 410  SnapShot *snap = &T->snap[snapno];
 411  SnapEntry *map = &T->snapmap[snap->mapofs];
 412  BloomFilter rfilt = snap_renamefilter(T, snapno);
 413  MSize n = 0;
 414  IRRef ref = 0;
 415  UNUSED(J);
 416  for ( ; ; ir++) {
 417    uint32_t rs;
 418    if (ir->o == IR_SLOAD) {
 419      if (!(ir->op2 & IRSLOAD_PARENT)) break;
 420      for ( ; ; n++) {
 421	lj_assertJ(n < snap->nent, "slot %d not found in snapshot", ir->op1);
 422	if (snap_slot(map[n]) == ir->op1) {
 423	  ref = snap_ref(map[n++]);
 424	  break;
 425	}
 426      }
 427    } else if (LJ_SOFTFP32 && ir->o == IR_HIOP) {
 428      ref++;
 429    } else if (ir->o == IR_PVAL) {
 430      ref = ir->op1 + REF_BIAS;
 431    } else {
 432      break;
 433    }
 434    rs = T->ir[ref].prev;
 435    if (bloomtest(rfilt, ref))
 436      rs = snap_renameref(T, snapno, ref, rs);
 437    ir->prev = (uint16_t)rs;
 438    lj_assertJ(regsp_used(rs), "unused IR %04d in snapshot", ref - REF_BIAS);
 439  }
 440  return ir;
 441}
 442
 443/* -- Snapshot replay ----------------------------------------------------- */
 444
 445/* Replay constant from parent trace. */
 446static TRef snap_replay_const(jit_State *J, IRIns *ir)
 447{
 448  /* Only have to deal with constants that can occur in stack slots. */
 449  switch ((IROp)ir->o) {
 450  case IR_KPRI: return TREF_PRI(irt_type(ir->t));
 451  case IR_KINT: return lj_ir_kint(J, ir->i);
 452  case IR_KGC: return lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t));
 453  case IR_KNUM: case IR_KINT64:
 454    return lj_ir_k64(J, (IROp)ir->o, ir_k64(ir)->u64);
 455  case IR_KPTR: return lj_ir_kptr(J, ir_kptr(ir));  /* Continuation. */
 456  case IR_KNULL: return lj_ir_knull(J, irt_type(ir->t));
 457  default: lj_assertJ(0, "bad IR constant op %d", ir->o); return TREF_NIL;
 458  }
 459}
 460
 461/* De-duplicate parent reference. */
 462static TRef snap_dedup(jit_State *J, SnapEntry *map, MSize nmax, IRRef ref)
 463{
 464  MSize j;
 465  for (j = 0; j < nmax; j++)
 466    if (snap_ref(map[j]) == ref)
 467      return J->slot[snap_slot(map[j])] & ~(SNAP_KEYINDEX|SNAP_CONT|SNAP_FRAME);
 468  return 0;
 469}
 470
 471/* Emit parent reference with de-duplication. */
 472static TRef snap_pref(jit_State *J, GCtrace *T, SnapEntry *map, MSize nmax,
 473		      BloomFilter seen, IRRef ref)
 474{
 475  IRIns *ir = &T->ir[ref];
 476  TRef tr;
 477  if (irref_isk(ref))
 478    tr = snap_replay_const(J, ir);
 479  else if (!regsp_used(ir->prev))
 480    tr = 0;
 481  else if (!bloomtest(seen, ref) || (tr = snap_dedup(J, map, nmax, ref)) == 0)
 482    tr = emitir(IRT(IR_PVAL, irt_type(ir->t)), ref - REF_BIAS, 0);
 483  return tr;
 484}
 485
 486/* Check whether a sunk store corresponds to an allocation. Slow path. */
 487static int snap_sunk_store2(GCtrace *T, IRIns *ira, IRIns *irs)
 488{
 489  if (irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
 490      irs->o == IR_FSTORE || irs->o == IR_XSTORE) {
 491    IRIns *irk = &T->ir[irs->op1];
 492    if (irk->o == IR_AREF || irk->o == IR_HREFK)
 493      irk = &T->ir[irk->op1];
 494    return (&T->ir[irk->op1] == ira);
 495  }
 496  return 0;
 497}
 498
 499/* Check whether a sunk store corresponds to an allocation. Fast path. */
 500static LJ_AINLINE int snap_sunk_store(GCtrace *T, IRIns *ira, IRIns *irs)
 501{
 502  if (irs->s != 255)
 503    return (ira + irs->s == irs);  /* Fast check. */
 504  return snap_sunk_store2(T, ira, irs);
 505}
 506
 507/* Replay snapshot state to setup side trace. */
 508void lj_snap_replay(jit_State *J, GCtrace *T)
 509{
 510  SnapShot *snap = &T->snap[J->exitno];
 511  SnapEntry *map = &T->snapmap[snap->mapofs];
 512  MSize n, nent = snap->nent;
 513  BloomFilter seen = 0;
 514  int pass23 = 0;
 515  J->framedepth = 0;
 516  /* Emit IR for slots inherited from parent snapshot. */
 517  for (n = 0; n < nent; n++) {
 518    SnapEntry sn = map[n];
 519    BCReg s = snap_slot(sn);
 520    IRRef ref = snap_ref(sn);
 521    IRIns *ir = &T->ir[ref];
 522    TRef tr;
 523    /* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
 524    if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0)
 525      goto setslot;
 526    bloomset(seen, ref);
 527    if (irref_isk(ref)) {
 528      /* See special treatment of LJ_FR2 slot 1 in snapshot_slots() above. */
 529      if (LJ_FR2 && (sn == SNAP(1, SNAP_FRAME | SNAP_NORESTORE, REF_NIL)))
 530	tr = 0;
 531      else
 532	tr = snap_replay_const(J, ir);
 533    } else if (!regsp_used(ir->prev)) {
 534      pass23 = 1;
 535      lj_assertJ(s != 0, "unused slot 0 in snapshot");
 536      tr = s;
 537    } else {
 538      IRType t = irt_type(ir->t);
 539      uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
 540      if (LJ_SOFTFP32 && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
 541      if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
 542      if ((sn & SNAP_KEYINDEX)) mode |= IRSLOAD_KEYINDEX;
 543      tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
 544    }
 545  setslot:
 546    /* Same as TREF_* flags. */
 547    J->slot[s] = tr | (sn&(SNAP_KEYINDEX|SNAP_CONT|SNAP_FRAME));
 548    J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && (s != LJ_FR2));
 549    if ((sn & SNAP_FRAME))
 550      J->baseslot = s+1;
 551  }
 552  if (pass23) {
 553    IRIns *irlast = &T->ir[snap->ref];
 554    pass23 = 0;
 555    /* Emit dependent PVALs. */
 556    for (n = 0; n < nent; n++) {
 557      SnapEntry sn = map[n];
 558      IRRef refp = snap_ref(sn);
 559      IRIns *ir = &T->ir[refp];
 560      if (regsp_reg(ir->r) == RID_SUNK) {
 561	uint8_t m;
 562	if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
 563	pass23 = 1;
 564	lj_assertJ(ir->o == IR_TNEW || ir->o == IR_TDUP ||
 565		   ir->o == IR_CNEW || ir->o == IR_CNEWI,
 566		   "sunk parent IR %04d has bad op %d", refp - REF_BIAS, ir->o);
 567	m = lj_ir_mode[ir->o];
 568	if (irm_op1(m) == IRMref) snap_pref(J, T, map, nent, seen, ir->op1);
 569	if (irm_op2(m) == IRMref) snap_pref(J, T, map, nent, seen, ir->op2);
 570	if (LJ_HASFFI && ir->o == IR_CNEWI) {
 571	  if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP)
 572	    snap_pref(J, T, map, nent, seen, (ir+1)->op2);
 573	} else {
 574	  IRIns *irs;
 575	  for (irs = ir+1; irs < irlast; irs++)
 576	    if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
 577	      if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
 578		snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
 579	      else if ((LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)) &&
 580		       irs+1 < irlast && (irs+1)->o == IR_HIOP)
 581		snap_pref(J, T, map, nent, seen, (irs+1)->op2);
 582	    }
 583	}
 584      } else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
 585	lj_assertJ(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT,
 586		   "sunk parent IR %04d has bad op %d", refp - REF_BIAS, ir->o);
 587	J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
 588      }
 589    }
 590    /* Replay sunk instructions. */
 591    for (n = 0; pass23 && n < nent; n++) {
 592      SnapEntry sn = map[n];
 593      IRRef refp = snap_ref(sn);
 594      IRIns *ir = &T->ir[refp];
 595      if (regsp_reg(ir->r) == RID_SUNK) {
 596	TRef op1, op2;
 597	uint8_t m;
 598	if (J->slot[snap_slot(sn)] != snap_slot(sn)) {  /* De-dup allocs. */
 599	  J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]];
 600	  continue;
 601	}
 602	op1 = ir->op1;
 603	m = lj_ir_mode[ir->o];
 604	if (irm_op1(m) == IRMref) op1 = snap_pref(J, T, map, nent, seen, op1);
 605	op2 = ir->op2;
 606	if (irm_op2(m) == IRMref) op2 = snap_pref(J, T, map, nent, seen, op2);
 607	if (LJ_HASFFI && ir->o == IR_CNEWI) {
 608	  if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) {
 609	    lj_needsplit(J);  /* Emit joining HIOP. */
 610	    op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2,
 611			     snap_pref(J, T, map, nent, seen, (ir+1)->op2));
 612	  }
 613	  J->slot[snap_slot(sn)] = emitir(ir->ot & ~(IRT_MARK|IRT_ISPHI), op1, op2);
 614	} else {
 615	  IRIns *irs;
 616	  TRef tr = emitir(ir->ot, op1, op2);
 617	  J->slot[snap_slot(sn)] = tr;
 618	  for (irs = ir+1; irs < irlast; irs++)
 619	    if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
 620	      IRIns *irr = &T->ir[irs->op1];
 621	      TRef val, key = irr->op2, tmp = tr;
 622	      if (irr->o != IR_FREF) {
 623		IRIns *irk = &T->ir[key];
 624		if (irr->o == IR_HREFK)
 625		  key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]),
 626				    irk->op2);
 627		else
 628		  key = snap_replay_const(J, irk);
 629		if (irr->o == IR_HREFK || irr->o == IR_AREF) {
 630		  IRIns *irf = &T->ir[irr->op1];
 631		  tmp = emitir(irf->ot, tmp, irf->op2);
 632		} else if (irr->o == IR_NEWREF) {
 633		  IRRef allocref = tref_ref(tr);
 634		  IRRef keyref = tref_ref(key);
 635		  IRRef newref_ref = J->chain[IR_NEWREF];
 636		  IRIns *newref = &J->cur.ir[newref_ref];
 637		  lj_assertJ(irref_isk(keyref),
 638			     "sunk store for parent IR %04d with bad key %04d",
 639			     refp - REF_BIAS, keyref - REF_BIAS);
 640		  if (newref_ref > allocref && newref->op2 == keyref) {
 641		    lj_assertJ(newref->op1 == allocref,
 642			       "sunk store for parent IR %04d with bad tab %04d",
 643			       refp - REF_BIAS, allocref - REF_BIAS);
 644		    tmp = newref_ref;
 645		    goto skip_newref;
 646		  }
 647		}
 648	      }
 649	      tmp = emitir(irr->ot, tmp, key);
 650	    skip_newref:
 651	      val = snap_pref(J, T, map, nent, seen, irs->op2);
 652	      if (val == 0) {
 653		IRIns *irc = &T->ir[irs->op2];
 654		lj_assertJ(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT,
 655			   "sunk store for parent IR %04d with bad op %d",
 656			   refp - REF_BIAS, irc->o);
 657		val = snap_pref(J, T, map, nent, seen, irc->op1);
 658		val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
 659	      } else if ((LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI)) &&
 660			 irs+1 < irlast && (irs+1)->o == IR_HIOP) {
 661		IRType t = IRT_I64;
 662		if (LJ_SOFTFP32 && irt_type((irs+1)->t) == IRT_SOFTFP)
 663		  t = IRT_NUM;
 664		lj_needsplit(J);
 665		if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
 666		  uint64_t k = (uint32_t)T->ir[irs->op2].i +
 667			       ((uint64_t)T->ir[(irs+1)->op2].i << 32);
 668		  val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM, k);
 669		} else {
 670		  val = emitir_raw(IRT(IR_HIOP, t), val,
 671			  snap_pref(J, T, map, nent, seen, (irs+1)->op2));
 672		}
 673		tmp = emitir(IRT(irs->o, t), tmp, val);
 674		continue;
 675	      }
 676	      tmp = emitir(irs->ot, tmp, val);
 677	    } else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) {
 678	      emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 679	    }
 680	}
 681      }
 682    }
 683  }
 684  J->base = J->slot + J->baseslot;
 685  J->maxslot = snap->nslots - J->baseslot;
 686  lj_snap_add(J);
 687  if (pass23)  /* Need explicit GC step _after_ initial snapshot. */
 688    emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0);
 689}
 690
 691/* -- Snapshot restore ---------------------------------------------------- */
 692
 693static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
 694			SnapNo snapno, BloomFilter rfilt,
 695			IRIns *ir, TValue *o);
 696
 697/* Restore a value from the trace exit state. */
 698static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
 699			    SnapNo snapno, BloomFilter rfilt,
 700			    IRRef ref, TValue *o)
 701{
 702  IRIns *ir = &T->ir[ref];
 703  IRType1 t = ir->t;
 704  RegSP rs = ir->prev;
 705  if (irref_isk(ref)) {  /* Restore constant slot. */
 706    if (ir->o == IR_KPTR) {
 707      o->u64 = (uint64_t)(uintptr_t)ir_kptr(ir);
 708    } else {
 709      lj_assertJ(!(ir->o == IR_KKPTR || ir->o == IR_KNULL),
 710		 "restore of const from IR %04d with bad op %d",
 711		 ref - REF_BIAS, ir->o);
 712      lj_ir_kvalue(J->L, o, ir);
 713    }
 714    return;
 715  }
 716  if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
 717    rs = snap_renameref(T, snapno, ref, rs);
 718  if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
 719    int32_t *sps = &ex->spill[regsp_spill(rs)];
 720    if (irt_isinteger(t)) {
 721      setintV(o, *sps);
 722#if !LJ_SOFTFP32
 723    } else if (irt_isnum(t)) {
 724      o->u64 = *(uint64_t *)sps;
 725#endif
 726#if LJ_64 && !LJ_GC64
 727    } else if (irt_islightud(t)) {
 728      /* 64 bit lightuserdata which may escape already has the tag bits. */
 729      o->u64 = *(uint64_t *)sps;
 730#endif
 731    } else {
 732      lj_assertJ(!irt_ispri(t), "PRI ref with spill slot");
 733      setgcV(J->L, o, (GCobj *)(uintptr_t)*(GCSize *)sps, irt_toitype(t));
 734    }
 735  } else {  /* Restore from register. */
 736    Reg r = regsp_reg(rs);
 737    if (ra_noreg(r)) {
 738      lj_assertJ(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT,
 739		 "restore from IR %04d has no reg", ref - REF_BIAS);
 740      snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o);
 741      if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o));
 742      return;
 743    } else if (irt_isinteger(t)) {
 744      setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
 745#if !LJ_SOFTFP
 746    } else if (irt_isnum(t)) {
 747      setnumV(o, ex->fpr[r-RID_MIN_FPR]);
 748#elif LJ_64  /* && LJ_SOFTFP */
 749    } else if (irt_isnum(t)) {
 750      o->u64 = ex->gpr[r-RID_MIN_GPR];
 751#endif
 752#if LJ_64 && !LJ_GC64
 753    } else if (irt_is64(t)) {
 754      /* 64 bit values that already have the tag bits. */
 755      o->u64 = ex->gpr[r-RID_MIN_GPR];
 756#endif
 757    } else if (irt_ispri(t)) {
 758      setpriV(o, irt_toitype(t));
 759    } else {
 760      setgcV(J->L, o, (GCobj *)ex->gpr[r-RID_MIN_GPR], irt_toitype(t));
 761    }
 762  }
 763}
 764
 765#if LJ_HASFFI
 766/* Restore raw data from the trace exit state. */
 767static void snap_restoredata(jit_State *J, GCtrace *T, ExitState *ex,
 768			     SnapNo snapno, BloomFilter rfilt,
 769			     IRRef ref, void *dst, CTSize sz)
 770{
 771  IRIns *ir = &T->ir[ref];
 772  RegSP rs = ir->prev;
 773  int32_t *src;
 774  uint64_t tmp;
 775  UNUSED(J);
 776  if (irref_isk(ref)) {
 777    if (ir_isk64(ir)) {
 778      src = (int32_t *)&ir[1];
 779    } else if (sz == 8) {
 780      tmp = (uint64_t)(uint32_t)ir->i;
 781      src = (int32_t *)&tmp;
 782    } else {
 783      src = &ir->i;
 784    }
 785  } else {
 786    if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
 787      rs = snap_renameref(T, snapno, ref, rs);
 788    if (ra_hasspill(regsp_spill(rs))) {
 789      src = &ex->spill[regsp_spill(rs)];
 790      if (sz == 8 && !irt_is64(ir->t)) {
 791	tmp = (uint64_t)(uint32_t)*src;
 792	src = (int32_t *)&tmp;
 793      }
 794    } else {
 795      Reg r = regsp_reg(rs);
 796      if (ra_noreg(r)) {
 797	/* Note: this assumes CNEWI is never used for SOFTFP split numbers. */
 798	lj_assertJ(sz == 8 && ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT,
 799		   "restore from IR %04d has no reg", ref - REF_BIAS);
 800	snap_restoredata(J, T, ex, snapno, rfilt, ir->op1, dst, 4);
 801	*(lua_Number *)dst = (lua_Number)*(int32_t *)dst;
 802	return;
 803      }
 804#if !LJ_SOFTFP
 805      if (r >= RID_MAX_GPR) {
 806	src = (int32_t *)&ex->fpr[r-RID_MIN_FPR];
 807#if LJ_TARGET_PPC
 808	if (sz == 4) {  /* PPC FPRs are always doubles. */
 809	  *(float *)dst = (float)*(double *)src;
 810	  return;
 811	}
 812#else
 813	if (LJ_BE && sz == 4) src++;
 814#endif
 815      } else
 816#endif
 817      {
 818	src = (int32_t *)&ex->gpr[r-RID_MIN_GPR];
 819	if (LJ_64 && LJ_BE && sz == 4) src++;
 820      }
 821    }
 822  }
 823  lj_assertJ(sz == 1 || sz == 2 || sz == 4 || sz == 8,
 824	     "restore from IR %04d with bad size %d", ref - REF_BIAS, sz);
 825  if (sz == 4) *(int32_t *)dst = *src;
 826  else if (sz == 8) *(int64_t *)dst = *(int64_t *)src;
 827  else if (sz == 1) *(int8_t *)dst = (int8_t)*src;
 828  else *(int16_t *)dst = (int16_t)*src;
 829}
 830#endif
 831
 832/* Unsink allocation from the trace exit state. Unsink sunk stores. */
 833static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
 834			SnapNo snapno, BloomFilter rfilt,
 835			IRIns *ir, TValue *o)
 836{
 837  lj_assertJ(ir->o == IR_TNEW || ir->o == IR_TDUP ||
 838	     ir->o == IR_CNEW || ir->o == IR_CNEWI,
 839	     "sunk allocation with bad op %d", ir->o);
 840#if LJ_HASFFI
 841  if (ir->o == IR_CNEW || ir->o == IR_CNEWI) {
 842    CTState *cts = ctype_cts(J->L);
 843    CTypeID id = (CTypeID)T->ir[ir->op1].i;
 844    CTSize sz;
 845    CTInfo info = lj_ctype_info(cts, id, &sz);
 846    GCcdata *cd = lj_cdata_newx(cts, id, sz, info);
 847    setcdataV(J->L, o, cd);
 848    if (ir->o == IR_CNEWI) {
 849      uint8_t *p = (uint8_t *)cdataptr(cd);
 850      lj_assertJ(sz == 4 || sz == 8, "sunk cdata with bad size %d", sz);
 851      if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) {
 852	snap_restoredata(J, T, ex, snapno, rfilt, (ir+1)->op2,
 853			 LJ_LE ? p+4 : p, 4);
 854	if (LJ_BE) p += 4;
 855	sz = 4;
 856      }
 857      snap_restoredata(J, T, ex, snapno, rfilt, ir->op2, p, sz);
 858    } else {
 859      IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref];
 860      for (irs = ir+1; irs < irlast; irs++)
 861	if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
 862	  IRIns *iro = &T->ir[T->ir[irs->op1].op2];
 863	  uint8_t *p = (uint8_t *)cd;
 864	  CTSize szs;
 865	  lj_assertJ(irs->o == IR_XSTORE, "sunk store with bad op %d", irs->o);
 866	  lj_assertJ(T->ir[irs->op1].o == IR_ADD,
 867		     "sunk store with bad add op %d", T->ir[irs->op1].o);
 868	  lj_assertJ(iro->o == IR_KINT || iro->o == IR_KINT64,
 869		     "sunk store with bad const offset op %d", iro->o);
 870	  if (irt_is64(irs->t)) szs = 8;
 871	  else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1;
 872	  else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2;
 873	  else szs = 4;
 874	  if (LJ_64 && iro->o == IR_KINT64)
 875	    p += (int64_t)ir_k64(iro)->u64;
 876	  else
 877	    p += iro->i;
 878	  lj_assertJ(p >= (uint8_t *)cdataptr(cd) &&
 879		     p + szs <= (uint8_t *)cdataptr(cd) + sz,
 880		     "sunk store with offset out of range");
 881	  if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
 882	    lj_assertJ(szs == 4, "sunk store with bad size %d", szs);
 883	    snap_restoredata(J, T, ex, snapno, rfilt, (irs+1)->op2,
 884			     LJ_LE ? p+4 : p, 4);
 885	    if (LJ_BE) p += 4;
 886	  }
 887	  snap_restoredata(J, T, ex, snapno, rfilt, irs->op2, p, szs);
 888	}
 889    }
 890  } else
 891#endif
 892  {
 893    IRIns *irs, *irlast;
 894    GCtab *t = ir->o == IR_TNEW ? lj_tab_new(J->L, ir->op1, ir->op2) :
 895				  lj_tab_dup(J->L, ir_ktab(&T->ir[ir->op1]));
 896    settabV(J->L, o, t);
 897    irlast = &T->ir[T->snap[snapno].ref];
 898    for (irs = ir+1; irs < irlast; irs++)
 899      if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
 900	IRIns *irk = &T->ir[irs->op1];
 901	TValue tmp, *val;
 902	lj_assertJ(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
 903		   irs->o == IR_FSTORE,
 904		   "sunk store with bad op %d", irs->o);
 905	if (irk->o == IR_FREF) {
 906	  switch (irk->op2) {
 907	  case IRFL_TAB_META:
 908	    if (T->ir[irs->op2].o == IR_KNULL) {
 909	      setgcrefnull(t->metatable);
 910	    } else {
 911	      snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp);
 912	      /* NOBARRIER: The table is new (marked white). */
 913	      setgcref(t->metatable, obj2gco(tabV(&tmp)));
 914	    }
 915	    break;
 916	  case IRFL_TAB_NOMM:
 917	    /* Negative metamethod cache invalidated by lj_tab_set() below. */
 918	    break;
 919	  default:
 920	    lj_assertJ(0, "sunk store with bad field %d", irk->op2);
 921	    break;
 922	  }
 923	} else {
 924	  irk = &T->ir[irk->op2];
 925	  if (irk->o == IR_KSLOT) irk = &T->ir[irk->op1];
 926	  lj_ir_kvalue(J->L, &tmp, irk);
 927	  val = lj_tab_set(J->L, t, &tmp);
 928	  /* NOBARRIER: The table is new (marked white). */
 929	  snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val);
 930	  if (LJ_SOFTFP32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
 931	    snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp);
 932	    val->u32.hi = tmp.u32.lo;
 933	  }
 934	}
 935      }
 936  }
 937}
 938
 939/* Restore interpreter state from exit state with the help of a snapshot. */
 940const BCIns *lj_snap_restore(jit_State *J, void *exptr)
 941{
 942  ExitState *ex = (ExitState *)exptr;
 943  SnapNo snapno = J->exitno;  /* For now, snapno == exitno. */
 944  GCtrace *T = traceref(J, J->parent);
 945  SnapShot *snap = &T->snap[snapno];
 946  MSize n, nent = snap->nent;
 947  SnapEntry *map = &T->snapmap[snap->mapofs];
 948#if !LJ_FR2 || defined(LUA_USE_ASSERT)
 949  SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1-LJ_FR2];
 950#endif
 951#if !LJ_FR2
 952  ptrdiff_t ftsz0;
 953#endif
 954  TValue *frame;
 955  BloomFilter rfilt = snap_renamefilter(T, snapno);
 956  const BCIns *pc = snap_pc(&map[nent]);
 957  lua_State *L = J->L;
 958
 959  /* Set interpreter PC to the next PC to get correct error messages.
 960  ** But not for returns or tail calls, since pc+1 may be out-of-range.
 961  */
 962  setcframe_pc(L->cframe, bc_isret_or_tail(bc_op(*pc)) ? pc : pc+1);
 963  setcframe_pc(cframe_raw(cframe_prev(L->cframe)), pc);
 964
 965  /* Make sure the stack is big enough for the slots from the snapshot. */
 966  if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) {
 967    L->top = curr_topL(L);
 968    lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize);
 969  }
 970
 971  /* Fill stack slots with data from the registers and spill slots. */
 972  frame = L->base-1-LJ_FR2;
 973#if !LJ_FR2
 974  ftsz0 = frame_ftsz(frame);  /* Preserve link to previous frame in slot #0. */
 975#endif
 976  for (n = 0; n < nent; n++) {
 977    SnapEntry sn = map[n];
 978    if (!(sn & SNAP_NORESTORE)) {
 979      TValue *o = &frame[snap_slot(sn)];
 980      IRRef ref = snap_ref(sn);
 981      IRIns *ir = &T->ir[ref];
 982      if (ir->r == RID_SUNK) {
 983	MSize j;
 984	for (j = 0; j < n; j++)
 985	  if (snap_ref(map[j]) == ref) {  /* De-duplicate sunk allocations. */
 986	    copyTV(L, o, &frame[snap_slot(map[j])]);
 987	    goto dupslot;
 988	  }
 989	snap_unsink(J, T, ex, snapno, rfilt, ir, o);
 990      dupslot:
 991	continue;
 992      }
 993      snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
 994      if (LJ_SOFTFP32 && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
 995	TValue tmp;
 996	snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
 997	o->u32.hi = tmp.u32.lo;
 998#if !LJ_FR2
 999      } else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1000	/* Overwrite tag with frame link. */
1001	setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
1002	L->base = o+1;
1003#endif
1004      } else if ((sn & SNAP_KEYINDEX)) {
1005	/* A IRT_INT key index slot is restored as a number. Undo this. */
1006	o->u32.lo = (uint32_t)(LJ_DUALNUM ? intV(o) : lj_num2int(numV(o)));
1007	o->u32.hi = LJ_KEYINDEX;
1008      }
1009    }
1010  }
1011#if LJ_FR2
1012  L->base += (map[nent+LJ_BE] & 0xff);
1013#endif
1014  lj_assertJ(map + nent == flinks, "inconsistent frames in snapshot");
1015
1016  /* Compute current stack top. */
1017  switch (bc_op(*pc)) {
1018  default:
1019    if (bc_op(*pc) < BC_FUNCF) {
1020      L->top = curr_topL(L);
1021      break;
1022    }
1023    /* fallthrough */
1024  case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
1025    L->top = frame + snap->nslots;
1026    break;
1027  }
1028  return pc;
1029}
1030
1031#undef emitir_raw
1032#undef emitir
1033
1034#endif