luna - luajit/src/lj_opt

  1/*
  2** SPLIT: Split 64 bit IR instructions into 32 bit IR instructions.
  3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
  4*/
  5
  6#define lj_opt_split_c
  7#define LUA_CORE
  8
  9#include "lj_obj.h"
 10
 11#if LJ_HASJIT && (LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI))
 12
 13#include "lj_err.h"
 14#include "lj_buf.h"
 15#include "lj_ir.h"
 16#include "lj_jit.h"
 17#include "lj_ircall.h"
 18#include "lj_iropt.h"
 19#include "lj_dispatch.h"
 20#include "lj_vm.h"
 21
 22/* SPLIT pass:
 23**
 24** This pass splits up 64 bit IR instructions into multiple 32 bit IR
 25** instructions. It's only active for soft-float targets or for 32 bit CPUs
 26** which lack native 64 bit integer operations (the FFI is currently the
 27** only emitter for 64 bit integer instructions).
 28**
 29** Splitting the IR in a separate pass keeps each 32 bit IR assembler
 30** backend simple. Only a small amount of extra functionality needs to be
 31** implemented. This is much easier than adding support for allocating
 32** register pairs to each backend (believe me, I tried). A few simple, but
 33** important optimizations can be performed by the SPLIT pass, which would
 34** be tedious to do in the backend.
 35**
 36** The basic idea is to replace each 64 bit IR instruction with its 32 bit
 37** equivalent plus an extra HIOP instruction. The splitted IR is not passed
 38** through FOLD or any other optimizations, so each HIOP is guaranteed to
 39** immediately follow it's counterpart. The actual functionality of HIOP is
 40** inferred from the previous instruction.
 41**
 42** The operands of HIOP hold the hiword input references. The output of HIOP
 43** is the hiword output reference, which is also used to hold the hiword
 44** register or spill slot information. The register allocator treats this
 45** instruction independently of any other instruction, which improves code
 46** quality compared to using fixed register pairs.
 47**
 48** It's easier to split up some instructions into two regular 32 bit
 49** instructions. E.g. XLOAD is split up into two XLOADs with two different
 50** addresses. Obviously 64 bit constants need to be split up into two 32 bit
 51** constants, too. Some hiword instructions can be entirely omitted, e.g.
 52** when zero-extending a 32 bit value to 64 bits. 64 bit arguments for calls
 53** are split up into two 32 bit arguments each.
 54**
 55** On soft-float targets, floating-point instructions are directly converted
 56** to soft-float calls by the SPLIT pass (except for comparisons and MIN/MAX).
 57** HIOP for number results has the type IRT_SOFTFP ("sfp" in -jdump).
 58**
 59** Here's the IR and x64 machine code for 'x.b = x.a + 1' for a struct with
 60** two int64_t fields:
 61**
 62** 0100    p32 ADD    base  +8
 63** 0101    i64 XLOAD  0100
 64** 0102    i64 ADD    0101  +1
 65** 0103    p32 ADD    base  +16
 66** 0104    i64 XSTORE 0103  0102
 67**
 68**         mov rax, [esi+0x8]
 69**         add rax, +0x01
 70**         mov [esi+0x10], rax
 71**
 72** Here's the transformed IR and the x86 machine code after the SPLIT pass:
 73**
 74** 0100    p32 ADD    base  +8
 75** 0101    int XLOAD  0100
 76** 0102    p32 ADD    base  +12
 77** 0103    int XLOAD  0102
 78** 0104    int ADD    0101  +1
 79** 0105    int HIOP   0103  +0
 80** 0106    p32 ADD    base  +16
 81** 0107    int XSTORE 0106  0104
 82** 0108    int HIOP   0106  0105
 83**
 84**         mov eax, [esi+0x8]
 85**         mov ecx, [esi+0xc]
 86**         add eax, +0x01
 87**         adc ecx, +0x00
 88**         mov [esi+0x10], eax
 89**         mov [esi+0x14], ecx
 90**
 91** You may notice the reassociated hiword address computation, which is
 92** later fused into the mov operands by the assembler.
 93*/
 94
 95/* Some local macros to save typing. Undef'd at the end. */
 96#define IR(ref)		(&J->cur.ir[(ref)])
 97
 98/* Directly emit the transformed IR without updating chains etc. */
 99static IRRef split_emit(jit_State *J, uint16_t ot, IRRef1 op1, IRRef1 op2)
100{
101  IRRef nref = lj_ir_nextins(J);
102  IRIns *ir = IR(nref);
103  ir->ot = ot;
104  ir->op1 = op1;
105  ir->op2 = op2;
106  return nref;
107}
108
109#if LJ_SOFTFP
110/* Emit a (checked) number to integer conversion. */
111static IRRef split_num2int(jit_State *J, IRRef lo, IRRef hi, int check)
112{
113  IRRef tmp, res;
114#if LJ_LE
115  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), lo, hi);
116#else
117  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hi, lo);
118#endif
119  res = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_softfp_d2i);
120  if (check) {
121    tmp = split_emit(J, IRTI(IR_CALLN), res, IRCALL_softfp_i2d);
122    split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
123    split_emit(J, IRTGI(IR_EQ), tmp, lo);
124    split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP), tmp+1, hi);
125  }
126  return res;
127}
128
129/* Emit a CALLN with one split 64 bit argument. */
130static IRRef split_call_l(jit_State *J, IRRef1 *hisubst, IRIns *oir,
131			  IRIns *ir, IRCallID id)
132{
133  IRRef tmp, op1 = ir->op1;
134  J->cur.nins--;
135#if LJ_LE
136  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
137#else
138  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
139#endif
140  ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
141  return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
142}
143#endif
144
145/* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
146static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
147			   IRIns *ir, IRCallID id)
148{
149  IRRef tmp, op1 = ir->op1, op2 = ir->op2;
150  J->cur.nins--;
151#if LJ_LE
152  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
153#else
154  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
155#endif
156  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
157  ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
158  return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
159}
160
161/* Emit a CALLN with two split 64 bit arguments. */
162static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir,
163			   IRIns *ir, IRCallID id)
164{
165  IRRef tmp, op1 = ir->op1, op2 = ir->op2;
166  J->cur.nins--;
167#if LJ_LE
168  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
169  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
170  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
171#else
172  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
173  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
174  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
175#endif
176  ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
177  return split_emit(J,
178    IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
179    tmp, tmp);
180}
181
182/* Get a pointer to the other 32 bit word (LE: hiword, BE: loword). */
183static IRRef split_ptr(jit_State *J, IRIns *oir, IRRef ref)
184{
185  IRRef nref = oir[ref].prev;
186  IRIns *ir = IR(nref);
187  int32_t ofs = 4;
188  if (ir->o == IR_KPTR)
189    return lj_ir_kptr(J, (char *)ir_kptr(ir) + ofs);
190  if (ir->o == IR_ADD && irref_isk(ir->op2) && !irt_isphi(oir[ref].t)) {
191    /* Reassociate address. */
192    ofs += IR(ir->op2)->i;
193    nref = ir->op1;
194    if (ofs == 0) return nref;
195  }
196  return split_emit(J, IRT(IR_ADD, IRT_PTR), nref, lj_ir_kint(J, ofs));
197}
198
199#if LJ_HASFFI
200static IRRef split_bitshift(jit_State *J, IRRef1 *hisubst,
201			    IRIns *oir, IRIns *nir, IRIns *ir)
202{
203  IROp op = ir->o;
204  IRRef kref = nir->op2;
205  if (irref_isk(kref)) {  /* Optimize constant shifts. */
206    int32_t k = (IR(kref)->i & 63);
207    IRRef lo = nir->op1, hi = hisubst[ir->op1];
208    if (op == IR_BROL || op == IR_BROR) {
209      if (op == IR_BROR) k = (-k & 63);
210      if (k >= 32) { IRRef t = lo; lo = hi; hi = t; k -= 32; }
211      if (k == 0) {
212      passthrough:
213	J->cur.nins--;
214	ir->prev = lo;
215	return hi;
216      } else {
217	TRef k1, k2;
218	IRRef t1, t2, t3, t4;
219	J->cur.nins--;
220	k1 = lj_ir_kint(J, k);
221	k2 = lj_ir_kint(J, (-k & 31));
222	t1 = split_emit(J, IRTI(IR_BSHL), lo, k1);
223	t2 = split_emit(J, IRTI(IR_BSHL), hi, k1);
224	t3 = split_emit(J, IRTI(IR_BSHR), lo, k2);
225	t4 = split_emit(J, IRTI(IR_BSHR), hi, k2);
226	ir->prev = split_emit(J, IRTI(IR_BOR), t1, t4);
227	return split_emit(J, IRTI(IR_BOR), t2, t3);
228      }
229    } else if (k == 0) {
230      goto passthrough;
231    } else if (k < 32) {
232      if (op == IR_BSHL) {
233	IRRef t1 = split_emit(J, IRTI(IR_BSHL), hi, kref);
234	IRRef t2 = split_emit(J, IRTI(IR_BSHR), lo, lj_ir_kint(J, (-k&31)));
235	return split_emit(J, IRTI(IR_BOR), t1, t2);
236      } else {
237	IRRef t1 = ir->prev, t2;
238	lj_assertJ(op == IR_BSHR || op == IR_BSAR, "bad usage");
239	nir->o = IR_BSHR;
240	t2 = split_emit(J, IRTI(IR_BSHL), hi, lj_ir_kint(J, (-k&31)));
241	ir->prev = split_emit(J, IRTI(IR_BOR), t1, t2);
242	return split_emit(J, IRTI(op), hi, kref);
243      }
244    } else {
245      if (op == IR_BSHL) {
246	if (k == 32)
247	  J->cur.nins--;
248	else
249	  lo = ir->prev;
250	ir->prev = lj_ir_kint(J, 0);
251	return lo;
252      } else {
253	lj_assertJ(op == IR_BSHR || op == IR_BSAR, "bad usage");
254	if (k == 32) {
255	  J->cur.nins--;
256	  ir->prev = hi;
257	} else {
258	  nir->op1 = hi;
259	}
260	if (op == IR_BSHR)
261	  return lj_ir_kint(J, 0);
262	else
263	  return split_emit(J, IRTI(IR_BSAR), hi, lj_ir_kint(J, 31));
264      }
265    }
266  }
267  return split_call_li(J, hisubst, oir, ir,
268		       op - IR_BSHL + IRCALL_lj_carith_shl64);
269}
270
271static IRRef split_bitop(jit_State *J, IRRef1 *hisubst,
272			 IRIns *nir, IRIns *ir)
273{
274  IROp op = ir->o;
275  IRRef hi, kref = nir->op2;
276  if (irref_isk(kref)) {  /* Optimize bit operations with lo constant. */
277    int32_t k = IR(kref)->i;
278    if (k == 0 || k == -1) {
279      if (op == IR_BAND) k = ~k;
280      if (k == 0) {
281	J->cur.nins--;
282	ir->prev = nir->op1;
283      } else if (op == IR_BXOR) {
284	nir->o = IR_BNOT;
285	nir->op2 = 0;
286      } else {
287	J->cur.nins--;
288	ir->prev = kref;
289      }
290    }
291  }
292  hi = hisubst[ir->op1];
293  kref = hisubst[ir->op2];
294  if (irref_isk(kref)) {  /* Optimize bit operations with hi constant. */
295    int32_t k = IR(kref)->i;
296    if (k == 0 || k == -1) {
297      if (op == IR_BAND) k = ~k;
298      if (k == 0) {
299	return hi;
300      } else if (op == IR_BXOR) {
301	return split_emit(J, IRTI(IR_BNOT), hi, 0);
302      } else {
303	return kref;
304      }
305    }
306  }
307  return split_emit(J, IRTI(op), hi, kref);
308}
309#endif
310
311/* Substitute references of a snapshot. */
312static void split_subst_snap(jit_State *J, SnapShot *snap, IRIns *oir)
313{
314  SnapEntry *map = &J->cur.snapmap[snap->mapofs];
315  MSize n, nent = snap->nent;
316  for (n = 0; n < nent; n++) {
317    SnapEntry sn = map[n];
318    IRIns *ir = &oir[snap_ref(sn)];
319    if (!(LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && irref_isk(snap_ref(sn))))
320      map[n] = ((sn & 0xffff0000) | ir->prev);
321  }
322}
323
324/* Transform the old IR to the new IR. */
325static void split_ir(jit_State *J)
326{
327  IRRef nins = J->cur.nins, nk = J->cur.nk;
328  MSize irlen = nins - nk;
329  MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
330  IRIns *oir = (IRIns *)lj_buf_tmp(J->L, need);
331  IRRef1 *hisubst;
332  IRRef ref, snref;
333  SnapShot *snap;
334
335  /* Copy old IR to buffer. */
336  memcpy(oir, IR(nk), irlen*sizeof(IRIns));
337  /* Bias hiword substitution table and old IR. Loword kept in field prev. */
338  hisubst = (IRRef1 *)&oir[irlen] - nk;
339  oir -= nk;
340
341  /* Remove all IR instructions, but retain IR constants. */
342  J->cur.nins = REF_FIRST;
343  J->loopref = 0;
344
345  /* Process constants and fixed references. */
346  for (ref = nk; ref <= REF_BASE; ref++) {
347    IRIns *ir = &oir[ref];
348    if ((LJ_SOFTFP && ir->o == IR_KNUM) || ir->o == IR_KINT64) {
349      /* Split up 64 bit constant. */
350      TValue tv = *ir_k64(ir);
351      ir->prev = lj_ir_kint(J, (int32_t)tv.u32.lo);
352      hisubst[ref] = lj_ir_kint(J, (int32_t)tv.u32.hi);
353    } else {
354      ir->prev = ref;  /* Identity substitution for loword. */
355      hisubst[ref] = 0;
356    }
357    if (irt_is64(ir->t) && ir->o != IR_KNULL)
358      ref++;
359  }
360
361  /* Process old IR instructions. */
362  snap = J->cur.snap;
363  snref = snap->ref;
364  for (ref = REF_FIRST; ref < nins; ref++) {
365    IRIns *ir = &oir[ref];
366    IRRef nref = lj_ir_nextins(J);
367    IRIns *nir = IR(nref);
368    IRRef hi = 0;
369
370    if (ref >= snref) {
371      snap->ref = nref;
372      split_subst_snap(J, snap++, oir);
373      snref = snap < &J->cur.snap[J->cur.nsnap] ? snap->ref : ~(IRRef)0;
374    }
375
376    /* Copy-substitute old instruction to new instruction. */
377    nir->op1 = ir->op1 < nk ? ir->op1 : oir[ir->op1].prev;
378    nir->op2 = ir->op2 < nk ? ir->op2 : oir[ir->op2].prev;
379    ir->prev = nref;  /* Loword substitution. */
380    nir->o = ir->o;
381    nir->t.irt = ir->t.irt & ~(IRT_MARK|IRT_ISPHI);
382    hisubst[ref] = 0;
383
384    /* Split 64 bit instructions. */
385#if LJ_SOFTFP
386    if (irt_isnum(ir->t)) {
387      nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
388      /* Note: hi ref = lo ref + 1! Required for SNAP_SOFTFPNUM logic. */
389      switch (ir->o) {
390      case IR_ADD:
391	hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_add);
392	break;
393      case IR_SUB:
394	hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_sub);
395	break;
396      case IR_MUL:
397	hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_mul);
398	break;
399      case IR_DIV:
400	hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div);
401	break;
402      case IR_POW:
403	hi = split_call_ll(J, hisubst, oir, ir, IRCALL_pow);
404	break;
405      case IR_FPMATH:
406	hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2);
407	break;
408      case IR_LDEXP:
409	hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp);
410	break;
411      case IR_NEG: case IR_ABS:
412	nir->o = IR_CONV;  /* Pass through loword. */
413	nir->op2 = (IRT_INT << 5) | IRT_INT;
414	hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP),
415	       hisubst[ir->op1],
416	       lj_ir_kint(J, (int32_t)(0x7fffffffu + (ir->o == IR_NEG))));
417	break;
418      case IR_SLOAD:
419	if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from int to number. */
420	  nir->op2 &= ~IRSLOAD_CONVERT;
421	  ir->prev = nref = split_emit(J, IRTI(IR_CALLN), nref,
422				       IRCALL_softfp_i2d);
423	  hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
424	  break;
425	}
426	/* fallthrough */
427      case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
428      case IR_STRTO:
429	hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
430	break;
431      case IR_FLOAD:
432	lj_assertJ(ir->op1 == REF_NIL, "expected FLOAD from GG_State");
433	hi = lj_ir_kint(J, *(int32_t*)((char*)J2GG(J) + ir->op2 + LJ_LE*4));
434	nir->op2 += LJ_BE*4;
435	break;
436      case IR_XLOAD: {
437	IRIns inslo = *nir;  /* Save/undo the emit of the lo XLOAD. */
438	J->cur.nins--;
439	hi = split_ptr(J, oir, ir->op1);  /* Insert the hiref ADD. */
440#if LJ_BE
441	hi = split_emit(J, IRT(IR_XLOAD, IRT_INT), hi, ir->op2);
442	inslo.t.irt = IRT_SOFTFP | (inslo.t.irt & IRT_GUARD);
443#endif
444	nref = lj_ir_nextins(J);
445	nir = IR(nref);
446	*nir = inslo;  /* Re-emit lo XLOAD. */
447#if LJ_LE
448	hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
449	ir->prev = nref;
450#else
451	ir->prev = hi; hi = nref;
452#endif
453	break;
454	}
455      case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_XSTORE:
456	split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nir->op1, hisubst[ir->op2]);
457	break;
458      case IR_CONV: {  /* Conversion to number. Others handled below. */
459	IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
460	UNUSED(st);
461#if LJ_32 && LJ_HASFFI
462	if (st == IRT_I64 || st == IRT_U64) {
463	  hi = split_call_l(J, hisubst, oir, ir,
464		 st == IRT_I64 ? IRCALL_fp64_l2d : IRCALL_fp64_ul2d);
465	  break;
466	}
467#endif
468	lj_assertJ(st == IRT_INT ||
469		   (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)),
470		   "bad source type for CONV");
471	nir->o = IR_CALLN;
472#if LJ_32 && LJ_HASFFI
473	nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d :
474		   st == IRT_FLOAT ? IRCALL_softfp_f2d :
475		   IRCALL_softfp_ui2d;
476#else
477	nir->op2 = IRCALL_softfp_i2d;
478#endif
479	hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
480	break;
481	}
482      case IR_CALLN:
483      case IR_CALLL:
484      case IR_CALLS:
485      case IR_CALLXS:
486	goto split_call;
487      case IR_PHI:
488	if (nir->op1 == nir->op2)
489	  J->cur.nins--;  /* Drop useless PHIs. */
490	if (hisubst[ir->op1] != hisubst[ir->op2])
491	  split_emit(J, IRT(IR_PHI, IRT_SOFTFP),
492		     hisubst[ir->op1], hisubst[ir->op2]);
493	break;
494      case IR_HIOP:
495	J->cur.nins--;  /* Drop joining HIOP. */
496	ir->prev = nir->op1;
497	hi = nir->op2;
498	break;
499      default:
500	lj_assertJ(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX,
501		   "bad IR op %d", ir->o);
502	hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP),
503			hisubst[ir->op1], hisubst[ir->op2]);
504	break;
505      }
506    } else
507#endif
508#if LJ_32 && LJ_HASFFI
509    if (irt_isint64(ir->t)) {
510      IRRef hiref = hisubst[ir->op1];
511      nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
512      switch (ir->o) {
513      case IR_ADD:
514      case IR_SUB:
515	/* Use plain op for hiword if loword cannot produce a carry/borrow. */
516	if (irref_isk(nir->op2) && IR(nir->op2)->i == 0) {
517	  ir->prev = nir->op1;  /* Pass through loword. */
518	  nir->op1 = hiref; nir->op2 = hisubst[ir->op2];
519	  hi = nref;
520	  break;
521	}
522	/* fallthrough */
523      case IR_NEG:
524	hi = split_emit(J, IRTI(IR_HIOP), hiref, hisubst[ir->op2]);
525	break;
526      case IR_MUL:
527	hi = split_call_ll(J, hisubst, oir, ir, IRCALL_lj_carith_mul64);
528	break;
529      case IR_DIV:
530	hi = split_call_ll(J, hisubst, oir, ir,
531			   irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
532					      IRCALL_lj_carith_divu64);
533	break;
534      case IR_MOD:
535	hi = split_call_ll(J, hisubst, oir, ir,
536			   irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
537					      IRCALL_lj_carith_modu64);
538	break;
539      case IR_POW:
540	hi = split_call_ll(J, hisubst, oir, ir,
541			   irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
542					      IRCALL_lj_carith_powu64);
543	break;
544      case IR_BNOT:
545	hi = split_emit(J, IRTI(IR_BNOT), hiref, 0);
546	break;
547      case IR_BSWAP:
548	ir->prev = split_emit(J, IRTI(IR_BSWAP), hiref, 0);
549	hi = nref;
550	break;
551      case IR_BAND: case IR_BOR: case IR_BXOR:
552	hi = split_bitop(J, hisubst, nir, ir);
553	break;
554      case IR_BSHL: case IR_BSHR: case IR_BSAR: case IR_BROL: case IR_BROR:
555	hi = split_bitshift(J, hisubst, oir, nir, ir);
556	break;
557      case IR_FLOAD:
558	lj_assertJ(ir->op2 == IRFL_CDATA_INT64, "only INT64 supported");
559	hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
560#if LJ_BE
561	ir->prev = hi; hi = nref;
562#endif
563	break;
564      case IR_XLOAD:
565	hi = split_emit(J, IRTI(IR_XLOAD), split_ptr(J, oir, ir->op1), ir->op2);
566#if LJ_BE
567	ir->prev = hi; hi = nref;
568#endif
569	break;
570      case IR_XSTORE:
571	split_emit(J, IRTI(IR_HIOP), nir->op1, hisubst[ir->op2]);
572	break;
573      case IR_CONV: {  /* Conversion to 64 bit integer. Others handled below. */
574	IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
575#if LJ_SOFTFP
576	lj_assertJ(st != IRT_FLOAT, "bad CONV *64.float emitted");
577	if (st == IRT_NUM) {  /* NUM to 64 bit int conv. */
578	  hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_num2u64);
579	}
580#else
581	if (st == IRT_NUM || st == IRT_FLOAT) {  /* FP to 64 bit int conv. */
582	  hi = split_emit(J, IRTI(IR_HIOP), nir->op1, nref);
583	}
584#endif
585	else if (st == IRT_I64 || st == IRT_U64) {  /* 64/64 bit cast. */
586	  /* Drop cast, since assembler doesn't care. But fwd both parts. */
587	  hi = hiref;
588	  goto fwdlo;
589	} else if ((ir->op2 & IRCONV_SEXT)) {  /* Sign-extend to 64 bit. */
590	  IRRef k31 = lj_ir_kint(J, 31);
591	  nir = IR(nref);  /* May have been reallocated. */
592	  ir->prev = nir->op1;  /* Pass through loword. */
593	  nir->o = IR_BSAR;  /* hi = bsar(lo, 31). */
594	  nir->op2 = k31;
595	  hi = nref;
596	} else {  /* Zero-extend to 64 bit. */
597	  hi = lj_ir_kint(J, 0);
598	  goto fwdlo;
599	}
600	break;
601	}
602      case IR_CALLXS:
603	goto split_call;
604      case IR_PHI: {
605	IRRef hiref2;
606	if ((irref_isk(nir->op1) && irref_isk(nir->op2)) ||
607	    nir->op1 == nir->op2)
608	  J->cur.nins--;  /* Drop useless PHIs. */
609	hiref2 = hisubst[ir->op2];
610	if (!((irref_isk(hiref) && irref_isk(hiref2)) || hiref == hiref2))
611	  split_emit(J, IRTI(IR_PHI), hiref, hiref2);
612	break;
613	}
614      case IR_HIOP:
615	J->cur.nins--;  /* Drop joining HIOP. */
616	ir->prev = nir->op1;
617	hi = nir->op2;
618	break;
619      default:
620	lj_assertJ(ir->o <= IR_NE, "bad IR op %d", ir->o);  /* Comparisons. */
621	split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]);
622	break;
623      }
624    } else
625#endif
626#if LJ_SOFTFP
627    if (ir->o == IR_SLOAD) {
628      if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from number to int. */
629	nir->op2 &= ~IRSLOAD_CONVERT;
630	if (!(nir->op2 & IRSLOAD_TYPECHECK))
631	  nir->t.irt = IRT_INT;  /* Drop guard. */
632	split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
633	ir->prev = split_num2int(J, nref, nref+1, irt_isguard(ir->t));
634      }
635    } else if (ir->o == IR_TOBIT) {
636      IRRef tmp, op1 = ir->op1;
637      J->cur.nins--;
638#if LJ_LE
639      tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
640#else
641      tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
642#endif
643      ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit);
644    } else if (ir->o == IR_TOSTR || ir->o == IR_TMPREF) {
645      if (hisubst[ir->op1]) {
646	if (irref_isk(ir->op1))
647	  nir->op1 = ir->op1;
648	else
649	  split_emit(J, IRT(IR_HIOP, IRT_NIL), hisubst[ir->op1], nref);
650      }
651    } else if (ir->o == IR_HREF || ir->o == IR_NEWREF) {
652      if (irref_isk(ir->op2) && hisubst[ir->op2])
653	nir->op2 = ir->op2;
654    } else
655#endif
656    if (ir->o == IR_CONV) {  /* See above, too. */
657      IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
658#if LJ_32 && LJ_HASFFI
659      if (st == IRT_I64 || st == IRT_U64) {  /* Conversion from 64 bit int. */
660#if LJ_SOFTFP
661	if (irt_isfloat(ir->t)) {
662	  split_call_l(J, hisubst, oir, ir,
663		       st == IRT_I64 ? IRCALL_fp64_l2f : IRCALL_fp64_ul2f);
664	  J->cur.nins--;  /* Drop unused HIOP. */
665	}
666#else
667	if (irt_isfp(ir->t)) {  /* 64 bit integer to FP conversion. */
668	  ir->prev = split_emit(J, IRT(IR_HIOP, irt_type(ir->t)),
669				hisubst[ir->op1], nref);
670	}
671#endif
672	else {  /* Truncate to lower 32 bits. */
673	fwdlo:
674	  ir->prev = nir->op1;  /* Forward loword. */
675	  /* Replace with NOP to avoid messing up the snapshot logic. */
676	  nir->ot = IRT(IR_NOP, IRT_NIL);
677	  nir->op1 = nir->op2 = 0;
678	}
679      }
680#endif
681#if LJ_SOFTFP && LJ_32 && LJ_HASFFI
682      else if (irt_isfloat(ir->t)) {
683	if (st == IRT_NUM) {
684	  split_call_l(J, hisubst, oir, ir, IRCALL_softfp_d2f);
685	  J->cur.nins--;  /* Drop unused HIOP. */
686	} else {
687	  nir->o = IR_CALLN;
688	  nir->op2 = st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f;
689	}
690      } else if (st == IRT_FLOAT) {
691	lj_assertJ(!irt_isu32(ir->t), "bad CONV u32.fp emitted");
692	nir->o = IR_CALLN;
693	nir->op2 = IRCALL_softfp_f2i;
694      } else
695#endif
696#if LJ_SOFTFP
697      if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) {
698	if (irt_isguard(ir->t)) {
699	  lj_assertJ(st == IRT_NUM && irt_isint(ir->t), "bad CONV types");
700	  J->cur.nins--;
701	  ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1);
702	} else {
703	  split_call_l(J, hisubst, oir, ir,
704#if LJ_32 && LJ_HASFFI
705	    st == IRT_NUM ? IRCALL_softfp_d2i : IRCALL_softfp_f2i
706#else
707	    IRCALL_softfp_d2i
708#endif
709	  );
710	  J->cur.nins--;  /* Drop unused HIOP. */
711	}
712      }
713#endif
714    } else if (ir->o == IR_CALLXS) {
715      IRRef hiref;
716    split_call:
717      hiref = hisubst[ir->op1];
718      if (hiref) {
719	IROpT ot = nir->ot;
720	IRRef op2 = nir->op2;
721	nir->ot = IRT(IR_CARG, IRT_NIL);
722#if LJ_LE
723	nir->op2 = hiref;
724#else
725	nir->op2 = nir->op1; nir->op1 = hiref;
726#endif
727	ir->prev = nref = split_emit(J, ot, nref, op2);
728      }
729      if (LJ_SOFTFP ? irt_is64(ir->t) : irt_isint64(ir->t))
730	hi = split_emit(J,
731	  IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
732	  nref, nref);
733    } else if (ir->o == IR_CARG) {
734      IRRef hiref = hisubst[ir->op1];
735      if (hiref) {
736	IRRef op2 = nir->op2;
737#if LJ_LE
738	nir->op2 = hiref;
739#else
740	nir->op2 = nir->op1; nir->op1 = hiref;
741#endif
742	ir->prev = nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
743	nir = IR(nref);
744      }
745      hiref = hisubst[ir->op2];
746      if (hiref) {
747#if !LJ_TARGET_X86
748	int carg = 0;
749	IRIns *cir;
750	for (cir = IR(nir->op1); cir->o == IR_CARG; cir = IR(cir->op1))
751	  carg++;
752	if ((carg & 1) == 0) {  /* Align 64 bit arguments. */
753	  IRRef op2 = nir->op2;
754	  nir->op2 = REF_NIL;
755	  nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
756	  nir = IR(nref);
757	}
758#endif
759#if LJ_BE
760	{ IRRef tmp = nir->op2; nir->op2 = hiref; hiref = tmp; }
761#endif
762	ir->prev = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, hiref);
763      }
764    } else if (ir->o == IR_CNEWI) {
765      if (hisubst[ir->op2])
766	split_emit(J, IRT(IR_HIOP, IRT_NIL), nref, hisubst[ir->op2]);
767    } else if (ir->o == IR_LOOP) {
768      J->loopref = nref;  /* Needed by assembler. */
769    }
770    hisubst[ref] = hi;  /* Store hiword substitution. */
771  }
772  if (snref == nins) {  /* Substitution for last snapshot. */
773    snap->ref = J->cur.nins;
774    split_subst_snap(J, snap, oir);
775  }
776
777  /* Add PHI marks. */
778  for (ref = J->cur.nins-1; ref >= REF_FIRST; ref--) {
779    IRIns *ir = IR(ref);
780    if (ir->o != IR_PHI) break;
781    if (!irref_isk(ir->op1)) irt_setphi(IR(ir->op1)->t);
782    if (ir->op2 > J->loopref) irt_setphi(IR(ir->op2)->t);
783  }
784}
785
786/* Protected callback for split pass. */
787static TValue *cpsplit(lua_State *L, lua_CFunction dummy, void *ud)
788{
789  jit_State *J = (jit_State *)ud;
790  split_ir(J);
791  UNUSED(L); UNUSED(dummy);
792  return NULL;
793}
794
795#if defined(LUA_USE_ASSERT) || LJ_SOFTFP
796/* Slow, but sure way to check whether a SPLIT pass is needed. */
797static int split_needsplit(jit_State *J)
798{
799  IRIns *ir, *irend;
800  IRRef ref;
801  for (ir = IR(REF_FIRST), irend = IR(J->cur.nins); ir < irend; ir++)
802    if (LJ_SOFTFP ? irt_is64orfp(ir->t) : irt_isint64(ir->t))
803      return 1;
804  if (LJ_SOFTFP) {
805    for (ref = J->chain[IR_SLOAD]; ref; ref = IR(ref)->prev)
806      if ((IR(ref)->op2 & IRSLOAD_CONVERT))
807	return 1;
808    if (J->chain[IR_TOBIT])
809      return 1;
810  }
811  for (ref = J->chain[IR_CONV]; ref; ref = IR(ref)->prev) {
812    IRType st = (IR(ref)->op2 & IRCONV_SRCMASK);
813    if ((LJ_SOFTFP && (st == IRT_NUM || st == IRT_FLOAT)) ||
814	st == IRT_I64 || st == IRT_U64)
815      return 1;
816  }
817  return 0;  /* Nope. */
818}
819#endif
820
821/* SPLIT pass. */
822void lj_opt_split(jit_State *J)
823{
824#if LJ_SOFTFP
825  if (!J->needsplit)
826    J->needsplit = split_needsplit(J);
827#else
828  lj_assertJ(J->needsplit >= split_needsplit(J), "bad SPLIT state");
829#endif
830  if (J->needsplit) {
831    int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit);
832    if (errcode) {
833      /* Completely reset the trace to avoid inconsistent dump on abort. */
834      J->cur.nins = J->cur.nk = REF_BASE;
835      J->cur.nsnap = 0;
836      lj_err_throw(J->L, errcode);  /* Propagate errors. */
837    }
838  }
839}
840
841#undef IR
842
843#endif