luna - luajit/src/vm

   1|// Low-level VM code for PowerPC 32 bit or 32on64 bit mode.
   2|// Bytecode interpreter, fast functions and helper functions.
   3|// Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4|
   5|.arch ppc
   6|.section code_op, code_sub
   7|
   8|.actionlist build_actionlist
   9|.globals GLOB_
  10|.globalnames globnames
  11|.externnames extnames
  12|
  13|// Note: The ragged indentation of the instructions is intentional.
  14|//       The starting columns indicate data dependencies.
  15|
  16|//-----------------------------------------------------------------------
  17|
  18|// DynASM defines used by the PPC port:
  19|//
  20|// P64     64 bit pointers (only for GPR64 testing).
  21|// GPR64   64 bit registers (but possibly 32 bit pointers, e.g. PS3).
  22|//         Affects reg saves, stack layout, carry/overflow/dot flags etc.
  23|// FRAME32 Use 32 bit frame layout, even with GPR64 (Xbox 360).
  24|// TOC     Need table of contents (64 bit or 32 bit variant, e.g. PS3).
  25|//         Function pointers are really a struct: code, TOC, env (optional).
  26|// TOCENV  Function pointers have an environment pointer, too (not on PS3).
  27|// PPE     Power Processor Element of Cell (PS3) or Xenon (Xbox 360).
  28|//         Must avoid (slow) micro-coded instructions.
  29|
  30|.if P64
  31|.define TOC, 1
  32|.define TOCENV, 1
  33|.macro lpx, a, b, c; ldx a, b, c; .endmacro
  34|.macro lp, a, b; ld a, b; .endmacro
  35|.macro stp, a, b; std a, b; .endmacro
  36|.define decode_OPP, decode_OP8
  37|.if FFI
  38|// Missing: Calling conventions, 64 bit regs, TOC.
  39|.error lib_ffi not yet implemented for PPC64
  40|.endif
  41|.else
  42|.macro lpx, a, b, c; lwzx a, b, c; .endmacro
  43|.macro lp, a, b; lwz a, b; .endmacro
  44|.macro stp, a, b; stw a, b; .endmacro
  45|.define decode_OPP, decode_OP4
  46|.endif
  47|
  48|// Convenience macros for TOC handling.
  49|.if TOC
  50|// Linker needs a TOC patch area for every external call relocation.
  51|.macro blex, target; bl extern target@plt; nop; .endmacro
  52|.macro .toc, a, b; a, b; .endmacro
  53|.if P64
  54|.define TOC_OFS,	 8
  55|.define ENV_OFS,	16
  56|.else
  57|.define TOC_OFS,	4
  58|.define ENV_OFS,	8
  59|.endif
  60|.else  // No TOC.
  61|.macro blex, target; bl extern target@plt; .endmacro
  62|.macro .toc, a, b; .endmacro
  63|.endif
  64|.macro .tocenv, a, b; .if TOCENV; a, b; .endif; .endmacro
  65|
  66|.macro .gpr64, a, b; .if GPR64; a, b; .endif; .endmacro
  67|
  68|.macro andix., y, a, i
  69|.if PPE
  70|  rlwinm y, a, 0, 31-lj_fls(i), 31-lj_ffs(i)
  71|  cmpwi y, 0
  72|.else
  73|  andi. y, a, i
  74|.endif
  75|.endmacro
  76|
  77|.macro clrso, reg
  78|.if PPE
  79|  li reg, 0
  80|  mtxer reg
  81|.else
  82|  mcrxr cr0
  83|.endif
  84|.endmacro
  85|
  86|.macro checkov, reg, noov
  87|.if PPE
  88|  mfxer reg
  89|  add reg, reg, reg
  90|  cmpwi reg, 0
  91|   li reg, 0
  92|   mtxer reg
  93|  bgey noov
  94|.else
  95|  mcrxr cr0
  96|  bley noov
  97|.endif
  98|.endmacro
  99|
 100|//-----------------------------------------------------------------------
 101|
 102|// Fixed register assignments for the interpreter.
 103|// Don't use: r1 = sp, r2 and r13 = reserved (TOC, TLS or SDATA)
 104|
 105|.macro .FPU, a, b
 106|.if FPU
 107|  a, b
 108|.endif
 109|.endmacro
 110|
 111|.macro .FPU, a, b, c
 112|.if FPU
 113|  a, b, c
 114|.endif
 115|.endmacro
 116|
 117|// The following must be C callee-save (but BASE is often refetched).
 118|.define BASE,		r14	// Base of current Lua stack frame.
 119|.define KBASE,		r15	// Constants of current Lua function.
 120|.define PC,		r16	// Next PC.
 121|.define DISPATCH,	r17	// Opcode dispatch table.
 122|.define LREG,		r18	// Register holding lua_State (also in SAVE_L).
 123|.define MULTRES,	r19	// Size of multi-result: (nresults+1)*8.
 124|.define JGL,		r31	// On-trace: global_State + 32768.
 125|
 126|// Constants for type-comparisons, stores and conversions. C callee-save.
 127|.define TISNUM,	r22
 128|.define TISNIL,	r23
 129|.define ZERO,		r24
 130|.if FPU
 131|.define TOBIT,		f30	// 2^52 + 2^51.
 132|.define TONUM,		f31	// 2^52 + 2^51 + 2^31.
 133|.endif
 134|
 135|// The following temporaries are not saved across C calls, except for RA.
 136|.define RA,		r20	// Callee-save.
 137|.define RB,		r10
 138|.define RC,		r11
 139|.define RD,		r12
 140|.define INS,		r7	// Overlaps CARG5.
 141|
 142|.define TMP0,		r0
 143|.define TMP1,		r8
 144|.define TMP2,		r9
 145|.define TMP3,		r6	// Overlaps CARG4.
 146|
 147|// Saved temporaries.
 148|.define SAVE0,		r21
 149|.define SAVE1,		r25
 150|
 151|// Calling conventions.
 152|.define CARG1,		r3
 153|.define CARG2,		r4
 154|.define CARG3,		r5
 155|.define CARG4,		r6	// Overlaps TMP3.
 156|.define CARG5,		r7	// Overlaps INS.
 157|
 158|.if FPU
 159|.define FARG1,		f1
 160|.define FARG2,		f2
 161|.endif
 162|
 163|.define CRET1,		r3
 164|.define CRET2,		r4
 165|
 166|.define TOCREG,	r2	// TOC register (only used by C code).
 167|.define ENVREG,	r11	// Environment pointer (nested C functions).
 168|
 169|// Stack layout while in interpreter. Must match with lj_frame.h.
 170|.if GPR64
 171|.if FRAME32
 172|
 173|//			456(sp) // \ 32/64 bit C frame info
 174|.define TONUM_LO,	452(sp) // |
 175|.define TONUM_HI,	448(sp) // |
 176|.define TMPD_LO,	444(sp) // |
 177|.define TMPD_HI,	440(sp) // |
 178|.define SAVE_CR,	432(sp) // | 64 bit CR save.
 179|.define SAVE_ERRF,	424(sp) //  > Parameter save area.
 180|.define SAVE_NRES,	420(sp) // |
 181|.define SAVE_L,	416(sp) // |
 182|.define SAVE_PC,	412(sp) // |
 183|.define SAVE_MULTRES,	408(sp) // |
 184|.define SAVE_CFRAME,	400(sp) // / 64 bit C frame chain.
 185|//			392(sp) // Reserved.
 186|.define CFRAME_SPACE,	384     // Delta for sp.
 187|// Back chain for sp:	384(sp) <-- sp entering interpreter
 188|.define SAVE_LR,	376(sp) // 32 bit LR stored in hi-part.
 189|.define SAVE_GPR_,	232     // .. 232+18*8: 64 bit GPR saves.
 190|.define SAVE_FPR_,	88      // .. 88+18*8: 64 bit FPR saves.
 191|//			80(sp) // Needed for 16 byte stack frame alignment.
 192|//			16(sp)  // Callee parameter save area (ABI mandated).
 193|//			8(sp)   // Reserved
 194|// Back chain for sp:	0(sp)   <-- sp while in interpreter
 195|// 32 bit sp stored in hi-part of 0(sp).
 196|
 197|.define TMPD_BLO,	447(sp)
 198|.define TMPD,		TMPD_HI
 199|.define TONUM_D,	TONUM_HI
 200|
 201|.else
 202|
 203|//			508(sp) // \ 32 bit C frame info.
 204|.define SAVE_ERRF,	472(sp) // |
 205|.define SAVE_NRES,	468(sp) // |
 206|.define SAVE_L,	464(sp) //  > Parameter save area.
 207|.define SAVE_PC,	460(sp) // |
 208|.define SAVE_MULTRES,	456(sp) // |
 209|.define SAVE_CFRAME,	448(sp) // / 64 bit C frame chain.
 210|.define SAVE_LR,	416(sp)
 211|.define CFRAME_SPACE,	400     // Delta for sp.
 212|// Back chain for sp:	400(sp) <-- sp entering interpreter
 213|.define SAVE_FPR_,	256     // .. 256+18*8: 64 bit FPR saves.
 214|.define SAVE_GPR_,	112     // .. 112+18*8: 64 bit GPR saves.
 215|//			48(sp)  // Callee parameter save area (ABI mandated).
 216|.define SAVE_TOC,	40(sp)  // TOC save area.
 217|.define TMPD_LO,	36(sp)  // \ Link editor temp (ABI mandated).
 218|.define TMPD_HI,	32(sp)  // /
 219|.define TONUM_LO,	28(sp)  // \ Compiler temp (ABI mandated).
 220|.define TONUM_HI,	24(sp)  // /
 221|// Next frame lr:	16(sp)
 222|.define SAVE_CR,	8(sp)  // 64 bit CR save.
 223|// Back chain for sp:	0(sp)	<-- sp while in interpreter
 224|
 225|.define TMPD_BLO,	39(sp)
 226|.define TMPD,		TMPD_HI
 227|.define TONUM_D,	TONUM_HI
 228|
 229|.endif
 230|.else
 231|
 232|.if FPU
 233|.define SAVE_LR,	276(sp)
 234|.define CFRAME_SPACE,	272     // Delta for sp.
 235|// Back chain for sp:	272(sp) <-- sp entering interpreter
 236|.define SAVE_FPR_,	128     // .. 128+18*8: 64 bit FPR saves.
 237|.else
 238|.define SAVE_LR,	132(sp)
 239|.define CFRAME_SPACE,	128     // Delta for sp.
 240|// Back chain for sp:	128(sp) <-- sp entering interpreter
 241|.endif
 242|.define SAVE_GPR_,	56      // .. 56+18*4: 32 bit GPR saves.
 243|.define SAVE_CR,	52(sp)  // 32 bit CR save.
 244|.define SAVE_ERRF,	48(sp)  // 32 bit C frame info.
 245|.define SAVE_NRES,	44(sp)
 246|.define SAVE_CFRAME,	40(sp)
 247|.define SAVE_L,	36(sp)
 248|.define SAVE_PC,	32(sp)
 249|.define SAVE_MULTRES,	28(sp)
 250|.define UNUSED1,	24(sp)
 251|.if FPU
 252|.define TMPD_LO,	20(sp)
 253|.define TMPD_HI,	16(sp)
 254|.define TONUM_LO,	12(sp)
 255|.define TONUM_HI,	8(sp)
 256|.else
 257|.define SFSAVE_4,	20(sp)
 258|.define SFSAVE_3,	16(sp)
 259|.define SFSAVE_2,	12(sp)
 260|.define SFSAVE_1,	8(sp)
 261|.endif
 262|// Next frame lr:	4(sp)
 263|// Back chain for sp:	0(sp)	<-- sp while in interpreter
 264|
 265|.if FPU
 266|.define TMPD_BLO,	23(sp)
 267|.define TMPD,		TMPD_HI
 268|.define TONUM_D,	TONUM_HI
 269|.endif
 270|
 271|.endif
 272|
 273|.macro save_, reg
 274|.if GPR64
 275|  std r..reg, SAVE_GPR_+(reg-14)*8(sp)
 276|.else
 277|  stw r..reg, SAVE_GPR_+(reg-14)*4(sp)
 278|.endif
 279|  .FPU stfd f..reg, SAVE_FPR_+(reg-14)*8(sp)
 280|.endmacro
 281|.macro rest_, reg
 282|.if GPR64
 283|  ld r..reg, SAVE_GPR_+(reg-14)*8(sp)
 284|.else
 285|  lwz r..reg, SAVE_GPR_+(reg-14)*4(sp)
 286|.endif
 287|  .FPU lfd f..reg, SAVE_FPR_+(reg-14)*8(sp)
 288|.endmacro
 289|
 290|.macro saveregs
 291|.if GPR64 and not FRAME32
 292|  stdu sp, -CFRAME_SPACE(sp)
 293|.else
 294|  stwu sp, -CFRAME_SPACE(sp)
 295|.endif
 296|  save_ 14; save_ 15; save_ 16
 297|  mflr r0
 298|  save_ 17; save_ 18; save_ 19; save_ 20; save_ 21; save_ 22
 299|.if GPR64 and not FRAME32
 300|  std r0, SAVE_LR
 301|.else
 302|  stw r0, SAVE_LR
 303|.endif
 304|  save_ 23; save_ 24; save_ 25
 305|  mfcr r0
 306|  save_ 26; save_ 27; save_ 28; save_ 29; save_ 30; save_ 31
 307|.if GPR64
 308|  std r0, SAVE_CR
 309|.else
 310|  stw r0, SAVE_CR
 311|.endif
 312|  .toc std TOCREG, SAVE_TOC
 313|.endmacro
 314|
 315|.macro restoreregs
 316|.if GPR64 and not FRAME32
 317|  ld r0, SAVE_LR
 318|.else
 319|  lwz r0, SAVE_LR
 320|.endif
 321|.if GPR64
 322|  ld r12, SAVE_CR
 323|.else
 324|  lwz r12, SAVE_CR
 325|.endif
 326|  rest_ 14; rest_ 15; rest_ 16; rest_ 17; rest_ 18; rest_ 19
 327|  mtlr r0;
 328|.if PPE; mtocrf 0x20, r12; .else; mtcrf 0x38, r12; .endif
 329|  rest_ 20; rest_ 21; rest_ 22; rest_ 23; rest_ 24; rest_ 25
 330|.if PPE; mtocrf 0x10, r12; .endif
 331|  rest_ 26; rest_ 27; rest_ 28; rest_ 29; rest_ 30; rest_ 31
 332|.if PPE; mtocrf 0x08, r12; .endif
 333|  addi sp, sp, CFRAME_SPACE
 334|.endmacro
 335|
 336|// Type definitions. Some of these are only used for documentation.
 337|.type L,		lua_State,	LREG
 338|.type GL,		global_State
 339|.type TVALUE,		TValue
 340|.type GCOBJ,		GCobj
 341|.type STR,		GCstr
 342|.type TAB,		GCtab
 343|.type LFUNC,		GCfuncL
 344|.type CFUNC,		GCfuncC
 345|.type PROTO,		GCproto
 346|.type UPVAL,		GCupval
 347|.type NODE,		Node
 348|.type NARGS8,		int
 349|.type TRACE,		GCtrace
 350|.type SBUF,		SBuf
 351|
 352|//-----------------------------------------------------------------------
 353|
 354|// Trap for not-yet-implemented parts.
 355|.macro NYI; tw 4, sp, sp; .endmacro
 356|
 357|.if FPU
 358|// int/FP conversions.
 359|.macro tonum_i, freg, reg
 360|  xoris reg, reg, 0x8000
 361|  stw reg, TONUM_LO
 362|  lfd freg, TONUM_D
 363|  fsub freg, freg, TONUM
 364|.endmacro
 365|
 366|.macro tonum_u, freg, reg
 367|  stw reg, TONUM_LO
 368|  lfd freg, TONUM_D
 369|  fsub freg, freg, TOBIT
 370|.endmacro
 371|
 372|.macro toint, reg, freg, tmpfreg
 373|  fctiwz tmpfreg, freg
 374|  stfd tmpfreg, TMPD
 375|  lwz reg, TMPD_LO
 376|.endmacro
 377|
 378|.macro toint, reg, freg
 379|  toint reg, freg, freg
 380|.endmacro
 381|.endif
 382|
 383|//-----------------------------------------------------------------------
 384|
 385|// Access to frame relative to BASE.
 386|.define FRAME_PC,	-8
 387|.define FRAME_FUNC,	-4
 388|
 389|// Instruction decode.
 390|.macro decode_OP4, dst, ins; rlwinm dst, ins, 2, 22, 29; .endmacro
 391|.macro decode_OP8, dst, ins; rlwinm dst, ins, 3, 21, 28; .endmacro
 392|.macro decode_RA8, dst, ins; rlwinm dst, ins, 27, 21, 28; .endmacro
 393|.macro decode_RB8, dst, ins; rlwinm dst, ins, 11, 21, 28; .endmacro
 394|.macro decode_RC8, dst, ins; rlwinm dst, ins, 19, 21, 28; .endmacro
 395|.macro decode_RD8, dst, ins; rlwinm dst, ins, 19, 13, 28; .endmacro
 396|
 397|.macro decode_OP1, dst, ins; rlwinm dst, ins, 0, 24, 31; .endmacro
 398|.macro decode_RD4, dst, ins; rlwinm dst, ins, 18, 14, 29; .endmacro
 399|
 400|// Instruction fetch.
 401|.macro ins_NEXT1
 402|  lwz INS, 0(PC)
 403|   addi PC, PC, 4
 404|.endmacro
 405|// Instruction decode+dispatch. Note: optimized for e300!
 406|.macro ins_NEXT2
 407|  decode_OPP TMP1, INS
 408|  lpx TMP0, DISPATCH, TMP1
 409|  mtctr TMP0
 410|   decode_RB8 RB, INS
 411|   decode_RD8 RD, INS
 412|   decode_RA8 RA, INS
 413|   decode_RC8 RC, INS
 414|  bctr
 415|.endmacro
 416|.macro ins_NEXT
 417|  ins_NEXT1
 418|  ins_NEXT2
 419|.endmacro
 420|
 421|// Instruction footer.
 422|.if 1
 423|  // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
 424|  .define ins_next, ins_NEXT
 425|  .define ins_next_, ins_NEXT
 426|  .define ins_next1, ins_NEXT1
 427|  .define ins_next2, ins_NEXT2
 428|.else
 429|  // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
 430|  // Affects only certain kinds of benchmarks (and only with -j off).
 431|  .macro ins_next
 432|    b ->ins_next
 433|  .endmacro
 434|  .macro ins_next1
 435|  .endmacro
 436|  .macro ins_next2
 437|    b ->ins_next
 438|  .endmacro
 439|  .macro ins_next_
 440|  ->ins_next:
 441|    ins_NEXT
 442|  .endmacro
 443|.endif
 444|
 445|// Call decode and dispatch.
 446|.macro ins_callt
 447|  // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
 448|  lwz PC, LFUNC:RB->pc
 449|  lwz INS, 0(PC)
 450|   addi PC, PC, 4
 451|  decode_OPP TMP1, INS
 452|   decode_RA8 RA, INS
 453|  lpx TMP0, DISPATCH, TMP1
 454|   add RA, RA, BASE
 455|  mtctr TMP0
 456|  bctr
 457|.endmacro
 458|
 459|.macro ins_call
 460|  // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
 461|  stw PC, FRAME_PC(BASE)
 462|  ins_callt
 463|.endmacro
 464|
 465|//-----------------------------------------------------------------------
 466|
 467|// Macros to test operand types.
 468|.macro checknum, reg; cmplw reg, TISNUM; .endmacro
 469|.macro checknum, cr, reg; cmplw cr, reg, TISNUM; .endmacro
 470|.macro checkstr, reg; cmpwi reg, LJ_TSTR; .endmacro
 471|.macro checktab, reg; cmpwi reg, LJ_TTAB; .endmacro
 472|.macro checkfunc, reg; cmpwi reg, LJ_TFUNC; .endmacro
 473|.macro checknil, reg; cmpwi reg, LJ_TNIL; .endmacro
 474|
 475|.macro branch_RD
 476|  srwi TMP0, RD, 1
 477|  addis PC, PC, -(BCBIAS_J*4 >> 16)
 478|  add PC, PC, TMP0
 479|.endmacro
 480|
 481|// Assumes DISPATCH is relative to GL.
 482#define DISPATCH_GL(field)	(GG_DISP2G + (int)offsetof(global_State, field))
 483#define DISPATCH_J(field)	(GG_DISP2J + (int)offsetof(jit_State, field))
 484|
 485#define PC2PROTO(field)  ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
 486|
 487|.macro hotcheck, delta, target
 488|  rlwinm TMP1, PC, 31, 25, 30
 489|  addi TMP1, TMP1, GG_DISP2HOT
 490|  lhzx TMP2, DISPATCH, TMP1
 491|  addic. TMP2, TMP2, -delta
 492|  sthx TMP2, DISPATCH, TMP1
 493|  blt target
 494|.endmacro
 495|
 496|.macro hotloop
 497|  hotcheck HOTCOUNT_LOOP, ->vm_hotloop
 498|.endmacro
 499|
 500|.macro hotcall
 501|  hotcheck HOTCOUNT_CALL, ->vm_hotcall
 502|.endmacro
 503|
 504|// Set current VM state. Uses TMP0.
 505|.macro li_vmstate, st; li TMP0, ~LJ_VMST_..st; .endmacro
 506|.macro st_vmstate; stw TMP0, DISPATCH_GL(vmstate)(DISPATCH); .endmacro
 507|
 508|// Move table write barrier back. Overwrites mark and tmp.
 509|.macro barrierback, tab, mark, tmp
 510|  lwz tmp, DISPATCH_GL(gc.grayagain)(DISPATCH)
 511|  // Assumes LJ_GC_BLACK is 0x04.
 512|   rlwinm mark, mark, 0, 30, 28		// black2gray(tab)
 513|  stw tab, DISPATCH_GL(gc.grayagain)(DISPATCH)
 514|   stb mark, tab->marked
 515|  stw tmp, tab->gclist
 516|.endmacro
 517|
 518|//-----------------------------------------------------------------------
 519
 520/* Generate subroutines used by opcodes and other parts of the VM. */
 521/* The .code_sub section should be last to help static branch prediction. */
 522static void build_subroutines(BuildCtx *ctx)
 523{
 524  |.code_sub
 525  |
 526  |//-----------------------------------------------------------------------
 527  |//-- Return handling ----------------------------------------------------
 528  |//-----------------------------------------------------------------------
 529  |
 530  |->vm_returnp:
 531  |  // See vm_return. Also: TMP2 = previous base.
 532  |  andix. TMP0, PC, FRAME_P
 533  |   li TMP1, LJ_TTRUE
 534  |  beq ->cont_dispatch
 535  |
 536  |  // Return from pcall or xpcall fast func.
 537  |  lwz PC, FRAME_PC(TMP2)		// Fetch PC of previous frame.
 538  |  mr BASE, TMP2			// Restore caller base.
 539  |  // Prepending may overwrite the pcall frame, so do it at the end.
 540  |   stwu TMP1, FRAME_PC(RA)		// Prepend true to results.
 541  |
 542  |->vm_returnc:
 543  |  addi RD, RD, 8			// RD = (nresults+1)*8.
 544  |   andix. TMP0, PC, FRAME_TYPE
 545  |  cmpwi cr1, RD, 0
 546  |  li CRET1, LUA_YIELD
 547  |  beq cr1, ->vm_unwind_c_eh
 548  |  mr MULTRES, RD
 549  |   beq ->BC_RET_Z			// Handle regular return to Lua.
 550  |
 551  |->vm_return:
 552  |  // BASE = base, RA = resultptr, RD/MULTRES = (nresults+1)*8, PC = return
 553  |  // TMP0 = PC & FRAME_TYPE
 554  |  cmpwi TMP0, FRAME_C
 555  |   rlwinm TMP2, PC, 0, 0, 28
 556  |    li_vmstate C
 557  |   sub TMP2, BASE, TMP2		// TMP2 = previous base.
 558  |  bney ->vm_returnp
 559  |
 560  |  addic. TMP1, RD, -8
 561  |   stp TMP2, L->base
 562  |   lwz TMP2, SAVE_NRES
 563  |    subi BASE, BASE, 8
 564  |    st_vmstate
 565  |   slwi TMP2, TMP2, 3
 566  |  beq >2
 567  |1:
 568  |  addic. TMP1, TMP1, -8
 569  |.if FPU
 570  |   lfd f0, 0(RA)
 571  |.else
 572  |   lwz CARG1, 0(RA)
 573  |   lwz CARG2, 4(RA)
 574  |.endif
 575  |    addi RA, RA, 8
 576  |.if FPU
 577  |   stfd f0, 0(BASE)
 578  |.else
 579  |   stw CARG1, 0(BASE)
 580  |   stw CARG2, 4(BASE)
 581  |.endif
 582  |    addi BASE, BASE, 8
 583  |  bney <1
 584  |
 585  |2:
 586  |  cmpw TMP2, RD			// More/less results wanted?
 587  |  bne >6
 588  |3:
 589  |  stp BASE, L->top			// Store new top.
 590  |
 591  |->vm_leave_cp:
 592  |  lp TMP0, SAVE_CFRAME		// Restore previous C frame.
 593  |   li CRET1, 0			// Ok return status for vm_pcall.
 594  |  stp TMP0, L->cframe
 595  |
 596  |->vm_leave_unw:
 597  |  restoreregs
 598  |  blr
 599  |
 600  |6:
 601  |  ble >7				// Less results wanted?
 602  |  // More results wanted. Check stack size and fill up results with nil.
 603  |  lwz TMP1, L->maxstack
 604  |  cmplw BASE, TMP1
 605  |  bge >8
 606  |  stw TISNIL, 0(BASE)
 607  |  addi RD, RD, 8
 608  |  addi BASE, BASE, 8
 609  |  b <2
 610  |
 611  |7:  // Less results wanted.
 612  |  subfic TMP3, TMP2, 0		// LUA_MULTRET+1 case?
 613  |   sub TMP0, RD, TMP2
 614  |  subfe TMP1, TMP1, TMP1		// TMP1 = TMP2 == 0 ? 0 : -1
 615  |   and TMP0, TMP0, TMP1
 616  |  sub BASE, BASE, TMP0		// Either keep top or shrink it.
 617  |  b <3
 618  |
 619  |8:  // Corner case: need to grow stack for filling up results.
 620  |  // This can happen if:
 621  |  // - A C function grows the stack (a lot).
 622  |  // - The GC shrinks the stack in between.
 623  |  // - A return back from a lua_call() with (high) nresults adjustment.
 624  |  stp BASE, L->top			// Save current top held in BASE (yes).
 625  |   mr SAVE0, RD
 626  |  srwi CARG2, TMP2, 3
 627  |  mr CARG1, L
 628  |  bl extern lj_state_growstack	// (lua_State *L, int n)
 629  |    lwz TMP2, SAVE_NRES
 630  |   mr RD, SAVE0
 631  |    slwi TMP2, TMP2, 3
 632  |  lp BASE, L->top			// Need the (realloced) L->top in BASE.
 633  |  b <2
 634  |
 635  |->vm_unwind_c:			// Unwind C stack, return from vm_pcall.
 636  |  // (void *cframe, int errcode)
 637  |  mr sp, CARG1
 638  |  mr CRET1, CARG2
 639  |->vm_unwind_c_eh:			// Landing pad for external unwinder.
 640  |  lwz L, SAVE_L
 641  |  .toc ld TOCREG, SAVE_TOC
 642  |   li TMP0, ~LJ_VMST_C
 643  |  lwz GL:TMP1, L->glref
 644  |   stw TMP0, GL:TMP1->vmstate
 645  |  b ->vm_leave_unw
 646  |
 647  |->vm_unwind_ff:			// Unwind C stack, return from ff pcall.
 648  |  // (void *cframe)
 649  |.if GPR64
 650  |  rldicr sp, CARG1, 0, 61
 651  |.else
 652  |  rlwinm sp, CARG1, 0, 0, 29
 653  |.endif
 654  |->vm_unwind_ff_eh:			// Landing pad for external unwinder.
 655  |  lwz L, SAVE_L
 656  |  .toc ld TOCREG, SAVE_TOC
 657  |     li TISNUM, LJ_TISNUM		// Setup type comparison constants.
 658  |  lp BASE, L->base
 659  |     .FPU lus TMP3, 0x59c0		// TOBIT = 2^52 + 2^51 (float).
 660  |   lwz DISPATCH, L->glref		// Setup pointer to dispatch table.
 661  |     li ZERO, 0
 662  |     .FPU stw TMP3, TMPD
 663  |  li TMP1, LJ_TFALSE
 664  |     .FPU ori TMP3, TMP3, 0x0004	// TONUM = 2^52 + 2^51 + 2^31 (float).
 665  |     li TISNIL, LJ_TNIL
 666  |    li_vmstate INTERP
 667  |     .FPU lfs TOBIT, TMPD
 668  |  lwz PC, FRAME_PC(BASE)		// Fetch PC of previous frame.
 669  |  la RA, -8(BASE)			// Results start at BASE-8.
 670  |     .FPU stw TMP3, TMPD
 671  |   addi DISPATCH, DISPATCH, GG_G2DISP
 672  |  stw TMP1, 0(RA)			// Prepend false to error message.
 673  |  li RD, 16				// 2 results: false + error message.
 674  |    st_vmstate
 675  |     .FPU lfs TONUM, TMPD
 676  |  b ->vm_returnc
 677  |
 678  |//-----------------------------------------------------------------------
 679  |//-- Grow stack for calls -----------------------------------------------
 680  |//-----------------------------------------------------------------------
 681  |
 682  |->vm_growstack_c:			// Grow stack for C function.
 683  |  li CARG2, LUA_MINSTACK
 684  |  b >2
 685  |
 686  |->vm_growstack_l:			// Grow stack for Lua function.
 687  |  // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
 688  |  add RC, BASE, RC
 689  |   sub RA, RA, BASE
 690  |  stp BASE, L->base
 691  |   addi PC, PC, 4			// Must point after first instruction.
 692  |  stp RC, L->top
 693  |   srwi CARG2, RA, 3
 694  |2:
 695  |  // L->base = new base, L->top = top
 696  |   stw PC, SAVE_PC
 697  |  mr CARG1, L
 698  |  bl extern lj_state_growstack	// (lua_State *L, int n)
 699  |  lp BASE, L->base
 700  |  lp RC, L->top
 701  |  lwz LFUNC:RB, FRAME_FUNC(BASE)
 702  |  sub RC, RC, BASE
 703  |  // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
 704  |  ins_callt				// Just retry the call.
 705  |
 706  |//-----------------------------------------------------------------------
 707  |//-- Entry points into the assembler VM ---------------------------------
 708  |//-----------------------------------------------------------------------
 709  |
 710  |->vm_resume:				// Setup C frame and resume thread.
 711  |  // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
 712  |  saveregs
 713  |  mr L, CARG1
 714  |    lwz DISPATCH, L->glref		// Setup pointer to dispatch table.
 715  |  mr BASE, CARG2
 716  |    lbz TMP1, L->status
 717  |   stw L, SAVE_L
 718  |  li PC, FRAME_CP
 719  |  addi TMP0, sp, CFRAME_RESUME
 720  |    addi DISPATCH, DISPATCH, GG_G2DISP
 721  |   stw CARG3, SAVE_NRES
 722  |    cmplwi TMP1, 0
 723  |   stw CARG3, SAVE_ERRF
 724  |   stp CARG3, SAVE_CFRAME
 725  |   stw CARG1, SAVE_PC		// Any value outside of bytecode is ok.
 726  |  stp TMP0, L->cframe
 727  |    beq >3
 728  |
 729  |  // Resume after yield (like a return).
 730  |  stw L, DISPATCH_GL(cur_L)(DISPATCH)
 731  |  mr RA, BASE
 732  |   lp BASE, L->base
 733  |     li TISNUM, LJ_TISNUM		// Setup type comparison constants.
 734  |   lp TMP1, L->top
 735  |  lwz PC, FRAME_PC(BASE)
 736  |     .FPU lus TMP3, 0x59c0		// TOBIT = 2^52 + 2^51 (float).
 737  |    stb CARG3, L->status
 738  |     .FPU stw TMP3, TMPD
 739  |     .FPU ori TMP3, TMP3, 0x0004	// TONUM = 2^52 + 2^51 + 2^31 (float).
 740  |     .FPU lfs TOBIT, TMPD
 741  |   sub RD, TMP1, BASE
 742  |     .FPU stw TMP3, TMPD
 743  |     .FPU lus TMP0, 0x4338		// Hiword of 2^52 + 2^51 (double)
 744  |   addi RD, RD, 8
 745  |     .FPU stw TMP0, TONUM_HI
 746  |    li_vmstate INTERP
 747  |     li ZERO, 0
 748  |    st_vmstate
 749  |  andix. TMP0, PC, FRAME_TYPE
 750  |   mr MULTRES, RD
 751  |     .FPU lfs TONUM, TMPD
 752  |     li TISNIL, LJ_TNIL
 753  |  beq ->BC_RET_Z
 754  |  b ->vm_return
 755  |
 756  |->vm_pcall:				// Setup protected C frame and enter VM.
 757  |  // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
 758  |  saveregs
 759  |  li PC, FRAME_CP
 760  |  stw CARG4, SAVE_ERRF
 761  |  b >1
 762  |
 763  |->vm_call:				// Setup C frame and enter VM.
 764  |  // (lua_State *L, TValue *base, int nres1)
 765  |  saveregs
 766  |  li PC, FRAME_C
 767  |
 768  |1:  // Entry point for vm_pcall above (PC = ftype).
 769  |  lp TMP1, L:CARG1->cframe
 770  |    mr L, CARG1
 771  |   stw CARG3, SAVE_NRES
 772  |    lwz DISPATCH, L->glref		// Setup pointer to dispatch table.
 773  |   stw CARG1, SAVE_L
 774  |     mr BASE, CARG2
 775  |    addi DISPATCH, DISPATCH, GG_G2DISP
 776  |   stw CARG1, SAVE_PC		// Any value outside of bytecode is ok.
 777  |  stp TMP1, SAVE_CFRAME
 778  |  stp sp, L->cframe			// Add our C frame to cframe chain.
 779  |
 780  |3:  // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
 781  |  stw L, DISPATCH_GL(cur_L)(DISPATCH)
 782  |  lp TMP2, L->base			// TMP2 = old base (used in vmeta_call).
 783  |     li TISNUM, LJ_TISNUM		// Setup type comparison constants.
 784  |   lp TMP1, L->top
 785  |     .FPU lus TMP3, 0x59c0		// TOBIT = 2^52 + 2^51 (float).
 786  |  add PC, PC, BASE
 787  |     .FPU stw TMP3, TMPD
 788  |     li ZERO, 0
 789  |     .FPU ori TMP3, TMP3, 0x0004	// TONUM = 2^52 + 2^51 + 2^31 (float).
 790  |     .FPU lfs TOBIT, TMPD
 791  |  sub PC, PC, TMP2			// PC = frame delta + frame type
 792  |     .FPU stw TMP3, TMPD
 793  |     .FPU lus TMP0, 0x4338		// Hiword of 2^52 + 2^51 (double)
 794  |   sub NARGS8:RC, TMP1, BASE
 795  |     .FPU stw TMP0, TONUM_HI
 796  |    li_vmstate INTERP
 797  |     .FPU lfs TONUM, TMPD
 798  |     li TISNIL, LJ_TNIL
 799  |    st_vmstate
 800  |
 801  |->vm_call_dispatch:
 802  |  // TMP2 = old base, BASE = new base, RC = nargs*8, PC = caller PC
 803  |  lwz TMP0, FRAME_PC(BASE)
 804  |   lwz LFUNC:RB, FRAME_FUNC(BASE)
 805  |  checkfunc TMP0; bne ->vmeta_call
 806  |
 807  |->vm_call_dispatch_f:
 808  |  ins_call
 809  |  // BASE = new base, RB = func, RC = nargs*8, PC = caller PC
 810  |
 811  |->vm_cpcall:				// Setup protected C frame, call C.
 812  |  // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
 813  |  saveregs
 814  |  mr L, CARG1
 815  |   lwz TMP0, L:CARG1->stack
 816  |  stw CARG1, SAVE_L
 817  |   lp TMP1, L->top
 818  |     lwz DISPATCH, L->glref		// Setup pointer to dispatch table.
 819  |  stw CARG1, SAVE_PC			// Any value outside of bytecode is ok.
 820  |   sub TMP0, TMP0, TMP1		// Compute -savestack(L, L->top).
 821  |    lp TMP1, L->cframe
 822  |     addi DISPATCH, DISPATCH, GG_G2DISP
 823  |  .toc lp CARG4, 0(CARG4)
 824  |  li TMP2, 0
 825  |   stw TMP0, SAVE_NRES		// Neg. delta means cframe w/o frame.
 826  |  stw TMP2, SAVE_ERRF		// No error function.
 827  |    stp TMP1, SAVE_CFRAME
 828  |    stp sp, L->cframe		// Add our C frame to cframe chain.
 829  |     stw L, DISPATCH_GL(cur_L)(DISPATCH)
 830  |  mtctr CARG4
 831  |  bctrl			// (lua_State *L, lua_CFunction func, void *ud)
 832  |.if PPE
 833  |  mr BASE, CRET1
 834  |  cmpwi CRET1, 0
 835  |.else
 836  |  mr. BASE, CRET1
 837  |.endif
 838  |   li PC, FRAME_CP
 839  |  bne <3				// Else continue with the call.
 840  |  b ->vm_leave_cp			// No base? Just remove C frame.
 841  |
 842  |//-----------------------------------------------------------------------
 843  |//-- Metamethod handling ------------------------------------------------
 844  |//-----------------------------------------------------------------------
 845  |
 846  |// The lj_meta_* functions (except for lj_meta_cat) don't reallocate the
 847  |// stack, so BASE doesn't need to be reloaded across these calls.
 848  |
 849  |//-- Continuation dispatch ----------------------------------------------
 850  |
 851  |->cont_dispatch:
 852  |  // BASE = meta base, RA = resultptr, RD = (nresults+1)*8
 853  |  lwz TMP0, -12(BASE)		// Continuation.
 854  |   mr RB, BASE
 855  |   mr BASE, TMP2			// Restore caller BASE.
 856  |    lwz LFUNC:TMP1, FRAME_FUNC(TMP2)
 857  |.if FFI
 858  |  cmplwi TMP0, 1
 859  |.endif
 860  |     lwz PC, -16(RB)			// Restore PC from [cont|PC].
 861  |   subi TMP2, RD, 8
 862  |   stwx TISNIL, RA, TMP2		// Ensure one valid arg.
 863  |.if FFI
 864  |  ble >1
 865  |.endif
 866  |    lwz TMP1, LFUNC:TMP1->pc
 867  |    lwz KBASE, PC2PROTO(k)(TMP1)
 868  |  // BASE = base, RA = resultptr, RB = meta base
 869  |  mtctr TMP0
 870  |  bctr				// Jump to continuation.
 871  |
 872  |.if FFI
 873  |1:
 874  |  beq ->cont_ffi_callback		// cont = 1: return from FFI callback.
 875  |  // cont = 0: tailcall from C function.
 876  |  subi TMP1, RB, 16
 877  |  sub RC, TMP1, BASE
 878  |  b ->vm_call_tail
 879  |.endif
 880  |
 881  |->cont_cat:				// RA = resultptr, RB = meta base
 882  |  lwz INS, -4(PC)
 883  |   subi CARG2, RB, 16
 884  |  decode_RB8 SAVE0, INS
 885  |.if FPU
 886  |   lfd f0, 0(RA)
 887  |.else
 888  |   lwz TMP2, 0(RA)
 889  |   lwz TMP3, 4(RA)
 890  |.endif
 891  |  add TMP1, BASE, SAVE0
 892  |   stp BASE, L->base
 893  |  cmplw TMP1, CARG2
 894  |   sub CARG3, CARG2, TMP1
 895  |  decode_RA8 RA, INS
 896  |.if FPU
 897  |   stfd f0, 0(CARG2)
 898  |.else
 899  |   stw TMP2, 0(CARG2)
 900  |   stw TMP3, 4(CARG2)
 901  |.endif
 902  |  bney ->BC_CAT_Z
 903  |.if FPU
 904  |   stfdx f0, BASE, RA
 905  |.else
 906  |   stwux TMP2, RA, BASE
 907  |   stw TMP3, 4(RA)
 908  |.endif
 909  |  b ->cont_nop
 910  |
 911  |//-- Table indexing metamethods -----------------------------------------
 912  |
 913  |->vmeta_tgets1:
 914  |  la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
 915  |  li TMP0, LJ_TSTR
 916  |   decode_RB8 RB, INS
 917  |  stw STR:RC, 4(CARG3)
 918  |   add CARG2, BASE, RB
 919  |  stw TMP0, 0(CARG3)
 920  |  b >1
 921  |
 922  |->vmeta_tgets:
 923  |  la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
 924  |  li TMP0, LJ_TTAB
 925  |  stw TAB:RB, 4(CARG2)
 926  |   la CARG3, DISPATCH_GL(tmptv2)(DISPATCH)
 927  |  stw TMP0, 0(CARG2)
 928  |   li TMP1, LJ_TSTR
 929  |   stw STR:RC, 4(CARG3)
 930  |   stw TMP1, 0(CARG3)
 931  |  b >1
 932  |
 933  |->vmeta_tgetb:			// TMP0 = index
 934  |.if not DUALNUM
 935  |  tonum_u f0, TMP0
 936  |.endif
 937  |   decode_RB8 RB, INS
 938  |  la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
 939  |   add CARG2, BASE, RB
 940  |.if DUALNUM
 941  |  stw TISNUM, 0(CARG3)
 942  |  stw TMP0, 4(CARG3)
 943  |.else
 944  |  stfd f0, 0(CARG3)
 945  |.endif
 946  |  b >1
 947  |
 948  |->vmeta_tgetv:
 949  |  decode_RB8 RB, INS
 950  |   decode_RC8 RC, INS
 951  |  add CARG2, BASE, RB
 952  |   add CARG3, BASE, RC
 953  |1:
 954  |  stp BASE, L->base
 955  |  mr CARG1, L
 956  |  stw PC, SAVE_PC
 957  |  bl extern lj_meta_tget		// (lua_State *L, TValue *o, TValue *k)
 958  |  // Returns TValue * (finished) or NULL (metamethod).
 959  |  cmplwi CRET1, 0
 960  |  beq >3
 961  |.if FPU
 962  |   lfd f0, 0(CRET1)
 963  |.else
 964  |   lwz TMP0, 0(CRET1)
 965  |   lwz TMP1, 4(CRET1)
 966  |.endif
 967  |  ins_next1
 968  |.if FPU
 969  |   stfdx f0, BASE, RA
 970  |.else
 971  |   stwux TMP0, RA, BASE
 972  |   stw TMP1, 4(RA)
 973  |.endif
 974  |  ins_next2
 975  |
 976  |3:  // Call __index metamethod.
 977  |  // BASE = base, L->top = new base, stack = cont/func/t/k
 978  |  subfic TMP1, BASE, FRAME_CONT
 979  |  lp BASE, L->top
 980  |  stw PC, -16(BASE)			// [cont|PC]
 981  |   add PC, TMP1, BASE
 982  |  lwz LFUNC:RB, FRAME_FUNC(BASE)	// Guaranteed to be a function here.
 983  |   li NARGS8:RC, 16			// 2 args for func(t, k).
 984  |  b ->vm_call_dispatch_f
 985  |
 986  |->vmeta_tgetr:
 987  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
 988  |  // Returns cTValue * or NULL.
 989  |  cmplwi CRET1, 0
 990  |  beq >1
 991  |.if FPU
 992  |  lfd f14, 0(CRET1)
 993  |.else
 994  |  lwz SAVE0, 0(CRET1)
 995  |  lwz SAVE1, 4(CRET1)
 996  |.endif
 997  |  b ->BC_TGETR_Z
 998  |1:
 999  |  stwx TISNIL, BASE, RA
1000  |  b ->cont_nop
1001  |
1002  |//-----------------------------------------------------------------------
1003  |
1004  |->vmeta_tsets1:
1005  |  la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
1006  |  li TMP0, LJ_TSTR
1007  |   decode_RB8 RB, INS
1008  |  stw STR:RC, 4(CARG3)
1009  |   add CARG2, BASE, RB
1010  |  stw TMP0, 0(CARG3)
1011  |  b >1
1012  |
1013  |->vmeta_tsets:
1014  |  la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
1015  |  li TMP0, LJ_TTAB
1016  |  stw TAB:RB, 4(CARG2)
1017  |   la CARG3, DISPATCH_GL(tmptv2)(DISPATCH)
1018  |  stw TMP0, 0(CARG2)
1019  |   li TMP1, LJ_TSTR
1020  |   stw STR:RC, 4(CARG3)
1021  |   stw TMP1, 0(CARG3)
1022  |  b >1
1023  |
1024  |->vmeta_tsetb:			// TMP0 = index
1025  |.if not DUALNUM
1026  |  tonum_u f0, TMP0
1027  |.endif
1028  |   decode_RB8 RB, INS
1029  |  la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
1030  |   add CARG2, BASE, RB
1031  |.if DUALNUM
1032  |  stw TISNUM, 0(CARG3)
1033  |  stw TMP0, 4(CARG3)
1034  |.else
1035  |  stfd f0, 0(CARG3)
1036  |.endif
1037  |  b >1
1038  |
1039  |->vmeta_tsetv:
1040  |  decode_RB8 RB, INS
1041  |   decode_RC8 RC, INS
1042  |  add CARG2, BASE, RB
1043  |   add CARG3, BASE, RC
1044  |1:
1045  |  stp BASE, L->base
1046  |  mr CARG1, L
1047  |  stw PC, SAVE_PC
1048  |  bl extern lj_meta_tset		// (lua_State *L, TValue *o, TValue *k)
1049  |  // Returns TValue * (finished) or NULL (metamethod).
1050  |  cmplwi CRET1, 0
1051  |.if FPU
1052  |   lfdx f0, BASE, RA
1053  |.else
1054  |   lwzux TMP2, RA, BASE
1055  |   lwz TMP3, 4(RA)
1056  |.endif
1057  |  beq >3
1058  |  // NOBARRIER: lj_meta_tset ensures the table is not black.
1059  |  ins_next1
1060  |.if FPU
1061  |   stfd f0, 0(CRET1)
1062  |.else
1063  |   stw TMP2, 0(CRET1)
1064  |   stw TMP3, 4(CRET1)
1065  |.endif
1066  |  ins_next2
1067  |
1068  |3:  // Call __newindex metamethod.
1069  |  // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
1070  |  subfic TMP1, BASE, FRAME_CONT
1071  |  lp BASE, L->top
1072  |  stw PC, -16(BASE)			// [cont|PC]
1073  |   add PC, TMP1, BASE
1074  |  lwz LFUNC:RB, FRAME_FUNC(BASE)	// Guaranteed to be a function here.
1075  |   li NARGS8:RC, 24			// 3 args for func(t, k, v)
1076  |.if FPU
1077  |  stfd f0, 16(BASE)			// Copy value to third argument.
1078  |.else
1079  |  stw TMP2, 16(BASE)
1080  |  stw TMP3, 20(BASE)
1081  |.endif
1082  |  b ->vm_call_dispatch_f
1083  |
1084  |->vmeta_tsetr:
1085  |  stp BASE, L->base
1086  |  mr CARG1, L
1087  |  stw PC, SAVE_PC
1088  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
1089  |  // Returns TValue *.
1090  |.if FPU
1091  |  stfd f14, 0(CRET1)
1092  |.else
1093  |  stw SAVE0, 0(CRET1)
1094  |  stw SAVE1, 4(CRET1)
1095  |.endif
1096  |  b ->cont_nop
1097  |
1098  |//-- Comparison metamethods ---------------------------------------------
1099  |
1100  |->vmeta_comp:
1101  |  mr CARG1, L
1102  |   subi PC, PC, 4
1103  |.if DUALNUM
1104  |  mr CARG2, RA
1105  |.else
1106  |  add CARG2, BASE, RA
1107  |.endif
1108  |   stw PC, SAVE_PC
1109  |.if DUALNUM
1110  |  mr CARG3, RD
1111  |.else
1112  |  add CARG3, BASE, RD
1113  |.endif
1114  |   stp BASE, L->base
1115  |  decode_OP1 CARG4, INS
1116  |  bl extern lj_meta_comp  // (lua_State *L, TValue *o1, *o2, int op)
1117  |  // Returns 0/1 or TValue * (metamethod).
1118  |3:
1119  |  cmplwi CRET1, 1
1120  |  bgt ->vmeta_binop
1121  |  subfic CRET1, CRET1, 0
1122  |4:
1123  |  lwz INS, 0(PC)
1124  |   addi PC, PC, 4
1125  |  decode_RD4 TMP2, INS
1126  |  addis TMP2, TMP2, -(BCBIAS_J*4 >> 16)
1127  |  and TMP2, TMP2, CRET1
1128  |  add PC, PC, TMP2
1129  |->cont_nop:
1130  |  ins_next
1131  |
1132  |->cont_ra:				// RA = resultptr
1133  |  lwz INS, -4(PC)
1134  |.if FPU
1135  |   lfd f0, 0(RA)
1136  |.else
1137  |   lwz CARG1, 0(RA)
1138  |   lwz CARG2, 4(RA)
1139  |.endif
1140  |  decode_RA8 TMP1, INS
1141  |.if FPU
1142  |   stfdx f0, BASE, TMP1
1143  |.else
1144  |   stwux CARG1, TMP1, BASE
1145  |   stw CARG2, 4(TMP1)
1146  |.endif
1147  |  b ->cont_nop
1148  |
1149  |->cont_condt:			// RA = resultptr
1150  |  lwz TMP0, 0(RA)
1151  |  .gpr64 extsw TMP0, TMP0
1152  |  subfic TMP0, TMP0, LJ_TTRUE	// Branch if result is true.
1153  |  subfe CRET1, CRET1, CRET1
1154  |  not CRET1, CRET1
1155  |  b <4
1156  |
1157  |->cont_condf:			// RA = resultptr
1158  |  lwz TMP0, 0(RA)
1159  |  .gpr64 extsw TMP0, TMP0
1160  |  subfic TMP0, TMP0, LJ_TTRUE	// Branch if result is false.
1161  |  subfe CRET1, CRET1, CRET1
1162  |  b <4
1163  |
1164  |->vmeta_equal:
1165  |  // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
1166  |  subi PC, PC, 4
1167  |   stp BASE, L->base
1168  |  mr CARG1, L
1169  |   stw PC, SAVE_PC
1170  |  bl extern lj_meta_equal  // (lua_State *L, GCobj *o1, *o2, int ne)
1171  |  // Returns 0/1 or TValue * (metamethod).
1172  |  b <3
1173  |
1174  |->vmeta_equal_cd:
1175  |.if FFI
1176  |  mr CARG2, INS
1177  |  subi PC, PC, 4
1178  |   stp BASE, L->base
1179  |  mr CARG1, L
1180  |   stw PC, SAVE_PC
1181  |  bl extern lj_meta_equal_cd		// (lua_State *L, BCIns op)
1182  |  // Returns 0/1 or TValue * (metamethod).
1183  |  b <3
1184  |.endif
1185  |
1186  |->vmeta_istype:
1187  |  subi PC, PC, 4
1188  |   stp BASE, L->base
1189  |   srwi CARG2, RA, 3
1190  |   mr CARG1, L
1191  |   srwi CARG3, RD, 3
1192  |  stw PC, SAVE_PC
1193  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
1194  |  b ->cont_nop
1195  |
1196  |//-- Arithmetic metamethods ---------------------------------------------
1197  |
1198  |->vmeta_arith_nv:
1199  |  add CARG3, KBASE, RC
1200  |  add CARG4, BASE, RB
1201  |  b >1
1202  |->vmeta_arith_nv2:
1203  |.if DUALNUM
1204  |  mr CARG3, RC
1205  |  mr CARG4, RB
1206  |  b >1
1207  |.endif
1208  |
1209  |->vmeta_unm:
1210  |  mr CARG3, RD
1211  |  mr CARG4, RD
1212  |  b >1
1213  |
1214  |->vmeta_arith_vn:
1215  |  add CARG3, BASE, RB
1216  |  add CARG4, KBASE, RC
1217  |  b >1
1218  |
1219  |->vmeta_arith_vv:
1220  |  add CARG3, BASE, RB
1221  |  add CARG4, BASE, RC
1222  |.if DUALNUM
1223  |  b >1
1224  |.endif
1225  |->vmeta_arith_vn2:
1226  |->vmeta_arith_vv2:
1227  |.if DUALNUM
1228  |  mr CARG3, RB
1229  |  mr CARG4, RC
1230  |.endif
1231  |1:
1232  |  add CARG2, BASE, RA
1233  |   stp BASE, L->base
1234  |  mr CARG1, L
1235  |   stw PC, SAVE_PC
1236  |  decode_OP1 CARG5, INS		// Caveat: CARG5 overlaps INS.
1237  |  bl extern lj_meta_arith  // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
1238  |  // Returns NULL (finished) or TValue * (metamethod).
1239  |  cmplwi CRET1, 0
1240  |  beq ->cont_nop
1241  |
1242  |  // Call metamethod for binary op.
1243  |->vmeta_binop:
1244  |  // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
1245  |  sub TMP1, CRET1, BASE
1246  |   stw PC, -16(CRET1)		// [cont|PC]
1247  |   mr TMP2, BASE
1248  |  addi PC, TMP1, FRAME_CONT
1249  |   mr BASE, CRET1
1250  |  li NARGS8:RC, 16			// 2 args for func(o1, o2).
1251  |  b ->vm_call_dispatch
1252  |
1253  |->vmeta_len:
1254#if LJ_52
1255  |  mr SAVE0, CARG1
1256#endif
1257  |  mr CARG2, RD
1258  |   stp BASE, L->base
1259  |  mr CARG1, L
1260  |   stw PC, SAVE_PC
1261  |  bl extern lj_meta_len		// (lua_State *L, TValue *o)
1262  |  // Returns NULL (retry) or TValue * (metamethod base).
1263#if LJ_52
1264  |  cmplwi CRET1, 0
1265  |  bne ->vmeta_binop			// Binop call for compatibility.
1266  |  mr CARG1, SAVE0
1267  |  b ->BC_LEN_Z
1268#else
1269  |  b ->vmeta_binop			// Binop call for compatibility.
1270#endif
1271  |
1272  |//-- Call metamethod ----------------------------------------------------
1273  |
1274  |->vmeta_call:			// Resolve and call __call metamethod.
1275  |  // TMP2 = old base, BASE = new base, RC = nargs*8
1276  |  mr CARG1, L
1277  |   stp TMP2, L->base			// This is the callers base!
1278  |  subi CARG2, BASE, 8
1279  |   stw PC, SAVE_PC
1280  |  add CARG3, BASE, RC
1281  |   mr SAVE0, NARGS8:RC
1282  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
1283  |  lwz LFUNC:RB, FRAME_FUNC(BASE)	// Guaranteed to be a function here.
1284  |   addi NARGS8:RC, SAVE0, 8		// Got one more argument now.
1285  |  ins_call
1286  |
1287  |->vmeta_callt:			// Resolve __call for BC_CALLT.
1288  |  // BASE = old base, RA = new base, RC = nargs*8
1289  |  mr CARG1, L
1290  |   stp BASE, L->base
1291  |  subi CARG2, RA, 8
1292  |   stw PC, SAVE_PC
1293  |  add CARG3, RA, RC
1294  |   mr SAVE0, NARGS8:RC
1295  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
1296  |  lwz TMP1, FRAME_PC(BASE)
1297  |   addi NARGS8:RC, SAVE0, 8		// Got one more argument now.
1298  |   lwz LFUNC:RB, FRAME_FUNC(RA)	// Guaranteed to be a function here.
1299  |  b ->BC_CALLT_Z
1300  |
1301  |//-- Argument coercion for 'for' statement ------------------------------
1302  |
1303  |->vmeta_for:
1304  |  mr CARG1, L
1305  |   stp BASE, L->base
1306  |  mr CARG2, RA
1307  |   stw PC, SAVE_PC
1308  |  mr SAVE0, INS
1309  |  bl extern lj_meta_for	// (lua_State *L, TValue *base)
1310  |.if JIT
1311  |   decode_OP1 TMP0, SAVE0
1312  |.endif
1313  |  decode_RA8 RA, SAVE0
1314  |.if JIT
1315  |   cmpwi TMP0, BC_JFORI
1316  |.endif
1317  |  decode_RD8 RD, SAVE0
1318  |.if JIT
1319  |   beqy =>BC_JFORI
1320  |.endif
1321  |  b =>BC_FORI
1322  |
1323  |//-----------------------------------------------------------------------
1324  |//-- Fast functions -----------------------------------------------------
1325  |//-----------------------------------------------------------------------
1326  |
1327  |.macro .ffunc, name
1328  |->ff_ .. name:
1329  |.endmacro
1330  |
1331  |.macro .ffunc_1, name
1332  |->ff_ .. name:
1333  |  cmplwi NARGS8:RC, 8
1334  |   lwz CARG3, 0(BASE)
1335  |    lwz CARG1, 4(BASE)
1336  |  blt ->fff_fallback
1337  |.endmacro
1338  |
1339  |.macro .ffunc_2, name
1340  |->ff_ .. name:
1341  |  cmplwi NARGS8:RC, 16
1342  |   lwz CARG3, 0(BASE)
1343  |    lwz CARG4, 8(BASE)
1344  |   lwz CARG1, 4(BASE)
1345  |    lwz CARG2, 12(BASE)
1346  |  blt ->fff_fallback
1347  |.endmacro
1348  |
1349  |.macro .ffunc_n, name
1350  |->ff_ .. name:
1351  |  cmplwi NARGS8:RC, 8
1352  |   lwz CARG1, 0(BASE)
1353  |.if FPU
1354  |    lfd FARG1, 0(BASE)
1355  |.else
1356  |    lwz CARG2, 4(BASE)
1357  |.endif
1358  |  blt ->fff_fallback
1359  |  checknum CARG1; bge ->fff_fallback
1360  |.endmacro
1361  |
1362  |.macro .ffunc_nn, name
1363  |->ff_ .. name:
1364  |  cmplwi NARGS8:RC, 16
1365  |   lwz CARG1, 0(BASE)
1366  |.if FPU
1367  |    lfd FARG1, 0(BASE)
1368  |   lwz CARG3, 8(BASE)
1369  |    lfd FARG2, 8(BASE)
1370  |.else
1371  |    lwz CARG2, 4(BASE)
1372  |   lwz CARG3, 8(BASE)
1373  |    lwz CARG4, 12(BASE)
1374  |.endif
1375  |  blt ->fff_fallback
1376  |  checknum CARG1; bge ->fff_fallback
1377  |  checknum CARG3; bge ->fff_fallback
1378  |.endmacro
1379  |
1380  |// Inlined GC threshold check. Caveat: uses TMP0 and TMP1.
1381  |.macro ffgccheck
1382  |  lwz TMP0, DISPATCH_GL(gc.total)(DISPATCH)
1383  |  lwz TMP1, DISPATCH_GL(gc.threshold)(DISPATCH)
1384  |  cmplw TMP0, TMP1
1385  |  bgel ->fff_gcstep
1386  |.endmacro
1387  |
1388  |//-- Base library: checks -----------------------------------------------
1389  |
1390  |.ffunc_1 assert
1391  |  li TMP1, LJ_TFALSE
1392  |   la RA, -8(BASE)
1393  |  cmplw cr1, CARG3, TMP1
1394  |    lwz PC, FRAME_PC(BASE)
1395  |  bge cr1, ->fff_fallback
1396  |   stw CARG3, 0(RA)
1397  |  addi RD, NARGS8:RC, 8		// Compute (nresults+1)*8.
1398  |  addi TMP1, BASE, 8
1399  |  add TMP2, RA, NARGS8:RC
1400  |   stw CARG1, 4(RA)
1401  |  beq ->fff_res			// Done if exactly 1 argument.
1402  |1:
1403  |  cmplw TMP1, TMP2
1404  |.if FPU
1405  |   lfd f0, 0(TMP1)
1406  |   stfd f0, 0(TMP1)
1407  |.else
1408  |   lwz CARG1, 0(TMP1)
1409  |   lwz CARG2, 4(TMP1)
1410  |   stw CARG1, -8(TMP1)
1411  |   stw CARG2, -4(TMP1)
1412  |.endif
1413  |    addi TMP1, TMP1, 8
1414  |  bney <1
1415  |  b ->fff_res
1416  |
1417  |.ffunc type
1418  |  cmplwi NARGS8:RC, 8
1419  |   lwz CARG1, 0(BASE)
1420  |  blt ->fff_fallback
1421  |  .gpr64 extsw CARG1, CARG1
1422  |  subfc TMP0, TISNUM, CARG1
1423  |  subfe TMP2, CARG1, CARG1
1424  |  orc TMP1, TMP2, TMP0
1425  |  addi TMP1, TMP1, ~LJ_TISNUM+1
1426  |  slwi TMP1, TMP1, 3
1427  |.if FPU
1428  |   la TMP2, CFUNC:RB->upvalue
1429  |  lfdx FARG1, TMP2, TMP1
1430  |.else
1431  |  add TMP1, CFUNC:RB, TMP1
1432  |  lwz CARG1, CFUNC:TMP1->upvalue[0].u32.hi
1433  |  lwz CARG2, CFUNC:TMP1->upvalue[0].u32.lo
1434  |.endif
1435  |  b ->fff_resn
1436  |
1437  |//-- Base library: getters and setters ---------------------------------
1438  |
1439  |.ffunc_1 getmetatable
1440  |  checktab CARG3; bne >6
1441  |1:  // Field metatable must be at same offset for GCtab and GCudata!
1442  |  lwz TAB:CARG1, TAB:CARG1->metatable
1443  |2:
1444  |  li CARG3, LJ_TNIL
1445  |   cmplwi TAB:CARG1, 0
1446  |  lwz STR:RC, DISPATCH_GL(gcroot[GCROOT_MMNAME+MM_metatable])(DISPATCH)
1447  |   beq ->fff_restv
1448  |  lwz TMP0, TAB:CARG1->hmask
1449  |   li CARG3, LJ_TTAB			// Use metatable as default result.
1450  |  lwz TMP1, STR:RC->sid
1451  |  lwz NODE:TMP2, TAB:CARG1->node
1452  |  and TMP1, TMP1, TMP0		// idx = str->sid & tab->hmask
1453  |  slwi TMP0, TMP1, 5
1454  |  slwi TMP1, TMP1, 3
1455  |  sub TMP1, TMP0, TMP1
1456  |  add NODE:TMP2, NODE:TMP2, TMP1	// node = tab->node + (idx*32-idx*8)
1457  |3:  // Rearranged logic, because we expect _not_ to find the key.
1458  |  lwz CARG4, NODE:TMP2->key
1459  |   lwz TMP0, 4+offsetof(Node, key)(NODE:TMP2)
1460  |    lwz CARG2, NODE:TMP2->val
1461  |     lwz TMP1, 4+offsetof(Node, val)(NODE:TMP2)
1462  |  checkstr CARG4; bne >4
1463  |   cmpw TMP0, STR:RC; beq >5
1464  |4:
1465  |  lwz NODE:TMP2, NODE:TMP2->next
1466  |  cmplwi NODE:TMP2, 0
1467  |  beq ->fff_restv			// Not found, keep default result.
1468  |  b <3
1469  |5:
1470  |  checknil CARG2
1471  |  beq ->fff_restv			// Ditto for nil value.
1472  |  mr CARG3, CARG2			// Return value of mt.__metatable.
1473  |  mr CARG1, TMP1
1474  |  b ->fff_restv
1475  |
1476  |6:
1477  |  cmpwi CARG3, LJ_TUDATA; beq <1
1478  |  .gpr64 extsw CARG3, CARG3
1479  |  subfc TMP0, TISNUM, CARG3
1480  |  subfe TMP2, CARG3, CARG3
1481  |  orc TMP1, TMP2, TMP0
1482  |  addi TMP1, TMP1, ~LJ_TISNUM+1
1483  |  slwi TMP1, TMP1, 2
1484  |   la TMP2, DISPATCH_GL(gcroot[GCROOT_BASEMT])(DISPATCH)
1485  |  lwzx TAB:CARG1, TMP2, TMP1
1486  |  b <2
1487  |
1488  |.ffunc_2 setmetatable
1489  |  // Fast path: no mt for table yet and not clearing the mt.
1490  |   checktab CARG3; bne ->fff_fallback
1491  |  lwz TAB:TMP1, TAB:CARG1->metatable
1492  |   checktab CARG4; bne ->fff_fallback
1493  |  cmplwi TAB:TMP1, 0
1494  |   lbz TMP3, TAB:CARG1->marked
1495  |  bne ->fff_fallback
1496  |   andix. TMP0, TMP3, LJ_GC_BLACK	// isblack(table)
1497  |    stw TAB:CARG2, TAB:CARG1->metatable
1498  |   beq ->fff_restv
1499  |  barrierback TAB:CARG1, TMP3, TMP0
1500  |  b ->fff_restv
1501  |
1502  |.ffunc rawget
1503  |  cmplwi NARGS8:RC, 16
1504  |   lwz CARG4, 0(BASE)
1505  |    lwz TAB:CARG2, 4(BASE)
1506  |  blt ->fff_fallback
1507  |  checktab CARG4; bne ->fff_fallback
1508  |   la CARG3, 8(BASE)
1509  |   mr CARG1, L
1510  |  bl extern lj_tab_get  // (lua_State *L, GCtab *t, cTValue *key)
1511  |  // Returns cTValue *.
1512  |.if FPU
1513  |  lfd FARG1, 0(CRET1)
1514  |.else
1515  |  lwz CARG2, 4(CRET1)
1516  |  lwz CARG1, 0(CRET1)	// Caveat: CARG1 == CRET1.
1517  |.endif
1518  |  b ->fff_resn
1519  |
1520  |//-- Base library: conversions ------------------------------------------
1521  |
1522  |.ffunc tonumber
1523  |  // Only handles the number case inline (without a base argument).
1524  |  cmplwi NARGS8:RC, 8
1525  |   lwz CARG1, 0(BASE)
1526  |.if FPU
1527  |    lfd FARG1, 0(BASE)
1528  |.else
1529  |    lwz CARG2, 4(BASE)
1530  |.endif
1531  |  bne ->fff_fallback			// Exactly one argument.
1532  |   checknum CARG1; bgt ->fff_fallback
1533  |  b ->fff_resn
1534  |
1535  |.ffunc_1 tostring
1536  |  // Only handles the string or number case inline.
1537  |  checkstr CARG3
1538  |  // A __tostring method in the string base metatable is ignored.
1539  |  beq ->fff_restv			// String key?
1540  |  // Handle numbers inline, unless a number base metatable is present.
1541  |  lwz TMP0, DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])(DISPATCH)
1542  |  checknum CARG3
1543  |  cmplwi cr1, TMP0, 0
1544  |   stp BASE, L->base			// Add frame since C call can throw.
1545  |  crorc 4*cr0+eq, 4*cr0+gt, 4*cr1+eq
1546  |   stw PC, SAVE_PC			// Redundant (but a defined value).
1547  |  beq ->fff_fallback
1548  |  ffgccheck
1549  |  mr CARG1, L
1550  |  mr CARG2, BASE
1551  |.if DUALNUM
1552  |  bl extern lj_strfmt_number		// (lua_State *L, cTValue *o)
1553  |.else
1554  |  bl extern lj_strfmt_num		// (lua_State *L, lua_Number *np)
1555  |.endif
1556  |  // Returns GCstr *.
1557  |  li CARG3, LJ_TSTR
1558  |  b ->fff_restv
1559  |
1560  |//-- Base library: iterators -------------------------------------------
1561  |
1562  |.ffunc_1 next
1563  |   stwx TISNIL, BASE, NARGS8:RC	// Set missing 2nd arg to nil.
1564  |  checktab CARG3
1565  |   lwz PC, FRAME_PC(BASE)
1566  |  bne ->fff_fallback
1567  |  la CARG2, 8(BASE)
1568  |  la CARG3, -8(BASE)
1569  |  bl extern lj_tab_next		// (GCtab *t, cTValue *key, TValue *o)
1570  |  // Returns 1=found, 0=end, -1=error.
1571  |  cmpwi CRET1, 0
1572  |   la RA, -8(BASE)
1573  |   li RD, (2+1)*8
1574  |  bgt ->fff_res			// Found key/value.
1575  |   li CARG3, LJ_TNIL
1576  |  beq ->fff_restv			// End of traversal: return nil.
1577  |   lwz CFUNC:RB, FRAME_FUNC(BASE)
1578  |   li NARGS8:RC, 2*8
1579  |  b ->fff_fallback			// Invalid key.
1580  |
1581  |.ffunc_1 pairs
1582  |  checktab CARG3
1583  |   lwz PC, FRAME_PC(BASE)
1584  |  bne ->fff_fallback
1585#if LJ_52
1586  |   lwz TAB:TMP2, TAB:CARG1->metatable
1587  |.if FPU
1588  |  lfd f0, CFUNC:RB->upvalue[0]
1589  |.else
1590  |  lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1591  |  lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1592  |.endif
1593  |   cmplwi TAB:TMP2, 0
1594  |  la RA, -8(BASE)
1595  |   bne ->fff_fallback
1596#else
1597  |.if FPU
1598  |  lfd f0, CFUNC:RB->upvalue[0]
1599  |.else
1600  |  lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1601  |  lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1602  |.endif
1603  |  la RA, -8(BASE)
1604#endif
1605  |   stw TISNIL, 8(BASE)
1606  |  li RD, (3+1)*8
1607  |.if FPU
1608  |  stfd f0, 0(RA)
1609  |.else
1610  |  stw TMP0, 0(RA)
1611  |  stw TMP1, 4(RA)
1612  |.endif
1613  |  b ->fff_res
1614  |
1615  |.ffunc ipairs_aux
1616  |  cmplwi NARGS8:RC, 16
1617  |   lwz CARG3, 0(BASE)
1618  |    lwz TAB:CARG1, 4(BASE)
1619  |   lwz CARG4, 8(BASE)
1620  |.if DUALNUM
1621  |    lwz TMP2, 12(BASE)
1622  |.else
1623  |    lfd FARG2, 8(BASE)
1624  |.endif
1625  |  blt ->fff_fallback
1626  |  checktab CARG3
1627  |  checknum cr1, CARG4
1628  |   lwz PC, FRAME_PC(BASE)
1629  |.if DUALNUM
1630  |  bne ->fff_fallback
1631  |  bne cr1, ->fff_fallback
1632  |.else
1633  |    lus TMP0, 0x3ff0
1634  |    stw ZERO, TMPD_LO
1635  |  bne ->fff_fallback
1636  |    stw TMP0, TMPD_HI
1637  |  bge cr1, ->fff_fallback
1638  |    lfd FARG1, TMPD
1639  |  toint TMP2, FARG2, f0
1640  |.endif
1641  |   lwz TMP0, TAB:CARG1->asize
1642  |   lwz TMP1, TAB:CARG1->array
1643  |.if not DUALNUM
1644  |  fadd FARG2, FARG2, FARG1
1645  |.endif
1646  |  addi TMP2, TMP2, 1
1647  |   la RA, -8(BASE)
1648  |  cmplw TMP0, TMP2
1649  |.if DUALNUM
1650  |  stw TISNUM, 0(RA)
1651  |   slwi TMP3, TMP2, 3
1652  |  stw TMP2, 4(RA)
1653  |.else
1654  |   slwi TMP3, TMP2, 3
1655  |  stfd FARG2, 0(RA)
1656  |.endif
1657  |  ble >2				// Not in array part?
1658  |.if FPU
1659  |  lwzx TMP2, TMP1, TMP3
1660  |  lfdx f0, TMP1, TMP3
1661  |.else
1662  |  lwzux TMP2, TMP1, TMP3
1663  |  lwz TMP3, 4(TMP1)
1664  |.endif
1665  |1:
1666  |  checknil TMP2
1667  |   li RD, (0+1)*8
1668  |  beq ->fff_res			// End of iteration, return 0 results.
1669  |   li RD, (2+1)*8
1670  |.if FPU
1671  |  stfd f0, 8(RA)
1672  |.else
1673  |  stw TMP2, 8(RA)
1674  |  stw TMP3, 12(RA)
1675  |.endif
1676  |  b ->fff_res
1677  |2:  // Check for empty hash part first. Otherwise call C function.
1678  |  lwz TMP0, TAB:CARG1->hmask
1679  |  cmplwi TMP0, 0
1680  |   li RD, (0+1)*8
1681  |  beq ->fff_res
1682  |   mr CARG2, TMP2
1683  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
1684  |  // Returns cTValue * or NULL.
1685  |  cmplwi CRET1, 0
1686  |   li RD, (0+1)*8
1687  |  beq ->fff_res
1688  |  lwz TMP2, 0(CRET1)
1689  |.if FPU
1690  |  lfd f0, 0(CRET1)
1691  |.else
1692  |  lwz TMP3, 4(CRET1)
1693  |.endif
1694  |  b <1
1695  |
1696  |.ffunc_1 ipairs
1697  |  checktab CARG3
1698  |   lwz PC, FRAME_PC(BASE)
1699  |  bne ->fff_fallback
1700#if LJ_52
1701  |   lwz TAB:TMP2, TAB:CARG1->metatable
1702  |.if FPU
1703  |  lfd f0, CFUNC:RB->upvalue[0]
1704  |.else
1705  |  lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1706  |  lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1707  |.endif
1708  |   cmplwi TAB:TMP2, 0
1709  |  la RA, -8(BASE)
1710  |   bne ->fff_fallback
1711#else
1712  |.if FPU
1713  |  lfd f0, CFUNC:RB->upvalue[0]
1714  |.else
1715  |  lwz TMP0, CFUNC:RB->upvalue[0].u32.hi
1716  |  lwz TMP1, CFUNC:RB->upvalue[0].u32.lo
1717  |.endif
1718  |  la RA, -8(BASE)
1719#endif
1720  |.if DUALNUM
1721  |  stw TISNUM, 8(BASE)
1722  |.else
1723  |  stw ZERO, 8(BASE)
1724  |.endif
1725  |   stw ZERO, 12(BASE)
1726  |  li RD, (3+1)*8
1727  |.if FPU
1728  |  stfd f0, 0(RA)
1729  |.else
1730  |  stw TMP0, 0(RA)
1731  |  stw TMP1, 4(RA)
1732  |.endif
1733  |  b ->fff_res
1734  |
1735  |//-- Base library: catch errors ----------------------------------------
1736  |
1737  |.ffunc pcall
1738  |    lwz TMP1, L->maxstack
1739  |    add TMP2, BASE, NARGS8:RC
1740  |  cmplwi NARGS8:RC, 8
1741  |   lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
1742  |    cmplw cr1, TMP1, TMP2
1743  |  cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
1744  |  blt ->fff_fallback
1745  |   mr TMP2, BASE
1746  |   la BASE, 8(BASE)
1747  |  // Remember active hook before pcall.
1748  |  rlwinm TMP3, TMP3, 32-HOOK_ACTIVE_SHIFT, 31, 31
1749  |   subi NARGS8:RC, NARGS8:RC, 8
1750  |  addi PC, TMP3, 8+FRAME_PCALL
1751  |  b ->vm_call_dispatch
1752  |
1753  |.ffunc xpcall
1754  |     lwz TMP1, L->maxstack
1755  |     add TMP2, BASE, NARGS8:RC
1756  |  cmplwi NARGS8:RC, 16
1757  |   lwz CARG3, 8(BASE)
1758  |     cmplw cr1, TMP1, TMP2
1759  |.if FPU
1760  |    lfd FARG2, 8(BASE)
1761  |  cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
1762  |    lfd FARG1, 0(BASE)
1763  |.else
1764  |    lwz CARG1, 0(BASE)
1765  |    lwz CARG2, 4(BASE)
1766  |  cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
1767  |    lwz CARG4, 12(BASE)
1768  |.endif
1769  |  blt ->fff_fallback
1770  |  lbz TMP1, DISPATCH_GL(hookmask)(DISPATCH)
1771  |   mr TMP2, BASE
1772  |  checkfunc CARG3; bne ->fff_fallback  // Traceback must be a function.
1773  |   la BASE, 16(BASE)
1774  |  // Remember active hook before pcall.
1775  |  rlwinm TMP1, TMP1, 32-HOOK_ACTIVE_SHIFT, 31, 31
1776  |.if FPU
1777  |    stfd FARG2, 0(TMP2)		// Swap function and traceback.
1778  |    stfd FARG1, 8(TMP2)
1779  |.else
1780  |    stw CARG3, 0(TMP2)
1781  |    stw CARG4, 4(TMP2)
1782  |    stw CARG1, 8(TMP2)
1783  |    stw CARG2, 12(TMP2)
1784  |.endif
1785  |  subi NARGS8:RC, NARGS8:RC, 16
1786  |  addi PC, TMP1, 16+FRAME_PCALL
1787  |  b ->vm_call_dispatch
1788  |
1789  |//-- Coroutine library --------------------------------------------------
1790  |
1791  |.macro coroutine_resume_wrap, resume
1792  |.if resume
1793  |.ffunc_1 coroutine_resume
1794  |  cmpwi CARG3, LJ_TTHREAD; bne ->fff_fallback
1795  |.else
1796  |.ffunc coroutine_wrap_aux
1797  |  lwz L:CARG1, CFUNC:RB->upvalue[0].gcr
1798  |.endif
1799  |  lbz TMP0, L:CARG1->status
1800  |   lp TMP1, L:CARG1->cframe
1801  |    lp CARG2, L:CARG1->top
1802  |  cmplwi cr0, TMP0, LUA_YIELD
1803  |    lp TMP2, L:CARG1->base
1804  |   cmplwi cr1, TMP1, 0
1805  |   lwz TMP0, L:CARG1->maxstack
1806  |    cmplw cr7, CARG2, TMP2
1807  |   lwz PC, FRAME_PC(BASE)
1808  |  crorc 4*cr6+lt, 4*cr0+gt, 4*cr1+eq		// st>LUA_YIELD || cframe!=0
1809  |   add TMP2, CARG2, NARGS8:RC
1810  |  crandc 4*cr6+gt, 4*cr7+eq, 4*cr0+eq	// base==top && st!=LUA_YIELD
1811  |   cmplw cr1, TMP2, TMP0
1812  |  cror 4*cr6+lt, 4*cr6+lt, 4*cr6+gt
1813  |   stw PC, SAVE_PC
1814  |  cror 4*cr6+lt, 4*cr6+lt, 4*cr1+gt		// cond1 || cond2 || stackov
1815  |   stp BASE, L->base
1816  |  blt cr6, ->fff_fallback
1817  |1:
1818  |.if resume
1819  |  addi BASE, BASE, 8			// Keep resumed thread in stack for GC.
1820  |  subi NARGS8:RC, NARGS8:RC, 8
1821  |  subi TMP2, TMP2, 8
1822  |.endif
1823  |  stp TMP2, L:CARG1->top
1824  |  li TMP1, 0
1825  |  stp BASE, L->top
1826  |2:  // Move args to coroutine.
1827  |  cmpw TMP1, NARGS8:RC
1828  |.if FPU
1829  |   lfdx f0, BASE, TMP1
1830  |.else
1831  |   add CARG3, BASE, TMP1
1832  |   lwz TMP2, 0(CARG3)
1833  |   lwz TMP3, 4(CARG3)
1834  |.endif
1835  |  beq >3
1836  |.if FPU
1837  |   stfdx f0, CARG2, TMP1
1838  |.else
1839  |   add CARG3, CARG2, TMP1
1840  |   stw TMP2, 0(CARG3)
1841  |   stw TMP3, 4(CARG3)
1842  |.endif
1843  |  addi TMP1, TMP1, 8
1844  |  b <2
1845  |3:
1846  |  li CARG3, 0
1847  |   mr L:SAVE0, L:CARG1
1848  |  li CARG4, 0
1849  |  bl ->vm_resume			// (lua_State *L, TValue *base, 0, 0)
1850  |  // Returns thread status.
1851  |4:
1852  |  lp TMP2, L:SAVE0->base
1853  |   cmplwi CRET1, LUA_YIELD
1854  |  lp TMP3, L:SAVE0->top
1855  |    li_vmstate INTERP
1856  |  lp BASE, L->base
1857  |    stw L, DISPATCH_GL(cur_L)(DISPATCH)
1858  |    st_vmstate
1859  |   bgt >8
1860  |  sub RD, TMP3, TMP2
1861  |   lwz TMP0, L->maxstack
1862  |  cmplwi RD, 0
1863  |   add TMP1, BASE, RD
1864  |  beq >6				// No results?
1865  |  cmplw TMP1, TMP0
1866  |   li TMP1, 0
1867  |  bgt >9				// Need to grow stack?
1868  |
1869  |  subi TMP3, RD, 8
1870  |   stp TMP2, L:SAVE0->top		// Clear coroutine stack.
1871  |5:  // Move results from coroutine.
1872  |  cmplw TMP1, TMP3
1873  |.if FPU
1874  |   lfdx f0, TMP2, TMP1
1875  |   stfdx f0, BASE, TMP1
1876  |.else
1877  |   add CARG3, TMP2, TMP1
1878  |   lwz CARG1, 0(CARG3)
1879  |   lwz CARG2, 4(CARG3)
1880  |   add CARG3, BASE, TMP1
1881  |   stw CARG1, 0(CARG3)
1882  |   stw CARG2, 4(CARG3)
1883  |.endif
1884  |    addi TMP1, TMP1, 8
1885  |  bne <5
1886  |6:
1887  |  andix. TMP0, PC, FRAME_TYPE
1888  |.if resume
1889  |  li TMP1, LJ_TTRUE
1890  |   la RA, -8(BASE)
1891  |  stw TMP1, -8(BASE)			// Prepend true to results.
1892  |  addi RD, RD, 16
1893  |.else
1894  |  mr RA, BASE
1895  |  addi RD, RD, 8
1896  |.endif
1897  |7:
1898  |    stw PC, SAVE_PC
1899  |   mr MULTRES, RD
1900  |  beq ->BC_RET_Z
1901  |  b ->vm_return
1902  |
1903  |8:  // Coroutine returned with error (at co->top-1).
1904  |.if resume
1905  |  andix. TMP0, PC, FRAME_TYPE
1906  |  la TMP3, -8(TMP3)
1907  |   li TMP1, LJ_TFALSE
1908  |.if FPU
1909  |  lfd f0, 0(TMP3)
1910  |.else
1911  |  lwz CARG1, 0(TMP3)
1912  |  lwz CARG2, 4(TMP3)
1913  |.endif
1914  |   stp TMP3, L:SAVE0->top		// Remove error from coroutine stack.
1915  |    li RD, (2+1)*8
1916  |   stw TMP1, -8(BASE)		// Prepend false to results.
1917  |    la RA, -8(BASE)
1918  |.if FPU
1919  |  stfd f0, 0(BASE)			// Copy error message.
1920  |.else
1921  |  stw CARG1, 0(BASE)			// Copy error message.
1922  |  stw CARG2, 4(BASE)
1923  |.endif
1924  |  b <7
1925  |.else
1926  |  mr CARG1, L
1927  |  mr CARG2, L:SAVE0
1928  |  bl extern lj_ffh_coroutine_wrap_err  // (lua_State *L, lua_State *co)
1929  |.endif
1930  |
1931  |9:  // Handle stack expansion on return from yield.
1932  |  mr CARG1, L
1933  |  srwi CARG2, RD, 3
1934  |  bl extern lj_state_growstack	// (lua_State *L, int n)
1935  |  li CRET1, 0
1936  |  b <4
1937  |.endmacro
1938  |
1939  |  coroutine_resume_wrap 1		// coroutine.resume
1940  |  coroutine_resume_wrap 0		// coroutine.wrap
1941  |
1942  |.ffunc coroutine_yield
1943  |  lp TMP0, L->cframe
1944  |   add TMP1, BASE, NARGS8:RC
1945  |   stp BASE, L->base
1946  |  andix. TMP0, TMP0, CFRAME_RESUME
1947  |   stp TMP1, L->top
1948  |    li CRET1, LUA_YIELD
1949  |  beq ->fff_fallback
1950  |   stp ZERO, L->cframe
1951  |    stb CRET1, L->status
1952  |  b ->vm_leave_unw
1953  |
1954  |//-- Math library -------------------------------------------------------
1955  |
1956  |.ffunc_1 math_abs
1957  |  checknum CARG3
1958  |.if DUALNUM
1959  |  bne >2
1960  |  srawi TMP1, CARG1, 31
1961  |  xor TMP2, TMP1, CARG1
1962  |.if GPR64
1963  |  lus TMP0, 0x8000
1964  |  sub CARG1, TMP2, TMP1
1965  |  cmplw CARG1, TMP0
1966  |  beq >1
1967  |.else
1968  |  sub. CARG1, TMP2, TMP1
1969  |  blt >1
1970  |.endif
1971  |->fff_resi:
1972  |  lwz PC, FRAME_PC(BASE)
1973  |  la RA, -8(BASE)
1974  |  stw TISNUM, -8(BASE)
1975  |  stw CRET1, -4(BASE)
1976  |  b ->fff_res1
1977  |1:
1978  |  lus CARG3, 0x41e0	// 2^31.
1979  |  li CARG1, 0
1980  |  b ->fff_restv
1981  |2:
1982  |.endif
1983  |  bge ->fff_fallback
1984  |  rlwinm CARG3, CARG3, 0, 1, 31
1985  |  // Fallthrough.
1986  |
1987  |->fff_restv:
1988  |  // CARG3/CARG1 = TValue result.
1989  |  lwz PC, FRAME_PC(BASE)
1990  |   stw CARG3, -8(BASE)
1991  |  la RA, -8(BASE)
1992  |   stw CARG1, -4(BASE)
1993  |->fff_res1:
1994  |  // RA = results, PC = return.
1995  |  li RD, (1+1)*8
1996  |->fff_res:
1997  |  // RA = results, RD = (nresults+1)*8, PC = return.
1998  |  andix. TMP0, PC, FRAME_TYPE
1999  |   mr MULTRES, RD
2000  |  bney ->vm_return
2001  |  lwz INS, -4(PC)
2002  |  decode_RB8 RB, INS
2003  |5:
2004  |  cmplw RB, RD			// More results expected?
2005  |   decode_RA8 TMP0, INS
2006  |  bgt >6
2007  |  ins_next1
2008  |  // Adjust BASE. KBASE is assumed to be set for the calling frame.
2009  |   sub BASE, RA, TMP0
2010  |  ins_next2
2011  |
2012  |6:  // Fill up results with nil.
2013  |  subi TMP1, RD, 8
2014  |   addi RD, RD, 8
2015  |  stwx TISNIL, RA, TMP1
2016  |  b <5
2017  |
2018  |.macro math_extern, func
2019  |  .ffunc_n math_ .. func
2020  |  blex func
2021  |  b ->fff_resn
2022  |.endmacro
2023  |
2024  |.macro math_extern2, func
2025  |  .ffunc_nn math_ .. func
2026  |  blex func
2027  |  b ->fff_resn
2028  |.endmacro
2029  |
2030  |.macro math_round, func
2031  |  .ffunc_1 math_ .. func
2032  |   checknum CARG3; beqy ->fff_restv
2033  |  rlwinm TMP2, CARG3, 12, 21, 31
2034  |   bge ->fff_fallback
2035  |  addic. TMP2, TMP2, -1023		// exp = exponent(x) - 1023
2036  |  cmplwi cr1, TMP2, 31		// 0 <= exp < 31?
2037  |   subfic TMP0, TMP2, 31
2038  |  blt >3
2039  |  slwi TMP1, CARG3, 11
2040  |   srwi TMP3, CARG1, 21
2041  |  oris TMP1, TMP1, 0x8000
2042  |   addi TMP2, TMP2, 1
2043  |  or TMP1, TMP1, TMP3
2044  |   slwi CARG2, CARG1, 11
2045  |  bge cr1, >4
2046  |   slw TMP3, TMP1, TMP2
2047  |  srw RD, TMP1, TMP0
2048  |   or TMP3, TMP3, CARG2
2049  |  srawi TMP2, CARG3, 31
2050  |.if "func" == "floor"
2051  |  and TMP1, TMP3, TMP2
2052  |  addic TMP0, TMP1, -1
2053  |  subfe TMP1, TMP0, TMP1
2054  |  add CARG1, RD, TMP1
2055  |  xor CARG1, CARG1, TMP2
2056  |  sub CARG1, CARG1, TMP2
2057  |  b ->fff_resi
2058  |.else
2059  |  andc TMP1, TMP3, TMP2
2060  |  addic TMP0, TMP1, -1
2061  |  subfe TMP1, TMP0, TMP1
2062  |  add CARG1, RD, TMP1
2063  |  cmpw CARG1, RD
2064  |  xor CARG1, CARG1, TMP2
2065  |  sub CARG1, CARG1, TMP2
2066  |  bge ->fff_resi
2067  |  // Overflow to 2^31.
2068  |  lus CARG3, 0x41e0			// 2^31.
2069  |  li CARG1, 0
2070  |  b ->fff_restv
2071  |.endif
2072  |3:  // |x| < 1
2073  |  slwi TMP2, CARG3, 1
2074  |   srawi TMP1, CARG3, 31
2075  |  or TMP2, CARG1, TMP2		// ztest = (hi+hi) | lo
2076  |.if "func" == "floor"
2077  |  and TMP1, TMP2, TMP1		// (ztest & sign) == 0 ? 0 : -1
2078  |  subfic TMP2, TMP1, 0
2079  |  subfe CARG1, CARG1, CARG1
2080  |.else
2081  |  andc TMP1, TMP2, TMP1		// (ztest & ~sign) == 0 ? 0 : 1
2082  |  addic TMP2, TMP1, -1
2083  |  subfe CARG1, TMP2, TMP1
2084  |.endif
2085  |  b ->fff_resi
2086  |4:  // exp >= 31. Check for -(2^31).
2087  |  xoris TMP1, TMP1, 0x8000
2088  |  srawi TMP2, CARG3, 31
2089  |.if "func" == "floor"
2090  |  or TMP1, TMP1, CARG2
2091  |.endif
2092  |.if PPE
2093  |  orc TMP1, TMP1, TMP2
2094  |  cmpwi TMP1, 0
2095  |.else
2096  |  orc. TMP1, TMP1, TMP2
2097  |.endif
2098  |  crand 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
2099  |  lus CARG1, 0x8000			// -(2^31).
2100  |  beqy ->fff_resi
2101  |5:
2102  |.if FPU
2103  |  lfd FARG1, 0(BASE)
2104  |.else
2105  |  lwz CARG1, 0(BASE)
2106  |  lwz CARG2, 4(BASE)
2107  |.endif
2108  |  blex func
2109  |  b ->fff_resn
2110  |.endmacro
2111  |
2112  |.if DUALNUM
2113  |  math_round floor
2114  |  math_round ceil
2115  |.else
2116  |  // NYI: use internal implementation.
2117  |  math_extern floor
2118  |  math_extern ceil
2119  |.endif
2120  |
2121  |.if SQRT
2122  |.ffunc_n math_sqrt
2123  |  fsqrt FARG1, FARG1
2124  |  b ->fff_resn
2125  |.else
2126  |  math_extern sqrt
2127  |.endif
2128  |
2129  |.ffunc math_log
2130  |  cmplwi NARGS8:RC, 8
2131  |   lwz CARG1, 0(BASE)
2132  |  bne ->fff_fallback			// Need exactly 1 argument.
2133  |  checknum CARG1; bge ->fff_fallback
2134  |.if FPU
2135  |  lfd FARG1, 0(BASE)
2136  |.else
2137  |  lwz CARG2, 4(BASE)
2138  |.endif
2139  |  blex log
2140  |  b ->fff_resn
2141  |
2142  |  math_extern log10
2143  |  math_extern exp
2144  |  math_extern sin
2145  |  math_extern cos
2146  |  math_extern tan
2147  |  math_extern asin
2148  |  math_extern acos
2149  |  math_extern atan
2150  |  math_extern sinh
2151  |  math_extern cosh
2152  |  math_extern tanh
2153  |  math_extern2 pow
2154  |  math_extern2 atan2
2155  |  math_extern2 fmod
2156  |
2157  |.if DUALNUM
2158  |.ffunc math_ldexp
2159  |  cmplwi NARGS8:RC, 16
2160  |   lwz TMP0, 0(BASE)
2161  |.if FPU
2162  |    lfd FARG1, 0(BASE)
2163  |.else
2164  |    lwz CARG1, 0(BASE)
2165  |    lwz CARG2, 4(BASE)
2166  |.endif
2167  |   lwz TMP1, 8(BASE)
2168  |.if GPR64
2169  |    lwz CARG2, 12(BASE)
2170  |.elif FPU
2171  |    lwz CARG1, 12(BASE)
2172  |.else
2173  |    lwz CARG3, 12(BASE)
2174  |.endif
2175  |  blt ->fff_fallback
2176  |  checknum TMP0; bge ->fff_fallback
2177  |  checknum TMP1; bne ->fff_fallback
2178  |.else
2179  |.ffunc_nn math_ldexp
2180  |.if GPR64
2181  |  toint CARG2, FARG2
2182  |.else
2183  |  toint CARG1, FARG2
2184  |.endif
2185  |.endif
2186  |  blex ldexp
2187  |  b ->fff_resn
2188  |
2189  |.ffunc_n math_frexp
2190  |.if GPR64
2191  |  la CARG2, DISPATCH_GL(tmptv)(DISPATCH)
2192  |.elif FPU
2193  |  la CARG1, DISPATCH_GL(tmptv)(DISPATCH)
2194  |.else
2195  |  la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
2196  |.endif
2197  |   lwz PC, FRAME_PC(BASE)
2198  |  blex frexp
2199  |   lwz TMP1, DISPATCH_GL(tmptv)(DISPATCH)
2200  |   la RA, -8(BASE)
2201  |.if not DUALNUM
2202  |   tonum_i FARG2, TMP1
2203  |.endif
2204  |.if FPU
2205  |  stfd FARG1, 0(RA)
2206  |.else
2207  |  stw CRET1, 0(RA)
2208  |  stw CRET2, 4(RA)
2209  |.endif
2210  |  li RD, (2+1)*8
2211  |.if DUALNUM
2212  |   stw TISNUM, 8(RA)
2213  |   stw TMP1, 12(RA)
2214  |.else
2215  |   stfd FARG2, 8(RA)
2216  |.endif
2217  |  b ->fff_res
2218  |
2219  |.ffunc_n math_modf
2220  |.if GPR64
2221  |  la CARG2, -8(BASE)
2222  |.elif FPU
2223  |  la CARG1, -8(BASE)
2224  |.else
2225  |  la CARG3, -8(BASE)
2226  |.endif
2227  |   lwz PC, FRAME_PC(BASE)
2228  |  blex modf
2229  |   la RA, -8(BASE)
2230  |.if FPU
2231  |  stfd FARG1, 0(BASE)
2232  |.else
2233  |  stw CRET1, 0(BASE)
2234  |  stw CRET2, 4(BASE)
2235  |.endif
2236  |  li RD, (2+1)*8
2237  |  b ->fff_res
2238  |
2239  |.macro math_minmax, name, ismax
2240  |.if DUALNUM
2241  |  .ffunc_1 name
2242  |  checknum CARG3
2243  |   addi SAVE0, BASE, 8
2244  |   add SAVE1, BASE, NARGS8:RC
2245  |  bne >4
2246  |1:  // Handle integers.
2247  |  lwz CARG4, 0(SAVE0)
2248  |   cmplw cr1, SAVE0, SAVE1
2249  |  lwz CARG2, 4(SAVE0)
2250  |   bge cr1, ->fff_resi
2251  |  checknum CARG4
2252  |   xoris TMP0, CARG1, 0x8000
2253  |   xoris TMP3, CARG2, 0x8000
2254  |  bne >3
2255  |  subfc TMP3, TMP3, TMP0
2256  |  subfe TMP0, TMP0, TMP0
2257  |.if ismax
2258  |  andc TMP3, TMP3, TMP0
2259  |.else
2260  |  and TMP3, TMP3, TMP0
2261  |.endif
2262  |  add CARG1, TMP3, CARG2
2263  |.if GPR64
2264  |  rldicl CARG1, CARG1, 0, 32
2265  |.endif
2266  |   addi SAVE0, SAVE0, 8
2267  |  b <1
2268  |3:
2269  |  bge ->fff_fallback
2270  |  // Convert intermediate result to number and continue below.
2271  |.if FPU
2272  |  tonum_i FARG1, CARG1
2273  |  lfd FARG2, 0(SAVE0)
2274  |.else
2275  |  mr CARG2, CARG1
2276  |  bl ->vm_sfi2d_1
2277  |  lwz CARG3, 0(SAVE0)
2278  |  lwz CARG4, 4(SAVE0)
2279  |.endif
2280  |  b >6
2281  |4:
2282  |.if FPU
2283  |   lfd FARG1, 0(BASE)
2284  |.else
2285  |   lwz CARG1, 0(BASE)
2286  |   lwz CARG2, 4(BASE)
2287  |.endif
2288  |  bge ->fff_fallback
2289  |5:  // Handle numbers.
2290  |  lwz CARG3, 0(SAVE0)
2291  |   cmplw cr1, SAVE0, SAVE1
2292  |.if FPU
2293  |  lfd FARG2, 0(SAVE0)
2294  |.else
2295  |  lwz CARG4, 4(SAVE0)
2296  |.endif
2297  |   bge cr1, ->fff_resn
2298  |  checknum CARG3; bge >7
2299  |6:
2300  |   addi SAVE0, SAVE0, 8
2301  |.if FPU
2302  |.if ismax
2303  |  fsub f0, FARG1, FARG2
2304  |.else
2305  |  fsub f0, FARG2, FARG1
2306  |.endif
2307  |  fsel FARG1, f0, FARG1, FARG2
2308  |.else
2309  |  stw CARG1, SFSAVE_1
2310  |  stw CARG2, SFSAVE_2
2311  |  stw CARG3, SFSAVE_3
2312  |  stw CARG4, SFSAVE_4
2313  |  blex __ledf2
2314  |  cmpwi CRET1, 0
2315  |.if ismax
2316  |  blt >8
2317  |.else
2318  |  bge >8
2319  |.endif
2320  |  lwz CARG1, SFSAVE_1
2321  |  lwz CARG2, SFSAVE_2
2322  |  b <5
2323  |8:
2324  |  lwz CARG1, SFSAVE_3
2325  |  lwz CARG2, SFSAVE_4
2326  |.endif
2327  |  b <5
2328  |7:  // Convert integer to number and continue above.
2329  |   lwz CARG3, 4(SAVE0)
2330  |  bne ->fff_fallback
2331  |.if FPU
2332  |  tonum_i FARG2, CARG3
2333  |.else
2334  |  bl ->vm_sfi2d_2
2335  |.endif
2336  |  b <6
2337  |.else
2338  |  .ffunc_n name
2339  |  li TMP1, 8
2340  |1:
2341  |   lwzx CARG2, BASE, TMP1
2342  |   lfdx FARG2, BASE, TMP1
2343  |  cmplw cr1, TMP1, NARGS8:RC
2344  |   checknum CARG2
2345  |  bge cr1, ->fff_resn
2346  |   bge ->fff_fallback
2347  |.if ismax
2348  |  fsub f0, FARG1, FARG2
2349  |.else
2350  |  fsub f0, FARG2, FARG1
2351  |.endif
2352  |   addi TMP1, TMP1, 8
2353  |  fsel FARG1, f0, FARG1, FARG2
2354  |  b <1
2355  |.endif
2356  |.endmacro
2357  |
2358  |  math_minmax math_min, 0
2359  |  math_minmax math_max, 1
2360  |
2361  |//-- String library -----------------------------------------------------
2362  |
2363  |.ffunc string_byte			// Only handle the 1-arg case here.
2364  |  cmplwi NARGS8:RC, 8
2365  |   lwz CARG3, 0(BASE)
2366  |    lwz STR:CARG1, 4(BASE)
2367  |  bne ->fff_fallback			// Need exactly 1 argument.
2368  |   checkstr CARG3
2369  |   bne ->fff_fallback
2370  |  lwz TMP0, STR:CARG1->len
2371  |.if DUALNUM
2372  |   lbz CARG1, STR:CARG1[1]		// Access is always ok (NUL at end).
2373  |   li RD, (0+1)*8
2374  |   lwz PC, FRAME_PC(BASE)
2375  |  cmplwi TMP0, 0
2376  |   la RA, -8(BASE)
2377  |  beqy ->fff_res
2378  |  b ->fff_resi
2379  |.else
2380  |   lbz TMP1, STR:CARG1[1]		// Access is always ok (NUL at end).
2381  |  addic TMP3, TMP0, -1		// RD = ((str->len != 0)+1)*8
2382  |  subfe RD, TMP3, TMP0
2383  |   stw TMP1, TONUM_LO		// Inlined tonum_u f0, TMP1.
2384  |  addi RD, RD, 1
2385  |   lfd f0, TONUM_D
2386  |  la RA, -8(BASE)
2387  |  lwz PC, FRAME_PC(BASE)
2388  |   fsub f0, f0, TOBIT
2389  |  slwi RD, RD, 3
2390  |   stfd f0, 0(RA)
2391  |  b ->fff_res
2392  |.endif
2393  |
2394  |.ffunc string_char			// Only handle the 1-arg case here.
2395  |  ffgccheck
2396  |  cmplwi NARGS8:RC, 8
2397  |   lwz CARG3, 0(BASE)
2398  |.if DUALNUM
2399  |    lwz TMP0, 4(BASE)
2400  |  bne ->fff_fallback			// Exactly 1 argument.
2401  |  checknum CARG3; bne ->fff_fallback
2402  |   la CARG2, 7(BASE)
2403  |.else
2404  |    lfd FARG1, 0(BASE)
2405  |  bne ->fff_fallback			// Exactly 1 argument.
2406  |  checknum CARG3; bge ->fff_fallback
2407  |  toint TMP0, FARG1
2408  |   la CARG2, TMPD_BLO
2409  |.endif
2410  |   li CARG3, 1
2411  |  cmplwi TMP0, 255; bgt ->fff_fallback
2412  |->fff_newstr:
2413  |  mr CARG1, L
2414  |  stp BASE, L->base
2415  |  stw PC, SAVE_PC
2416  |  bl extern lj_str_new		// (lua_State *L, char *str, size_t l)
2417  |->fff_resstr:
2418  |  // Returns GCstr *.
2419  |  lp BASE, L->base
2420  |  li CARG3, LJ_TSTR
2421  |  b ->fff_restv
2422  |
2423  |.ffunc string_sub
2424  |  ffgccheck
2425  |  cmplwi NARGS8:RC, 16
2426  |   lwz CARG3, 16(BASE)
2427  |.if not DUALNUM
2428  |    lfd f0, 16(BASE)
2429  |.endif
2430  |   lwz TMP0, 0(BASE)
2431  |    lwz STR:CARG1, 4(BASE)
2432  |  blt ->fff_fallback
2433  |   lwz CARG2, 8(BASE)
2434  |.if DUALNUM
2435  |    lwz TMP1, 12(BASE)
2436  |.else
2437  |    lfd f1, 8(BASE)
2438  |.endif
2439  |   li TMP2, -1
2440  |  beq >1
2441  |.if DUALNUM
2442  |  checknum CARG3
2443  |   lwz TMP2, 20(BASE)
2444  |  bne ->fff_fallback
2445  |1:
2446  |  checknum CARG2; bne ->fff_fallback
2447  |.else
2448  |  checknum CARG3; bge ->fff_fallback
2449  |  toint TMP2, f0
2450  |1:
2451  |  checknum CARG2; bge ->fff_fallback
2452  |.endif
2453  |  checkstr TMP0; bne ->fff_fallback
2454  |.if not DUALNUM
2455  |   toint TMP1, f1
2456  |.endif
2457  |   lwz TMP0, STR:CARG1->len
2458  |  cmplw TMP0, TMP2			// len < end? (unsigned compare)
2459  |   addi TMP3, TMP2, 1
2460  |  blt >5
2461  |2:
2462  |  cmpwi TMP1, 0			// start <= 0?
2463  |   add TMP3, TMP1, TMP0
2464  |  ble >7
2465  |3:
2466  |  sub CARG3, TMP2, TMP1
2467  |    addi CARG2, STR:CARG1, #STR-1
2468  |  srawi TMP0, CARG3, 31
2469  |   addi CARG3, CARG3, 1
2470  |    add CARG2, CARG2, TMP1
2471  |  andc CARG3, CARG3, TMP0
2472  |.if GPR64
2473  |  rldicl CARG2, CARG2, 0, 32
2474  |  rldicl CARG3, CARG3, 0, 32
2475  |.endif
2476  |  b ->fff_newstr
2477  |
2478  |5:  // Negative end or overflow.
2479  |  cmpw TMP0, TMP2			// len >= end? (signed compare)
2480  |   add TMP2, TMP0, TMP3		// Negative end: end = end+len+1.
2481  |  bge <2
2482  |   mr TMP2, TMP0			// Overflow: end = len.
2483  |  b <2
2484  |
2485  |7:  // Negative start or underflow.
2486  |  .gpr64 extsw TMP1, TMP1
2487  |  addic CARG3, TMP1, -1
2488  |  subfe CARG3, CARG3, CARG3
2489  |   srawi CARG2, TMP3, 31		// Note: modifies carry.
2490  |  andc TMP3, TMP3, CARG3
2491  |   andc TMP1, TMP3, CARG2
2492  |  addi TMP1, TMP1, 1			// start = 1 + (start ? start+len : 0)
2493  |  b <3
2494  |
2495  |.macro ffstring_op, name
2496  |  .ffunc string_ .. name
2497  |  ffgccheck
2498  |  cmplwi NARGS8:RC, 8
2499  |   lwz CARG3, 0(BASE)
2500  |    lwz STR:CARG2, 4(BASE)
2501  |  blt ->fff_fallback
2502  |  checkstr CARG3
2503  |   la SBUF:CARG1, DISPATCH_GL(tmpbuf)(DISPATCH)
2504  |  bne ->fff_fallback
2505  |   lwz TMP0, SBUF:CARG1->b
2506  |  stw L, SBUF:CARG1->L
2507  |  stp BASE, L->base
2508  |  stw PC, SAVE_PC
2509  |   stw TMP0, SBUF:CARG1->w
2510  |  bl extern lj_buf_putstr_ .. name
2511  |  bl extern lj_buf_tostr
2512  |  b ->fff_resstr
2513  |.endmacro
2514  |
2515  |ffstring_op reverse
2516  |ffstring_op lower
2517  |ffstring_op upper
2518  |
2519  |//-- Bit library --------------------------------------------------------
2520  |
2521  |.macro .ffunc_bit, name
2522  |.if DUALNUM
2523  |  .ffunc_1 bit_..name
2524  |  checknum CARG3; bnel ->fff_tobit_fb
2525  |.else
2526  |  .ffunc_n bit_..name
2527  |  fadd FARG1, FARG1, TOBIT
2528  |  stfd FARG1, TMPD
2529  |  lwz CARG1, TMPD_LO
2530  |.endif
2531  |.endmacro
2532  |
2533  |.macro .ffunc_bit_op, name, ins
2534  |  .ffunc_bit name
2535  |  addi SAVE0, BASE, 8
2536  |  add SAVE1, BASE, NARGS8:RC
2537  |1:
2538  |  lwz CARG4, 0(SAVE0)
2539  |   cmplw cr1, SAVE0, SAVE1
2540  |.if DUALNUM
2541  |  lwz CARG2, 4(SAVE0)
2542  |.else
2543  |  lfd FARG1, 0(SAVE0)
2544  |.endif
2545  |   bgey cr1, ->fff_resi
2546  |  checknum CARG4
2547  |.if DUALNUM
2548  |.if FPU
2549  |  bnel ->fff_bitop_fb
2550  |.else
2551  |  beq >3
2552  |  stw CARG1, SFSAVE_1
2553  |  bl ->fff_bitop_fb
2554  |  mr CARG2, CARG1
2555  |  lwz CARG1, SFSAVE_1
2556  |3:
2557  |.endif
2558  |.else
2559  |  fadd FARG1, FARG1, TOBIT
2560  |  bge ->fff_fallback
2561  |  stfd FARG1, TMPD
2562  |  lwz CARG2, TMPD_LO
2563  |.endif
2564  |  ins CARG1, CARG1, CARG2
2565  |   addi SAVE0, SAVE0, 8
2566  |  b <1
2567  |.endmacro
2568  |
2569  |.ffunc_bit_op band, and
2570  |.ffunc_bit_op bor, or
2571  |.ffunc_bit_op bxor, xor
2572  |
2573  |.ffunc_bit bswap
2574  |  rotlwi TMP0, CARG1, 8
2575  |  rlwimi TMP0, CARG1, 24, 0, 7
2576  |  rlwimi TMP0, CARG1, 24, 16, 23
2577  |  mr CRET1, TMP0
2578  |  b ->fff_resi
2579  |
2580  |.ffunc_bit bnot
2581  |  not CRET1, CARG1
2582  |  b ->fff_resi
2583  |
2584  |.macro .ffunc_bit_sh, name, ins, shmod
2585  |.if DUALNUM
2586  |  .ffunc_2 bit_..name
2587  |.if FPU
2588  |  checknum CARG3; bnel ->fff_tobit_fb
2589  |.else
2590  |  checknum CARG3; beq >1
2591  |  bl ->fff_tobit_fb
2592  |  lwz CARG2, 12(BASE)	// Conversion polluted CARG2.
2593  |1:
2594  |.endif
2595  |  // Note: no inline conversion from number for 2nd argument!
2596  |  checknum CARG4; bne ->fff_fallback
2597  |.else
2598  |  .ffunc_nn bit_..name
2599  |  fadd FARG1, FARG1, TOBIT
2600  |  fadd FARG2, FARG2, TOBIT
2601  |  stfd FARG1, TMPD
2602  |  lwz CARG1, TMPD_LO
2603  |  stfd FARG2, TMPD
2604  |  lwz CARG2, TMPD_LO
2605  |.endif
2606  |.if shmod == 1
2607  |  rlwinm CARG2, CARG2, 0, 27, 31
2608  |.elif shmod == 2
2609  |  neg CARG2, CARG2
2610  |.endif
2611  |  ins CRET1, CARG1, CARG2
2612  |  b ->fff_resi
2613  |.endmacro
2614  |
2615  |.ffunc_bit_sh lshift, slw, 1
2616  |.ffunc_bit_sh rshift, srw, 1
2617  |.ffunc_bit_sh arshift, sraw, 1
2618  |.ffunc_bit_sh rol, rotlw, 0
2619  |.ffunc_bit_sh ror, rotlw, 2
2620  |
2621  |.ffunc_bit tobit
2622  |.if DUALNUM
2623  |  b ->fff_resi
2624  |.else
2625  |->fff_resi:
2626  |  tonum_i FARG1, CRET1
2627  |.endif
2628  |->fff_resn:
2629  |  lwz PC, FRAME_PC(BASE)
2630  |  la RA, -8(BASE)
2631  |.if FPU
2632  |  stfd FARG1, -8(BASE)
2633  |.else
2634  |  stw CARG1, -8(BASE)
2635  |  stw CARG2, -4(BASE)
2636  |.endif
2637  |  b ->fff_res1
2638  |
2639  |// Fallback FP number to bit conversion.
2640  |->fff_tobit_fb:
2641  |.if DUALNUM
2642  |.if FPU
2643  |  lfd FARG1, 0(BASE)
2644  |  bgt ->fff_fallback
2645  |  fadd FARG1, FARG1, TOBIT
2646  |  stfd FARG1, TMPD
2647  |  lwz CARG1, TMPD_LO
2648  |  blr
2649  |.else
2650  |  bgt ->fff_fallback
2651  |  mr CARG2, CARG1
2652  |  mr CARG1, CARG3
2653  |// Modifies: CARG1, CARG2, TMP0, TMP1, TMP2.
2654  |->vm_tobit:
2655  |  slwi TMP2, CARG1, 1
2656  |  addis TMP2, TMP2, 0x0020
2657  |  cmpwi TMP2, 0
2658  |  bge >2
2659  |   li TMP1, 0x3e0
2660  |  srawi TMP2, TMP2, 21
2661  |   not TMP1, TMP1
2662  |  sub. TMP2, TMP1, TMP2
2663  |    cmpwi cr7, CARG1, 0
2664  |  blt >1
2665  |   slwi TMP1, CARG1, 11
2666  |    srwi TMP0, CARG2, 21
2667  |   oris TMP1, TMP1, 0x8000
2668  |   or TMP1, TMP1, TMP0
2669  |   srw CARG1, TMP1, TMP2
2670  |  bclr 4, 28			// Return if cr7[lt] == 0, no hint.
2671  |   neg CARG1, CARG1
2672  |  blr
2673  |1:
2674  |  addi TMP2, TMP2, 21
2675  |  srw TMP1, CARG2, TMP2
2676  |   slwi CARG2, CARG1, 12
2677  |  subfic TMP2, TMP2, 20
2678  |   slw TMP0, CARG2, TMP2
2679  |   or CARG1, TMP1, TMP0
2680  |  bclr 4, 28			// Return if cr7[lt] == 0, no hint.
2681  |   neg CARG1, CARG1
2682  |  blr
2683  |2:
2684  |  li CARG1, 0
2685  |  blr
2686  |.endif
2687  |.endif
2688  |->fff_bitop_fb:
2689  |.if DUALNUM
2690  |.if FPU
2691  |  lfd FARG1, 0(SAVE0)
2692  |  bgt ->fff_fallback
2693  |  fadd FARG1, FARG1, TOBIT
2694  |  stfd FARG1, TMPD
2695  |  lwz CARG2, TMPD_LO
2696  |  blr
2697  |.else
2698  |  bgt ->fff_fallback
2699  |  mr CARG1, CARG4
2700  |  b ->vm_tobit
2701  |.endif
2702  |.endif
2703  |
2704  |//-----------------------------------------------------------------------
2705  |
2706  |->fff_fallback:			// Call fast function fallback handler.
2707  |  // BASE = new base, RB = CFUNC, RC = nargs*8
2708  |  lp TMP3, CFUNC:RB->f
2709  |    add TMP1, BASE, NARGS8:RC
2710  |   lwz PC, FRAME_PC(BASE)		// Fallback may overwrite PC.
2711  |    addi TMP0, TMP1, 8*LUA_MINSTACK
2712  |     lwz TMP2, L->maxstack
2713  |   stw PC, SAVE_PC			// Redundant (but a defined value).
2714  |  .toc lp TMP3, 0(TMP3)
2715  |  cmplw TMP0, TMP2
2716  |     stp BASE, L->base
2717  |    stp TMP1, L->top
2718  |   mr CARG1, L
2719  |  bgt >5				// Need to grow stack.
2720  |  mtctr TMP3
2721  |  bctrl				// (lua_State *L)
2722  |  // Either throws an error, or recovers and returns -1, 0 or nresults+1.
2723  |  lp BASE, L->base
2724  |  cmpwi CRET1, 0
2725  |   slwi RD, CRET1, 3
2726  |   la RA, -8(BASE)
2727  |  bgt ->fff_res			// Returned nresults+1?
2728  |1:  // Returned 0 or -1: retry fast path.
2729  |  lp TMP0, L->top
2730  |   lwz LFUNC:RB, FRAME_FUNC(BASE)
2731  |  sub NARGS8:RC, TMP0, BASE
2732  |  bne ->vm_call_tail			// Returned -1?
2733  |  ins_callt				// Returned 0: retry fast path.
2734  |
2735  |// Reconstruct previous base for vmeta_call during tailcall.
2736  |->vm_call_tail:
2737  |  andix. TMP0, PC, FRAME_TYPE
2738  |   rlwinm TMP1, PC, 0, 0, 28
2739  |  bne >3
2740  |  lwz INS, -4(PC)
2741  |  decode_RA8 TMP1, INS
2742  |  addi TMP1, TMP1, 8
2743  |3:
2744  |  sub TMP2, BASE, TMP1
2745  |  b ->vm_call_dispatch		// Resolve again for tailcall.
2746  |
2747  |5:  // Grow stack for fallback handler.
2748  |  li CARG2, LUA_MINSTACK
2749  |  bl extern lj_state_growstack	// (lua_State *L, int n)
2750  |  lp BASE, L->base
2751  |  cmpw TMP0, TMP0			// Set 4*cr0+eq to force retry.
2752  |  b <1
2753  |
2754  |->fff_gcstep:			// Call GC step function.
2755  |  // BASE = new base, RC = nargs*8
2756  |  mflr SAVE0
2757  |   stp BASE, L->base
2758  |  add TMP0, BASE, NARGS8:RC
2759  |   stw PC, SAVE_PC			// Redundant (but a defined value).
2760  |  stp TMP0, L->top
2761  |  mr CARG1, L
2762  |  bl extern lj_gc_step		// (lua_State *L)
2763  |   lp BASE, L->base
2764  |  mtlr SAVE0
2765  |    lp TMP0, L->top
2766  |   sub NARGS8:RC, TMP0, BASE
2767  |   lwz CFUNC:RB, FRAME_FUNC(BASE)
2768  |  blr
2769  |
2770  |//-----------------------------------------------------------------------
2771  |//-- Special dispatch targets -------------------------------------------
2772  |//-----------------------------------------------------------------------
2773  |
2774  |->vm_record:				// Dispatch target for recording phase.
2775  |.if JIT
2776  |  lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
2777  |  andix. TMP0, TMP3, HOOK_VMEVENT	// No recording while in vmevent.
2778  |  bne >5
2779  |  // Decrement the hookcount for consistency, but always do the call.
2780  |   lwz TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2781  |  andix. TMP0, TMP3, HOOK_ACTIVE
2782  |  bne >1
2783  |   subi TMP2, TMP2, 1
2784  |  andi. TMP0, TMP3, LUA_MASKLINE|LUA_MASKCOUNT
2785  |  beqy >1
2786  |   stw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2787  |  b >1
2788  |.endif
2789  |
2790  |->vm_rethook:			// Dispatch target for return hooks.
2791  |  lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
2792  |  andix. TMP0, TMP3, HOOK_ACTIVE	// Hook already active?
2793  |  beq >1
2794  |5:  // Re-dispatch to static ins.
2795  |  addi TMP1, TMP1, GG_DISP2STATIC	// Assumes decode_OPP TMP1, INS.
2796  |  lpx TMP0, DISPATCH, TMP1
2797  |  mtctr TMP0
2798  |  bctr
2799  |
2800  |->vm_inshook:			// Dispatch target for instr/line hooks.
2801  |  lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
2802  |  lwz TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2803  |  andix. TMP0, TMP3, HOOK_ACTIVE	// Hook already active?
2804  |   rlwinm TMP0, TMP3, 31-LUA_HOOKLINE, 31, 0
2805  |  bne <5
2806  |
2807  |   cmpwi cr1, TMP0, 0
2808  |  addic. TMP2, TMP2, -1
2809  |   beq cr1, <5
2810  |  stw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
2811  |  beq >1
2812  |   bge cr1, <5
2813  |1:
2814  |  mr CARG1, L
2815  |   stw MULTRES, SAVE_MULTRES
2816  |  mr CARG2, PC
2817  |   stp BASE, L->base
2818  |  // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
2819  |  bl extern lj_dispatch_ins		// (lua_State *L, const BCIns *pc)
2820  |3:
2821  |  lp BASE, L->base
2822  |4:  // Re-dispatch to static ins.
2823  |  lwz INS, -4(PC)
2824  |  decode_OPP TMP1, INS
2825  |   decode_RB8 RB, INS
2826  |  addi TMP1, TMP1, GG_DISP2STATIC
2827  |   decode_RD8 RD, INS
2828  |  lpx TMP0, DISPATCH, TMP1
2829  |   decode_RA8 RA, INS
2830  |   decode_RC8 RC, INS
2831  |  mtctr TMP0
2832  |  bctr
2833  |
2834  |->cont_hook:				// Continue from hook yield.
2835  |  addi PC, PC, 4
2836  |  lwz MULTRES, -20(RB)		// Restore MULTRES for *M ins.
2837  |  b <4
2838  |
2839  |->vm_hotloop:			// Hot loop counter underflow.
2840  |.if JIT
2841  |  lwz LFUNC:TMP1, FRAME_FUNC(BASE)
2842  |   addi CARG1, DISPATCH, GG_DISP2J
2843  |   stw PC, SAVE_PC
2844  |  lwz TMP1, LFUNC:TMP1->pc
2845  |   mr CARG2, PC
2846  |   stw L, DISPATCH_J(L)(DISPATCH)
2847  |  lbz TMP1, PC2PROTO(framesize)(TMP1)
2848  |   stp BASE, L->base
2849  |  slwi TMP1, TMP1, 3
2850  |  add TMP1, BASE, TMP1
2851  |  stp TMP1, L->top
2852  |  bl extern lj_trace_hot		// (jit_State *J, const BCIns *pc)
2853  |  b <3
2854  |.endif
2855  |
2856  |->vm_callhook:			// Dispatch target for call hooks.
2857  |  mr CARG2, PC
2858  |.if JIT
2859  |  b >1
2860  |.endif
2861  |
2862  |->vm_hotcall:			// Hot call counter underflow.
2863  |.if JIT
2864  |  ori CARG2, PC, 1
2865  |1:
2866  |.endif
2867  |  add TMP0, BASE, RC
2868  |   stw PC, SAVE_PC
2869  |  mr CARG1, L
2870  |   stp BASE, L->base
2871  |  sub RA, RA, BASE
2872  |   stp TMP0, L->top
2873  |  bl extern lj_dispatch_call		// (lua_State *L, const BCIns *pc)
2874  |  // Returns ASMFunction.
2875  |  lp BASE, L->base
2876  |   lp TMP0, L->top
2877  |   stw ZERO, SAVE_PC			// Invalidate for subsequent line hook.
2878  |  sub NARGS8:RC, TMP0, BASE
2879  |  add RA, BASE, RA
2880  |  lwz LFUNC:RB, FRAME_FUNC(BASE)
2881  |  lwz INS, -4(PC)
2882  |  mtctr CRET1
2883  |  bctr
2884  |
2885  |->cont_stitch:			// Trace stitching.
2886  |.if JIT
2887  |  // RA = resultptr, RB = meta base
2888  |  lwz INS, -4(PC)
2889  |    lwz TRACE:TMP2, -20(RB)		// Save previous trace.
2890  |   addic. TMP1, MULTRES, -8
2891  |  decode_RA8 RC, INS			// Call base.
2892  |   beq >2
2893  |1:  // Move results down.
2894  |.if FPU
2895  |  lfd f0, 0(RA)
2896  |.else
2897  |  lwz CARG1, 0(RA)
2898  |  lwz CARG2, 4(RA)
2899  |.endif
2900  |   addic. TMP1, TMP1, -8
2901  |    addi RA, RA, 8
2902  |.if FPU
2903  |  stfdx f0, BASE, RC
2904  |.else
2905  |  add CARG3, BASE, RC
2906  |  stw CARG1, 0(CARG3)
2907  |  stw CARG2, 4(CARG3)
2908  |.endif
2909  |    addi RC, RC, 8
2910  |   bne <1
2911  |2:
2912  |   decode_RA8 RA, INS
2913  |   decode_RB8 RB, INS
2914  |   add RA, RA, RB
2915  |3:
2916  |   cmplw RA, RC
2917  |   bgt >9				// More results wanted?
2918  |
2919  |  lhz TMP3, TRACE:TMP2->traceno
2920  |  lhz RD, TRACE:TMP2->link
2921  |  cmpw RD, TMP3
2922  |   cmpwi cr1, RD, 0
2923  |  beq ->cont_nop			// Blacklisted.
2924  |    slwi RD, RD, 3
2925  |   bne cr1, =>BC_JLOOP		// Jump to stitched trace.
2926  |
2927  |  // Stitch a new trace to the previous trace.
2928  |  stw TMP3, DISPATCH_J(exitno)(DISPATCH)
2929  |  stp L, DISPATCH_J(L)(DISPATCH)
2930  |  stp BASE, L->base
2931  |  addi CARG1, DISPATCH, GG_DISP2J
2932  |  mr CARG2, PC
2933  |  bl extern lj_dispatch_stitch	// (jit_State *J, const BCIns *pc)
2934  |  lp BASE, L->base
2935  |  b ->cont_nop
2936  |
2937  |9:
2938  |  stwx TISNIL, BASE, RC
2939  |  addi RC, RC, 8
2940  |  b <3
2941  |.endif
2942  |
2943  |->vm_profhook:			// Dispatch target for profiler hook.
2944#if LJ_HASPROFILE
2945  |  mr CARG1, L
2946  |   stw MULTRES, SAVE_MULTRES
2947  |  mr CARG2, PC
2948  |   stp BASE, L->base
2949  |  bl extern lj_dispatch_profile	// (lua_State *L, const BCIns *pc)
2950  |  // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2951  |  lp BASE, L->base
2952  |  subi PC, PC, 4
2953  |  b ->cont_nop
2954#endif
2955  |
2956  |//-----------------------------------------------------------------------
2957  |//-- Trace exit handler -------------------------------------------------
2958  |//-----------------------------------------------------------------------
2959  |
2960  |.macro savex_, a, b, c, d
2961  |.if FPU
2962  |  stfd f..a, 16+a*8(sp)
2963  |  stfd f..b, 16+b*8(sp)
2964  |  stfd f..c, 16+c*8(sp)
2965  |  stfd f..d, 16+d*8(sp)
2966  |.endif
2967  |.endmacro
2968  |
2969  |->vm_exit_handler:
2970  |.if JIT
2971  |  addi sp, sp, -(16+32*8+32*4)
2972  |  stmw r2, 16+32*8+2*4(sp)
2973  |    addi DISPATCH, JGL, -GG_DISP2G-32768
2974  |    li CARG2, ~LJ_VMST_EXIT
2975  |   lwz CARG1, 16+32*8+32*4(sp)	// Get stack chain.
2976  |    stw CARG2, DISPATCH_GL(vmstate)(DISPATCH)
2977  |  savex_ 0,1,2,3
2978  |   stw CARG1, 0(sp)			// Store extended stack chain.
2979  |   clrso TMP1
2980  |  savex_ 4,5,6,7
2981  |   addi CARG2, sp, 16+32*8+32*4	// Recompute original value of sp.
2982  |  savex_ 8,9,10,11
2983  |   stw CARG2, 16+32*8+1*4(sp)	// Store sp in RID_SP.
2984  |  savex_ 12,13,14,15
2985  |   mflr CARG3
2986  |   li TMP1, 0
2987  |  savex_ 16,17,18,19
2988  |   stw TMP1, 16+32*8+0*4(sp)		// Clear RID_TMP.
2989  |  savex_ 20,21,22,23
2990  |   lhz CARG4, 2(CARG3)		// Load trace number.
2991  |  savex_ 24,25,26,27
2992  |  lwz L, DISPATCH_GL(cur_L)(DISPATCH)
2993  |  savex_ 28,29,30,31
2994  |   sub CARG3, TMP0, CARG3		// Compute exit number.
2995  |  lp BASE, DISPATCH_GL(jit_base)(DISPATCH)
2996  |   srwi CARG3, CARG3, 2
2997  |  stp L, DISPATCH_J(L)(DISPATCH)
2998  |   subi CARG3, CARG3, 2
2999  |  stp BASE, L->base
3000  |   stw CARG4, DISPATCH_J(parent)(DISPATCH)
3001  |  stw TMP1, DISPATCH_GL(jit_base)(DISPATCH)
3002  |  addi CARG1, DISPATCH, GG_DISP2J
3003  |   stw CARG3, DISPATCH_J(exitno)(DISPATCH)
3004  |  addi CARG2, sp, 16
3005  |  bl extern lj_trace_exit		// (jit_State *J, ExitState *ex)
3006  |  // Returns MULTRES (unscaled) or negated error code.
3007  |  lp TMP1, L->cframe
3008  |  lwz TMP2, 0(sp)
3009  |   lp BASE, L->base
3010  |.if GPR64
3011  |  rldicr sp, TMP1, 0, 61
3012  |.else
3013  |  rlwinm sp, TMP1, 0, 0, 29
3014  |.endif
3015  |   lwz PC, SAVE_PC			// Get SAVE_PC.
3016  |  stw TMP2, 0(sp)
3017  |  stw L, SAVE_L			// Set SAVE_L (on-trace resume/yield).
3018  |  b >1
3019  |.endif
3020  |->vm_exit_interp:
3021  |.if JIT
3022  |  // CARG1 = MULTRES or negated error code, BASE, PC and JGL set.
3023  |  lwz L, SAVE_L
3024  |  addi DISPATCH, JGL, -GG_DISP2G-32768
3025  |  stp BASE, L->base
3026  |1:
3027  |  li TMP2, -LUA_ERRERR
3028  |  cmplw CARG1, TMP2
3029  |  bge >9				// Check for error from exit.
3030  |  lwz LFUNC:RB, FRAME_FUNC(BASE)
3031  |   slwi MULTRES, CARG1, 3
3032  |    li TMP2, 0
3033  |   stw MULTRES, SAVE_MULTRES
3034  |  lwz TMP1, LFUNC:RB->pc
3035  |    stw TMP2, DISPATCH_GL(jit_base)(DISPATCH)
3036  |  lwz KBASE, PC2PROTO(k)(TMP1)
3037  |  // Setup type comparison constants.
3038  |  li TISNUM, LJ_TISNUM
3039  |  .FPU lus TMP3, 0x59c0		// TOBIT = 2^52 + 2^51 (float).
3040  |  .FPU stw TMP3, TMPD
3041  |  li ZERO, 0
3042  |  .FPU ori TMP3, TMP3, 0x0004	// TONUM = 2^52 + 2^51 + 2^31 (float).
3043  |  .FPU lfs TOBIT, TMPD
3044  |  .FPU stw TMP3, TMPD
3045  |  .FPU lus TMP0, 0x4338			// Hiword of 2^52 + 2^51 (double)
3046  |    li TISNIL, LJ_TNIL
3047  |  .FPU stw TMP0, TONUM_HI
3048  |  .FPU lfs TONUM, TMPD
3049  |  // Modified copy of ins_next which handles function header dispatch, too.
3050  |  lwz INS, 0(PC)
3051  |   addi PC, PC, 4
3052  |    // Assumes TISNIL == ~LJ_VMST_INTERP == -1.
3053  |    stw TISNIL, DISPATCH_GL(vmstate)(DISPATCH)
3054  |  cmpwi CARG1, -17			// Static dispatch?
3055  |  beq >5
3056  |  decode_OPP TMP1, INS
3057  |   decode_RA8 RA, INS
3058  |  lpx TMP0, DISPATCH, TMP1
3059  |  mtctr TMP0
3060  |  cmplwi TMP1, BC_FUNCF*4		// Function header?
3061  |  bge >2
3062  |   decode_RB8 RB, INS
3063  |   decode_RD8 RD, INS
3064  |   decode_RC8 RC, INS
3065  |  bctr
3066  |2:
3067  |  cmplwi TMP1, (BC_FUNCC+2)*4	// Fast function?
3068  |  blt >3
3069  |  // Check frame below fast function.
3070  |  lwz TMP1, FRAME_PC(BASE)
3071  |  andix. TMP0, TMP1, FRAME_TYPE
3072  |  bney >3				// Trace stitching continuation?
3073  |  // Otherwise set KBASE for Lua function below fast function.
3074  |  lwz TMP2, -4(TMP1)
3075  |  decode_RA8 TMP0, TMP2
3076  |  sub TMP1, BASE, TMP0
3077  |  lwz LFUNC:TMP2, -12(TMP1)
3078  |  lwz TMP1, LFUNC:TMP2->pc
3079  |  lwz KBASE, PC2PROTO(k)(TMP1)
3080  |3:
3081  |   subi RC, MULTRES, 8
3082  |   add RA, RA, BASE
3083  |  bctr
3084  |
3085  |5:  // Dispatch to static entry of original ins replaced by BC_JLOOP.
3086  |  lwz TMP1, DISPATCH_J(trace)(DISPATCH)
3087  |  decode_RD4 RD, INS
3088  |  lwzx TRACE:TMP1, TMP1, RD
3089  |  lwz INS, TRACE:TMP1->startins
3090  |  decode_OPP TMP1, INS
3091  |  addi TMP1, TMP1, GG_DISP2STATIC
3092  |  lpx TMP0, DISPATCH, TMP1
3093  |  mtctr TMP0
3094  |   decode_RB8 RB, INS
3095  |   decode_RD8 RD, INS
3096  |   decode_RA8 RA, INS
3097  |   decode_RC8 RC, INS
3098  |  bctr
3099  |
3100  |9:  // Rethrow error from the right C frame.
3101  |  neg CARG2, CARG1
3102  |  mr CARG1, L
3103  |  bl extern lj_err_trace		// (lua_State *L, int errcode)
3104  |.endif
3105  |
3106  |//-----------------------------------------------------------------------
3107  |//-- Math helper functions ----------------------------------------------
3108  |//-----------------------------------------------------------------------
3109  |
3110  |// NYI: Use internal implementations of floor, ceil, trunc, sfcmp.
3111  |
3112  |.macro sfi2d, AHI, ALO
3113  |.if not FPU
3114  |  mr. AHI, ALO
3115  |  bclr 12, 2				// Handle zero first.
3116  |  srawi TMP0, ALO, 31
3117  |  xor TMP1, ALO, TMP0
3118  |  sub TMP1, TMP1, TMP0		// Absolute value in TMP1.
3119  |  cntlzw AHI, TMP1
3120  |  andix. TMP0, TMP0, 0x800		// Mask sign bit.
3121  |  slw TMP1, TMP1, AHI		// Align mantissa left with leading 1.
3122  |  subfic AHI, AHI, 0x3ff+31-1	// Exponent -1 in AHI.
3123  |  slwi ALO, TMP1, 21
3124  |  or AHI, AHI, TMP0			// Sign | Exponent.
3125  |  srwi TMP1, TMP1, 11
3126  |  slwi AHI, AHI, 20			// Align left.
3127  |  add AHI, AHI, TMP1			// Add mantissa, increment exponent.
3128  |  blr
3129  |.endif
3130  |.endmacro
3131  |
3132  |// Input: CARG2. Output: CARG1, CARG2. Temporaries: TMP0, TMP1.
3133  |->vm_sfi2d_1:
3134  |  sfi2d CARG1, CARG2
3135  |
3136  |// Input: CARG4. Output: CARG3, CARG4. Temporaries: TMP0, TMP1.
3137  |->vm_sfi2d_2:
3138  |  sfi2d CARG3, CARG4
3139  |
3140  |->vm_modi:
3141  |  divwo. TMP0, CARG1, CARG2
3142  |  bso >1
3143  |.if GPR64
3144  |   xor CARG3, CARG1, CARG2
3145  |   cmpwi CARG3, 0
3146  |.else
3147  |   xor. CARG3, CARG1, CARG2
3148  |.endif
3149  |  mullw TMP0, TMP0, CARG2
3150  |  sub CARG1, CARG1, TMP0
3151  |   bgelr
3152  |  cmpwi CARG1, 0; beqlr
3153  |  add CARG1, CARG1, CARG2
3154  |  blr
3155  |1:
3156  |  cmpwi CARG2, 0
3157  |   li CARG1, 0
3158  |  beqlr
3159  |  clrso TMP0			// Clear SO for -2147483648 % -1 and return 0.
3160  |  blr
3161  |
3162  |//-----------------------------------------------------------------------
3163  |//-- Number conversion functions ----------------------------------------
3164  |//-----------------------------------------------------------------------
3165  |
3166  |// int64_t lj_vm_num2int_check(double x)
3167  |->vm_num2int_check:
3168  |.if FPU
3169  |  subi sp, sp, 16
3170  |  stfd FARG1, 0(sp)
3171  |  lwz CARG1, 0(sp)
3172  |  lwz CARG2, 4(sp)
3173  |.endif
3174  |  slwi TMP1, CARG1, 1
3175  |.if PPE
3176  |  or TMP1, TMP1, CARG2
3177  |  cmpwi TMP1, 0
3178  |.else
3179  |  or. TMP1, TMP1, CARG2
3180  |.endif
3181  |  beq >2				// Return 0 for +-0.
3182  |  rlwinm RB, CARG1, 12, 21, 31
3183  |  subfic RB, RB, 1054
3184  |  cmplwi RB, 32
3185  |  bge >1				// Fail if |x| < 0x1p0 || |x| >= 0x1p32.
3186  |  slwi CARG3, CARG1, 11
3187  |  rlwimi CARG3, CARG2, 11, 21, 31	// Left-aligned mantissa.
3188  |  subfic TMP1, RB, 32
3189  |  slw TMP1, CARG3, TMP1
3190  |  slwi TMP2, CARG2, 11
3191  |.if PPE
3192  |  or. TMP1, TMP1, TMP2
3193  |.else
3194  |  or TMP1, TMP1, TMP2
3195  |  cmpwi TMP1, 0
3196  |.endif
3197  |  bne >1				// Fail if fractional part != 0.
3198  |  oris CARG3, CARG3, 0x8000		// Merge leading 1.
3199  |  srw CRET2, CARG3, RB		// lo = right-aligned absolute value.
3200  |  srawi CARG4, CARG1, 31		// sign = 0 or -1.
3201  |.if GPR64
3202  |  add CRET2, CRET2, CARG4
3203  |  cmpwi CRET2, 0
3204  |.else
3205  |  add. CRET2, CRET2, CARG4
3206  |.endif
3207  |  blt >1				// Fail if fractional part != 0.
3208  |  xor CRET2, CRET2, CARG4		// lo = sign?-lo:lo = (lo+sign)^sign.
3209  |2:
3210  |.if GPR64
3211  |  rldicl CRET1, CRET1, 0, 32
3212  |.else
3213  |  li CRET1, 0
3214  |.endif
3215  |.if FPU
3216  |  addi sp, sp, 16
3217  |.endif
3218  |  blr
3219  |1:
3220  |.if GPR64
3221  |  lus CRET1, 0x8000
3222  |  rldicr CRET1, CRET1, 32, 32
3223  |.else
3224  |  lus CRET1, 0x8000
3225  |  lus CRET2, 0x8000
3226  |.endif
3227  |.if FPU
3228  |  addi sp, sp, 16
3229  |.endif
3230  |  blr
3231  |
3232  |// int64_t lj_vm_num2i64(double x)
3233  |->vm_num2i64:
3234  |// fallthrough, same as lj_vm_num2u64.
3235  |
3236  |// uint64_t lj_vm_num2u64(double x)
3237  |->vm_num2u64:
3238  |.if FPU
3239  |  subi sp, sp, 16
3240  |  stfd FARG1, 0(sp)
3241  |  lwz CARG1, 0(sp)
3242  |  lwz CARG2, 4(sp)
3243  |.endif
3244  |  rlwinm RB, CARG1, 12, 21, 31
3245  |  addi RB, RB, -1023
3246  |  cmplwi RB, 116
3247  |  bge >3				// Exponent out of range.
3248  |  srawi CARG4, CARG1, 31		// sign = 0 or -1.
3249  |  clrlwi CARG1, CARG1, 12
3250  |   subfic RB, RB, 52
3251  |  oris CARG1, CARG1, 0x0010
3252  |   cmpwi RB, 0
3253  |   blt >2				// Shift mantissa left or right?
3254  |   subfic TMP1, RB, 32		// 64 bit right shift.
3255  |  srw CARG2, CARG2, RB
3256  |   slw TMP2, CARG1, TMP1
3257  |   addi TMP1, RB, -32
3258  |  or CARG2, CARG2, TMP2
3259  |   srw TMP2, CARG1, TMP1
3260  |  or CARG2, CARG2, TMP2
3261  |  srw CARG1, CARG1, RB
3262  |1:
3263  |  addc CARG2, CARG2, CARG4
3264  |  adde CARG1, CARG1, CARG4
3265  |  xor CRET2, CARG2, CARG4
3266  |  xor CRET1, CARG1, CARG4
3267  |.if GPR64
3268  |  rldimi CRET2, CRET1, 0, 32
3269  |  mr CRET1, CRET2
3270  |.endif
3271  |.if FPU
3272  |  addi sp, sp, 16
3273  |.endif
3274  |  blr
3275  |2:
3276  |  subfic TMP1, RB, 0			// 64 bit left shift.
3277  |   addi RB, RB, -32
3278  |  slw CARG1, CARG1, TMP1
3279  |   srw TMP2, CARG2, RB
3280  |   addi RB, TMP1, -32
3281  |  or CARG1, CARG1, TMP2
3282  |   slw TMP2, CARG2, RB
3283  |  or CARG1, CARG1, TMP2
3284  |  slw CARG2, CARG2, TMP1
3285  |  b <1
3286  |3:
3287  |  li CRET1, 0
3288  |.if not GPR64
3289  |  li CRET2, 0
3290  |.endif
3291  |.if FPU
3292  |  addi sp, sp, 16
3293  |.endif
3294  |  blr
3295  |
3296  |// int32_t lj_vm_tobit(double x)
3297  |.if FPU
3298  |->vm_tobit:
3299  |  lus TMP0, 0x59c0			// 2^52 + 2^51 (float).
3300  |  subi sp, sp, 16
3301  |  stw TMP0, 0(sp)
3302  |  lfs FARG2, 0(sp)
3303  |  fadd FARG1, FARG1, FARG2
3304  |  stfd FARG1, 0(sp)
3305  |  lwz CRET1, 4(sp)
3306  |  addi sp, sp, 16
3307  |  blr
3308  |.endif
3309  |
3310  |//-----------------------------------------------------------------------
3311  |//-- Miscellaneous functions --------------------------------------------
3312  |//-----------------------------------------------------------------------
3313  |
3314  |// void lj_vm_cachesync(void *start, void *end)
3315  |// Flush D-Cache and invalidate I-Cache. Assumes 32 byte cache line size.
3316  |// This is a good lower bound, except for very ancient PPC models.
3317  |->vm_cachesync:
3318  |.if JIT or FFI
3319  |  // Compute start of first cache line and number of cache lines.
3320  |  rlwinm CARG1, CARG1, 0, 0, 26
3321  |  sub CARG2, CARG2, CARG1
3322  |  addi CARG2, CARG2, 31
3323  |  rlwinm. CARG2, CARG2, 27, 5, 31
3324  |  beqlr
3325  |  mtctr CARG2
3326  |  mr CARG3, CARG1
3327  |1:  // Flush D-Cache.
3328  |  dcbst r0, CARG1
3329  |  addi CARG1, CARG1, 32
3330  |  bdnz <1
3331  |  sync
3332  |  mtctr CARG2
3333  |1:  // Invalidate I-Cache.
3334  |  icbi r0, CARG3
3335  |  addi CARG3, CARG3, 32
3336  |  bdnz <1
3337  |  isync
3338  |  blr
3339  |.endif
3340  |
3341  |->vm_next:
3342  |.if JIT
3343  |  NYI  // On big-endian.
3344  |.endif
3345  |
3346  |//-----------------------------------------------------------------------
3347  |//-- FFI helper functions -----------------------------------------------
3348  |//-----------------------------------------------------------------------
3349  |
3350  |// Handler for callback functions. Callback slot number in r11, g in r12.
3351  |->vm_ffi_callback:
3352  |.if FFI
3353  |.type CTSTATE, CTState, PC
3354  |  saveregs
3355  |  lwz CTSTATE, GL:r12->ctype_state
3356  |   addi DISPATCH, r12, GG_G2DISP
3357  |  stw r11, CTSTATE->cb.slot
3358  |  stw r3, CTSTATE->cb.gpr[0]
3359  |   .FPU stfd f1, CTSTATE->cb.fpr[0]
3360  |  stw r4, CTSTATE->cb.gpr[1]
3361  |   .FPU stfd f2, CTSTATE->cb.fpr[1]
3362  |  stw r5, CTSTATE->cb.gpr[2]
3363  |   .FPU stfd f3, CTSTATE->cb.fpr[2]
3364  |  stw r6, CTSTATE->cb.gpr[3]
3365  |   .FPU stfd f4, CTSTATE->cb.fpr[3]
3366  |  stw r7, CTSTATE->cb.gpr[4]
3367  |   .FPU stfd f5, CTSTATE->cb.fpr[4]
3368  |  stw r8, CTSTATE->cb.gpr[5]
3369  |   .FPU stfd f6, CTSTATE->cb.fpr[5]
3370  |  stw r9, CTSTATE->cb.gpr[6]
3371  |   .FPU stfd f7, CTSTATE->cb.fpr[6]
3372  |  stw r10, CTSTATE->cb.gpr[7]
3373  |   .FPU stfd f8, CTSTATE->cb.fpr[7]
3374  |  addi TMP0, sp, CFRAME_SPACE+8
3375  |  stw TMP0, CTSTATE->cb.stack
3376  |   mr CARG1, CTSTATE
3377  |  stw CTSTATE, SAVE_PC		// Any value outside of bytecode is ok.
3378  |   mr CARG2, sp
3379  |  bl extern lj_ccallback_enter	// (CTState *cts, void *cf)
3380  |  // Returns lua_State *.
3381  |  lp BASE, L:CRET1->base
3382  |     li TISNUM, LJ_TISNUM		// Setup type comparison constants.
3383  |  lp RC, L:CRET1->top
3384  |     .FPU lus TMP3, 0x59c0		// TOBIT = 2^52 + 2^51 (float).
3385  |     li ZERO, 0
3386  |   mr L, CRET1
3387  |     .FPU stw TMP3, TMPD
3388  |     .FPU lus TMP0, 0x4338		// Hiword of 2^52 + 2^51 (double)
3389  |  lwz LFUNC:RB, FRAME_FUNC(BASE)
3390  |     .FPU ori TMP3, TMP3, 0x0004	// TONUM = 2^52 + 2^51 + 2^31 (float).
3391  |     .FPU stw TMP0, TONUM_HI
3392  |     li TISNIL, LJ_TNIL
3393  |    li_vmstate INTERP
3394  |     .FPU lfs TOBIT, TMPD
3395  |     .FPU stw TMP3, TMPD
3396  |  sub RC, RC, BASE
3397  |    st_vmstate
3398  |     .FPU lfs TONUM, TMPD
3399  |  ins_callt
3400  |.endif
3401  |
3402  |->cont_ffi_callback:			// Return from FFI callback.
3403  |.if FFI
3404  |  lwz CTSTATE, DISPATCH_GL(ctype_state)(DISPATCH)
3405  |   stp BASE, L->base
3406  |   stp RB, L->top
3407  |  stp L, CTSTATE->L
3408  |  mr CARG1, CTSTATE
3409  |  mr CARG2, RA
3410  |  bl extern lj_ccallback_leave	// (CTState *cts, TValue *o)
3411  |  lwz CRET1, CTSTATE->cb.gpr[0]
3412  |  .FPU lfd FARG1, CTSTATE->cb.fpr[0]
3413  |  lwz CRET2, CTSTATE->cb.gpr[1]
3414  |  b ->vm_leave_unw
3415  |.endif
3416  |
3417  |->vm_ffi_call:			// Call C function via FFI.
3418  |  // Caveat: needs special frame unwinding, see below.
3419  |.if FFI
3420  |  .type CCSTATE, CCallState, CARG1
3421  |  lwz TMP1, CCSTATE->spadj
3422  |    mflr TMP0
3423  |   lbz CARG2, CCSTATE->nsp
3424  |   lbz CARG3, CCSTATE->nfpr
3425  |  neg TMP1, TMP1
3426  |    stw TMP0, 4(sp)
3427  |   cmpwi cr1, CARG3, 0
3428  |  mr TMP2, sp
3429  |   addic. CARG2, CARG2, -4
3430  |  stwux sp, sp, TMP1
3431  |   crnot 4*cr1+eq, 4*cr1+eq		// For vararg calls.
3432  |  stw r14, -4(TMP2)
3433  |  stw CCSTATE, -8(TMP2)
3434  |  mr r14, TMP2
3435  |  la TMP1, CCSTATE->stack
3436  |   blty >2
3437  |  la TMP2, 8(sp)
3438  |1:
3439  |  lwzx TMP0, TMP1, CARG2
3440  |  stwx TMP0, TMP2, CARG2
3441  |   addic. CARG2, CARG2, -4
3442  |  bge <1
3443  |2:
3444  |  bney cr1, >3
3445  |  .FPU lfd f1, CCSTATE->fpr[0]
3446  |  .FPU lfd f2, CCSTATE->fpr[1]
3447  |  .FPU lfd f3, CCSTATE->fpr[2]
3448  |  .FPU lfd f4, CCSTATE->fpr[3]
3449  |  .FPU lfd f5, CCSTATE->fpr[4]
3450  |  .FPU lfd f6, CCSTATE->fpr[5]
3451  |  .FPU lfd f7, CCSTATE->fpr[6]
3452  |  .FPU lfd f8, CCSTATE->fpr[7]
3453  |3:
3454  |   lp TMP0, CCSTATE->func
3455  |  lwz CARG2, CCSTATE->gpr[1]
3456  |  lwz CARG3, CCSTATE->gpr[2]
3457  |  lwz CARG4, CCSTATE->gpr[3]
3458  |  lwz CARG5, CCSTATE->gpr[4]
3459  |   mtctr TMP0
3460  |  lwz r8, CCSTATE->gpr[5]
3461  |  lwz r9, CCSTATE->gpr[6]
3462  |  lwz r10, CCSTATE->gpr[7]
3463  |  lwz CARG1, CCSTATE->gpr[0]		// Do this last, since CCSTATE is CARG1.
3464  |   bctrl
3465  |  lwz CCSTATE:TMP1, -8(r14)
3466  |  lwz TMP2, -4(r14)
3467  |   lwz TMP0, 4(r14)
3468  |  stw CARG1, CCSTATE:TMP1->gpr[0]
3469  |  .FPU stfd FARG1, CCSTATE:TMP1->fpr[0]
3470  |  stw CARG2, CCSTATE:TMP1->gpr[1]
3471  |   mtlr TMP0
3472  |  stw CARG3, CCSTATE:TMP1->gpr[2]
3473  |   mr sp, r14
3474  |  stw CARG4, CCSTATE:TMP1->gpr[3]
3475  |   mr r14, TMP2
3476  |  blr
3477  |.endif
3478  |// Note: vm_ffi_call must be the last function in this object file!
3479  |
3480  |//-----------------------------------------------------------------------
3481}
3482
3483/* Generate the code for a single instruction. */
3484static void build_ins(BuildCtx *ctx, BCOp op, int defop)
3485{
3486  int vk = 0;
3487  |=>defop:
3488
3489  switch (op) {
3490
3491  /* -- Comparison ops ---------------------------------------------------- */
3492
3493  /* Remember: all ops branch for a true comparison, fall through otherwise. */
3494
3495  case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
3496    |  // RA = src1*8, RD = src2*8, JMP with RD = target
3497    |.if DUALNUM
3498    |  lwzux CARG1, RA, BASE
3499    |    addi PC, PC, 4
3500    |   lwz CARG2, 4(RA)
3501    |  lwzux CARG3, RD, BASE
3502    |    lwz TMP2, -4(PC)
3503    |  checknum cr0, CARG1
3504    |   lwz CARG4, 4(RD)
3505    |    decode_RD4 TMP2, TMP2
3506    |  checknum cr1, CARG3
3507    |    addis SAVE0, TMP2, -(BCBIAS_J*4 >> 16)
3508    |  bne cr0, >7
3509    |  bne cr1, >8
3510    |   cmpw CARG2, CARG4
3511    if (op == BC_ISLT) {
3512      |  bge >2
3513    } else if (op == BC_ISGE) {
3514      |  blt >2
3515    } else if (op == BC_ISLE) {
3516      |  bgt >2
3517    } else {
3518      |  ble >2
3519    }
3520    |1:
3521    |  add PC, PC, SAVE0
3522    |2:
3523    |  ins_next
3524    |
3525    |7:  // RA is not an integer.
3526    |  bgt cr0, ->vmeta_comp
3527    |  // RA is a number.
3528    |   .FPU lfd f0, 0(RA)
3529    |  bgt cr1, ->vmeta_comp
3530    |  blt cr1, >4
3531    |  // RA is a number, RD is an integer.
3532    |.if FPU
3533    |  tonum_i f1, CARG4
3534    |.else
3535    |  bl ->vm_sfi2d_2
3536    |.endif
3537    |  b >5
3538    |
3539    |8: // RA is an integer, RD is not an integer.
3540    |  bgt cr1, ->vmeta_comp
3541    |  // RA is an integer, RD is a number.
3542    |.if FPU
3543    |  tonum_i f0, CARG2
3544    |.else
3545    |  bl ->vm_sfi2d_1
3546    |.endif
3547    |4:
3548    |  .FPU lfd f1, 0(RD)
3549    |5:
3550    |.if FPU
3551    |  fcmpu cr0, f0, f1
3552    |.else
3553    |  blex __ledf2
3554    |  cmpwi CRET1, 0
3555    |.endif
3556    if (op == BC_ISLT) {
3557      |  bge <2
3558    } else if (op == BC_ISGE) {
3559      |  blt <2
3560    } else if (op == BC_ISLE) {
3561      |  cror 4*cr0+lt, 4*cr0+lt, 4*cr0+eq
3562      |  bge <2
3563    } else {
3564      |  cror 4*cr0+lt, 4*cr0+lt, 4*cr0+eq
3565      |  blt <2
3566    }
3567    |  b <1
3568    |.else
3569    |  lwzx TMP0, BASE, RA
3570    |    addi PC, PC, 4
3571    |   lfdx f0, BASE, RA
3572    |  lwzx TMP1, BASE, RD
3573    |  checknum cr0, TMP0
3574    |    lwz TMP2, -4(PC)
3575    |   lfdx f1, BASE, RD
3576    |  checknum cr1, TMP1
3577    |    decode_RD4 TMP2, TMP2
3578    |  bge cr0, ->vmeta_comp
3579    |    addis TMP2, TMP2, -(BCBIAS_J*4 >> 16)
3580    |  bge cr1, ->vmeta_comp
3581    |  fcmpu cr0, f0, f1
3582    if (op == BC_ISLT) {
3583      |  bge >1
3584    } else if (op == BC_ISGE) {
3585      |  blt >1
3586    } else if (op == BC_ISLE) {
3587      |  cror 4*cr0+lt, 4*cr0+lt, 4*cr0+eq
3588      |  bge >1
3589    } else {
3590      |  cror 4*cr0+lt, 4*cr0+lt, 4*cr0+eq
3591      |  blt >1
3592    }
3593    |  add PC, PC, TMP2
3594    |1:
3595    |  ins_next
3596    |.endif
3597    break;
3598
3599  case BC_ISEQV: case BC_ISNEV:
3600    vk = op == BC_ISEQV;
3601    |  // RA = src1*8, RD = src2*8, JMP with RD = target
3602    |.if DUALNUM
3603    |  lwzux CARG1, RA, BASE
3604    |    addi PC, PC, 4
3605    |   lwz CARG2, 4(RA)
3606    |  lwzux CARG3, RD, BASE
3607    |  checknum cr0, CARG1
3608    |    lwz SAVE0, -4(PC)
3609    |  checknum cr1, CARG3
3610    |    decode_RD4 SAVE0, SAVE0
3611    |   lwz CARG4, 4(RD)
3612    |  cror 4*cr7+gt, 4*cr0+gt, 4*cr1+gt
3613    |    addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3614    if (vk) {
3615      |  ble cr7, ->BC_ISEQN_Z
3616    } else {
3617      |  ble cr7, ->BC_ISNEN_Z
3618    }
3619    |.else
3620    |  lwzux CARG1, RA, BASE
3621    |   lwz SAVE0, 0(PC)
3622    |    lfd f0, 0(RA)
3623    |   addi PC, PC, 4
3624    |  lwzux CARG3, RD, BASE
3625    |  checknum cr0, CARG1
3626    |   decode_RD4 SAVE0, SAVE0
3627    |    lfd f1, 0(RD)
3628    |  checknum cr1, CARG3
3629    |   addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3630    |  bge cr0, >5
3631    |  bge cr1, >5
3632    |  fcmpu cr0, f0, f1
3633    if (vk) {
3634      |  bne >1
3635      |  add PC, PC, SAVE0
3636    } else {
3637      |  beq >1
3638      |  add PC, PC, SAVE0
3639    }
3640    |1:
3641    |  ins_next
3642    |.endif
3643    |5:  // Either or both types are not numbers.
3644    |.if not DUALNUM
3645    |    lwz CARG2, 4(RA)
3646    |    lwz CARG4, 4(RD)
3647    |.endif
3648    |.if FFI
3649    |  cmpwi cr7, CARG1, LJ_TCDATA
3650    |  cmpwi cr5, CARG3, LJ_TCDATA
3651    |.endif
3652    |   not TMP2, CARG1
3653    |  cmplw CARG1, CARG3
3654    |   cmplwi cr1, TMP2, ~LJ_TISPRI		// Primitive?
3655    |.if FFI
3656    |  cror 4*cr7+eq, 4*cr7+eq, 4*cr5+eq
3657    |.endif
3658    |   cmplwi cr6, TMP2, ~LJ_TISTABUD		// Table or userdata?
3659    |.if FFI
3660    |  beq cr7, ->vmeta_equal_cd
3661    |.endif
3662    |    cmplw cr5, CARG2, CARG4
3663    |  crandc 4*cr0+gt, 4*cr0+eq, 4*cr1+gt	// 2: Same type and primitive.
3664    |  crorc 4*cr0+lt, 4*cr5+eq, 4*cr0+eq	// 1: Same tv or different type.
3665    |  crand 4*cr0+eq, 4*cr0+eq, 4*cr5+eq	// 0: Same type and same tv.
3666    |   mr SAVE1, PC
3667    |  cror 4*cr0+eq, 4*cr0+eq, 4*cr0+gt	// 0 or 2.
3668    |  cror 4*cr0+lt, 4*cr0+lt, 4*cr0+gt	// 1 or 2.
3669    if (vk) {
3670      |  bne cr0, >6
3671      |  add PC, PC, SAVE0
3672      |6:
3673    } else {
3674      |  beq cr0, >6
3675      |  add PC, PC, SAVE0
3676      |6:
3677    }
3678    |.if DUALNUM
3679    |  bge cr0, >2			// Done if 1 or 2.
3680    |1:
3681    |  ins_next
3682    |2:
3683    |.else
3684    |  blt cr0, <1			// Done if 1 or 2.
3685    |.endif
3686    |  blt cr6, <1			// Done if not tab/ud.
3687    |
3688    |  // Different tables or userdatas. Need to check __eq metamethod.
3689    |  // Field metatable must be at same offset for GCtab and GCudata!
3690    |   mr CARG3, CARG4
3691    |  lwz TAB:TMP2, TAB:CARG2->metatable
3692    |   li CARG4, 1-vk			// ne = 0 or 1.
3693    |  cmplwi TAB:TMP2, 0
3694    |  beq <1				// No metatable?
3695    |  lbz TMP2, TAB:TMP2->nomm
3696    |  andix. TMP2, TMP2, 1<<MM_eq
3697    |  bne <1				// Or 'no __eq' flag set?
3698    |  mr PC, SAVE1			// Restore old PC.
3699    |  b ->vmeta_equal			// Handle __eq metamethod.
3700    break;
3701
3702  case BC_ISEQS: case BC_ISNES:
3703    vk = op == BC_ISEQS;
3704    |  // RA = src*8, RD = str_const*8 (~), JMP with RD = target
3705    |  lwzux TMP0, RA, BASE
3706    |   srwi RD, RD, 1
3707    |  lwz STR:TMP3, 4(RA)
3708    |    lwz TMP2, 0(PC)
3709    |   subfic RD, RD, -4
3710    |    addi PC, PC, 4
3711    |.if FFI
3712    |  cmpwi TMP0, LJ_TCDATA
3713    |.endif
3714    |   lwzx STR:TMP1, KBASE, RD	// KBASE-4-str_const*4
3715    |  .gpr64 extsw TMP0, TMP0
3716    |  subfic TMP0, TMP0, LJ_TSTR
3717    |.if FFI
3718    |  beq ->vmeta_equal_cd
3719    |.endif
3720    |  sub TMP1, STR:TMP1, STR:TMP3
3721    |  or TMP0, TMP0, TMP1
3722    |    decode_RD4 TMP2, TMP2
3723    |  subfic TMP0, TMP0, 0
3724    |    addis TMP2, TMP2, -(BCBIAS_J*4 >> 16)
3725    |  subfe TMP1, TMP1, TMP1
3726    if (vk) {
3727      |  andc TMP2, TMP2, TMP1
3728    } else {
3729      |  and TMP2, TMP2, TMP1
3730    }
3731    |  add PC, PC, TMP2
3732    |  ins_next
3733    break;
3734
3735  case BC_ISEQN: case BC_ISNEN:
3736    vk = op == BC_ISEQN;
3737    |  // RA = src*8, RD = num_const*8, JMP with RD = target
3738    |.if DUALNUM
3739    |  lwzux CARG1, RA, BASE
3740    |    addi PC, PC, 4
3741    |   lwz CARG2, 4(RA)
3742    |  lwzux CARG3, RD, KBASE
3743    |  checknum cr0, CARG1
3744    |    lwz SAVE0, -4(PC)
3745    |  checknum cr1, CARG3
3746    |    decode_RD4 SAVE0, SAVE0
3747    |   lwz CARG4, 4(RD)
3748    |    addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3749    if (vk) {
3750      |->BC_ISEQN_Z:
3751    } else {
3752      |->BC_ISNEN_Z:
3753    }
3754    |  bne cr0, >7
3755    |  bne cr1, >8
3756    |   cmpw CARG2, CARG4
3757    |4:
3758    |.else
3759    if (vk) {
3760      |->BC_ISEQN_Z:  // Dummy label.
3761    } else {
3762      |->BC_ISNEN_Z:  // Dummy label.
3763    }
3764    |  lwzx CARG1, BASE, RA
3765    |    addi PC, PC, 4
3766    |   lfdx f0, BASE, RA
3767    |    lwz SAVE0, -4(PC)
3768    |  lfdx f1, KBASE, RD
3769    |    decode_RD4 SAVE0, SAVE0
3770    |  checknum CARG1
3771    |    addis SAVE0, SAVE0, -(BCBIAS_J*4 >> 16)
3772    |  bge >3
3773    |  fcmpu cr0, f0, f1
3774    |.endif
3775    if (vk) {
3776      |  bne >1
3777      |  add PC, PC, SAVE0
3778      |1:
3779      |.if not FFI
3780      |3:
3781      |.endif
3782    } else {
3783      |  beq >2
3784      |1:
3785      |.if not FFI
3786      |3:
3787      |.endif
3788      |  add PC, PC, SAVE0
3789      |2:
3790    }
3791    |  ins_next
3792    |.if FFI
3793    |3:
3794    |  cmpwi CARG1, LJ_TCDATA
3795    |  beq ->vmeta_equal_cd
3796    |  b <1
3797    |.endif
3798    |.if DUALNUM
3799    |7:  // RA is not an integer.
3800    |  bge cr0, <3
3801    |  // RA is a number.
3802    |   .FPU lfd f0, 0(RA)
3803    |  blt cr1, >1
3804    |  // RA is a number, RD is an integer.
3805    |.if FPU
3806    |  tonum_i f1, CARG4
3807    |.else
3808    |  bl ->vm_sfi2d_2
3809    |.endif
3810    |  b >2
3811    |
3812    |8: // RA is an integer, RD is a number.
3813    |.if FPU
3814    |  tonum_i f0, CARG2
3815    |.else
3816    |  bl ->vm_sfi2d_1
3817    |.endif
3818    |1:
3819    |  .FPU lfd f1, 0(RD)
3820    |2:
3821    |.if FPU
3822    |  fcmpu cr0, f0, f1
3823    |.else
3824    |  blex __ledf2
3825    |  cmpwi CRET1, 0
3826    |.endif
3827    |  b <4
3828    |.endif
3829    break;
3830
3831  case BC_ISEQP: case BC_ISNEP:
3832    vk = op == BC_ISEQP;
3833    |  // RA = src*8, RD = primitive_type*8 (~), JMP with RD = target
3834    |  lwzx TMP0, BASE, RA
3835    |   srwi TMP1, RD, 3
3836    |    lwz TMP2, 0(PC)
3837    |   not TMP1, TMP1
3838    |    addi PC, PC, 4
3839    |.if FFI
3840    |  cmpwi TMP0, LJ_TCDATA
3841    |.endif
3842    |  sub TMP0, TMP0, TMP1
3843    |.if FFI
3844    |  beq ->vmeta_equal_cd
3845    |.endif
3846    |    decode_RD4 TMP2, TMP2
3847    |  .gpr64 extsw TMP0, TMP0
3848    |  addic TMP0, TMP0, -1
3849    |    addis TMP2, TMP2, -(BCBIAS_J*4 >> 16)
3850    |  subfe TMP1, TMP1, TMP1
3851    if (vk) {
3852      |  and TMP2, TMP2, TMP1
3853    } else {
3854      |  andc TMP2, TMP2, TMP1
3855    }
3856    |  add PC, PC, TMP2
3857    |  ins_next
3858    break;
3859
3860  /* -- Unary test and copy ops ------------------------------------------- */
3861
3862  case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
3863    |  // RA = dst*8 or unused, RD = src*8, JMP with RD = target
3864    |  lwzx TMP0, BASE, RD
3865    |   lwz INS, 0(PC)
3866    |   addi PC, PC, 4
3867    if (op == BC_IST || op == BC_ISF) {
3868      |  .gpr64 extsw TMP0, TMP0
3869      |  subfic TMP0, TMP0, LJ_TTRUE
3870      |   decode_RD4 TMP2, INS
3871      |  subfe TMP1, TMP1, TMP1
3872      |   addis TMP2, TMP2, -(BCBIAS_J*4 >> 16)
3873      if (op == BC_IST) {
3874	|  andc TMP2, TMP2, TMP1
3875      } else {
3876	|  and TMP2, TMP2, TMP1
3877      }
3878      |  add PC, PC, TMP2
3879    } else {
3880      |  li TMP1, LJ_TFALSE
3881      |.if FPU
3882      |   lfdx f0, BASE, RD
3883      |.else
3884      |   lwzux CARG1, RD, BASE
3885      |   lwz CARG2, 4(RD)
3886      |.endif
3887      |  cmplw TMP0, TMP1
3888      if (op == BC_ISTC) {
3889	|  bge >1
3890      } else {
3891	|  blt >1
3892      }
3893      |  addis PC, PC, -(BCBIAS_J*4 >> 16)
3894      |  decode_RD4 TMP2, INS
3895      |.if FPU
3896      |   stfdx f0, BASE, RA
3897      |.else
3898      |   stwux CARG1, RA, BASE
3899      |   stw CARG2, 4(RA)
3900      |.endif
3901      |  add PC, PC, TMP2
3902      |1:
3903    }
3904    |  ins_next
3905    break;
3906
3907  case BC_ISTYPE:
3908    |  // RA = src*8, RD = -type*8
3909    |  lwzx TMP0, BASE, RA
3910    |  srwi TMP1, RD, 3
3911    |  ins_next1
3912    |.if not PPE and not GPR64
3913    |  add. TMP0, TMP0, TMP1
3914    |.else
3915    |  neg TMP1, TMP1
3916    |  cmpw TMP0, TMP1
3917    |.endif
3918    |  bne ->vmeta_istype
3919    |  ins_next2
3920    break;
3921  case BC_ISNUM:
3922    |  // RA = src*8, RD = -(TISNUM-1)*8
3923    |  lwzx TMP0, BASE, RA
3924    |  ins_next1
3925    |  checknum TMP0
3926    |  bge ->vmeta_istype
3927    |  ins_next2
3928    break;
3929
3930  /* -- Unary ops --------------------------------------------------------- */
3931
3932  case BC_MOV:
3933    |  // RA = dst*8, RD = src*8
3934    |  ins_next1
3935    |.if FPU
3936    |  lfdx f0, BASE, RD
3937    |  stfdx f0, BASE, RA
3938    |.else
3939    |  lwzux TMP0, RD, BASE
3940    |  lwz TMP1, 4(RD)
3941    |  stwux TMP0, RA, BASE
3942    |  stw TMP1, 4(RA)
3943    |.endif
3944    |  ins_next2
3945    break;
3946  case BC_NOT:
3947    |  // RA = dst*8, RD = src*8
3948    |  ins_next1
3949    |  lwzx TMP0, BASE, RD
3950    |  .gpr64 extsw TMP0, TMP0
3951    |  subfic TMP1, TMP0, LJ_TTRUE
3952    |  adde TMP0, TMP0, TMP1
3953    |  stwx TMP0, BASE, RA
3954    |  ins_next2
3955    break;
3956  case BC_UNM:
3957    |  // RA = dst*8, RD = src*8
3958    |  lwzux TMP1, RD, BASE
3959    |   lwz TMP0, 4(RD)
3960    |  checknum TMP1
3961    |.if DUALNUM
3962    |  bne >5
3963    |.if GPR64
3964    |  lus TMP2, 0x8000
3965    |  neg. TMP0, TMP0
3966    |  beq >8
3967    |  cmplw TMP0, TMP2
3968    |  beq >4
3969    |.else
3970    |  nego. TMP0, TMP0
3971    |  beq >8
3972    |  bso >4
3973    |1:
3974    |.endif
3975    |  ins_next1
3976    |  stwux TISNUM, RA, BASE
3977    |   stw TMP0, 4(RA)
3978    |3:
3979    |  ins_next2
3980    |4:
3981    |.if not GPR64
3982    |  // Potential overflow.
3983    |  checkov TMP1, <1			// Ignore unrelated overflow.
3984    |.endif
3985    |  lus TMP1, 0x41e0			// 2^31.
3986    |  li TMP0, 0
3987    |  b >7
3988    |.endif
3989    |5:
3990    |  bge ->vmeta_unm
3991    |  xoris TMP1, TMP1, 0x8000
3992    |7:
3993    |  ins_next1
3994    |  stwux TMP1, RA, BASE
3995    |   stw TMP0, 4(RA)
3996    |.if DUALNUM
3997    |  b <3
3998    |8:
3999    |  lus TMP1, 0x8000			// -0.
4000    |  b <7
4001    |.else
4002    |  ins_next2
4003    |.endif
4004    break;
4005  case BC_LEN:
4006    |  // RA = dst*8, RD = src*8
4007    |  lwzux TMP0, RD, BASE
4008    |   lwz CARG1, 4(RD)
4009    |  checkstr TMP0; bne >2
4010    |  lwz CRET1, STR:CARG1->len
4011    |1:
4012    |.if DUALNUM
4013    |  ins_next1
4014    |  stwux TISNUM, RA, BASE
4015    |   stw CRET1, 4(RA)
4016    |.else
4017    |  tonum_u f0, CRET1		// Result is a non-negative integer.
4018    |  ins_next1
4019    |  stfdx f0, BASE, RA
4020    |.endif
4021    |  ins_next2
4022    |2:
4023    |  checktab TMP0; bne ->vmeta_len
4024#if LJ_52
4025    |  lwz TAB:TMP2, TAB:CARG1->metatable
4026    |  cmplwi TAB:TMP2, 0
4027    |  bne >9
4028    |3:
4029#endif
4030    |->BC_LEN_Z:
4031    |  bl extern lj_tab_len		// (GCtab *t)
4032    |  // Returns uint32_t (but less than 2^31).
4033    |  b <1
4034#if LJ_52
4035    |9:
4036    |  lbz TMP0, TAB:TMP2->nomm
4037    |  andix. TMP0, TMP0, 1<<MM_len
4038    |  bne <3				// 'no __len' flag set: done.
4039    |  b ->vmeta_len
4040#endif
4041    break;
4042
4043  /* -- Binary ops -------------------------------------------------------- */
4044
4045    |.macro ins_arithpre
4046    |  // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
4047    ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
4048    ||switch (vk) {
4049    ||case 0:
4050    |   lwzx CARG1, BASE, RB
4051    |   .if DUALNUM
4052    |     lwzx CARG3, KBASE, RC
4053    |   .endif
4054    |   .if FPU
4055    |    lfdx f14, BASE, RB
4056    |    lfdx f15, KBASE, RC
4057    |   .else
4058    |    add TMP1, BASE, RB
4059    |    add TMP2, KBASE, RC
4060    |    lwz CARG2, 4(TMP1)
4061    |    lwz CARG4, 4(TMP2)
4062    |   .endif
4063    |   .if DUALNUM
4064    |     checknum cr0, CARG1
4065    |     checknum cr1, CARG3
4066    |     crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
4067    |     bge ->vmeta_arith_vn
4068    |   .else
4069    |     checknum CARG1; bge ->vmeta_arith_vn
4070    |   .endif
4071    ||  break;
4072    ||case 1:
4073    |   lwzx CARG1, BASE, RB
4074    |   .if DUALNUM
4075    |     lwzx CARG3, KBASE, RC
4076    |   .endif
4077    |   .if FPU
4078    |    lfdx f15, BASE, RB
4079    |    lfdx f14, KBASE, RC
4080    |   .else
4081    |    add TMP1, BASE, RB
4082    |    add TMP2, KBASE, RC
4083    |    lwz CARG2, 4(TMP1)
4084    |    lwz CARG4, 4(TMP2)
4085    |   .endif
4086    |   .if DUALNUM
4087    |     checknum cr0, CARG1
4088    |     checknum cr1, CARG3
4089    |     crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
4090    |     bge ->vmeta_arith_nv
4091    |   .else
4092    |     checknum CARG1; bge ->vmeta_arith_nv
4093    |   .endif
4094    ||  break;
4095    ||default:
4096    |   lwzx CARG1, BASE, RB
4097    |   lwzx CARG3, BASE, RC
4098    |   .if FPU
4099    |    lfdx f14, BASE, RB
4100    |    lfdx f15, BASE, RC
4101    |   .else
4102    |    add TMP1, BASE, RB
4103    |    add TMP2, BASE, RC
4104    |    lwz CARG2, 4(TMP1)
4105    |    lwz CARG4, 4(TMP2)
4106    |   .endif
4107    |   checknum cr0, CARG1
4108    |   checknum cr1, CARG3
4109    |   crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
4110    |   bge ->vmeta_arith_vv
4111    ||  break;
4112    ||}
4113    |.endmacro
4114    |
4115    |.macro ins_arithfallback, ins
4116    ||switch (vk) {
4117    ||case 0:
4118    |   ins ->vmeta_arith_vn2
4119    ||  break;
4120    ||case 1:
4121    |   ins ->vmeta_arith_nv2
4122    ||  break;
4123    ||default:
4124    |   ins ->vmeta_arith_vv2
4125    ||  break;
4126    ||}
4127    |.endmacro
4128    |
4129    |.macro intmod, a, b, c
4130    |  bl ->vm_modi
4131    |.endmacro
4132    |
4133    |.macro fpmod, a, b, c
4134    |->BC_MODVN_Z:
4135    |  fdiv FARG1, b, c
4136    |  // NYI: Use internal implementation of floor.
4137    |  blex floor			// floor(b/c)
4138    |  fmul a, FARG1, c
4139    |  fsub a, b, a			// b - floor(b/c)*c
4140    |.endmacro
4141    |
4142    |.macro sfpmod
4143    |->BC_MODVN_Z:
4144    |  stw CARG1, SFSAVE_1
4145    |  stw CARG2, SFSAVE_2
4146    |  mr SAVE0, CARG3
4147    |  mr SAVE1, CARG4
4148    |  blex __divdf3
4149    |  blex floor
4150    |  mr CARG3, SAVE0
4151    |  mr CARG4, SAVE1
4152    |  blex __muldf3
4153    |  mr CARG3, CRET1
4154    |  mr CARG4, CRET2
4155    |  lwz CARG1, SFSAVE_1
4156    |  lwz CARG2, SFSAVE_2
4157    |  blex __subdf3
4158    |.endmacro
4159    |
4160    |.macro ins_arithfp, fpins
4161    |  ins_arithpre
4162    |.if "fpins" == "fpmod_"
4163    |  b ->BC_MODVN_Z			// Avoid 3 copies. It's slow anyway.
4164    |.elif FPU
4165    |  fpins f0, f14, f15
4166    |  ins_next1
4167    |  stfdx f0, BASE, RA
4168    |  ins_next2
4169    |.else
4170    |  blex __divdf3			// Only soft-float div uses this macro.
4171    |  ins_next1
4172    |  stwux CRET1, RA, BASE
4173    |  stw CRET2, 4(RA)
4174    |  ins_next2
4175    |.endif
4176    |.endmacro
4177    |
4178    |.macro ins_arithdn, intins, fpins, fpcall
4179    |  // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
4180    ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
4181    ||switch (vk) {
4182    ||case 0:
4183    |   lwzux CARG1, RB, BASE
4184    |   lwzux CARG3, RC, KBASE
4185    |    lwz CARG2, 4(RB)
4186    |   checknum cr0, CARG1
4187    |    lwz CARG4, 4(RC)
4188    |   checknum cr1, CARG3
4189    ||  break;
4190    ||case 1:
4191    |   lwzux CARG3, RB, BASE
4192    |   lwzux CARG1, RC, KBASE
4193    |    lwz CARG4, 4(RB)
4194    |   checknum cr0, CARG3
4195    |    lwz CARG2, 4(RC)
4196    |   checknum cr1, CARG1
4197    ||  break;
4198    ||default:
4199    |   lwzux CARG1, RB, BASE
4200    |   lwzux CARG3, RC, BASE
4201    |    lwz CARG2, 4(RB)
4202    |   checknum cr0, CARG1
4203    |    lwz CARG4, 4(RC)
4204    |   checknum cr1, CARG3
4205    ||  break;
4206    ||}
4207    |  bne >5
4208    |  bne cr1, >5
4209    |.if "intins" == "intmod"
4210    |  mr CARG1, CARG2
4211    |  mr CARG2, CARG4
4212    |.endif
4213    |  intins CARG1, CARG2, CARG4
4214    |  bso >4
4215    |1:
4216    |  ins_next1
4217    |  stwux TISNUM, RA, BASE
4218    |  stw CARG1, 4(RA)
4219    |2:
4220    |  ins_next2
4221    |4:  // Overflow.
4222    |  checkov TMP0, <1			// Ignore unrelated overflow.
4223    |  ins_arithfallback b
4224    |5:  // FP variant.
4225    |.if FPU
4226    ||if (vk == 1) {
4227    |  lfd f15, 0(RB)
4228    |  lfd f14, 0(RC)
4229    ||} else {
4230    |  lfd f14, 0(RB)
4231    |  lfd f15, 0(RC)
4232    ||}
4233    |.endif
4234    |  crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
4235    |   ins_arithfallback bge
4236    |.if "fpins" == "fpmod_"
4237    |  b ->BC_MODVN_Z			// Avoid 3 copies. It's slow anyway.
4238    |.else
4239    |.if FPU
4240    |  fpins f0, f14, f15
4241    |  stfdx f0, BASE, RA
4242    |.else
4243    |.if "fpcall" == "sfpmod"
4244    |  sfpmod
4245    |.else
4246    |  blex fpcall
4247    |.endif
4248    |  stwux CRET1, RA, BASE
4249    |  stw CRET2, 4(RA)
4250    |.endif
4251    |  ins_next1
4252    |  b <2
4253    |.endif
4254    |.endmacro
4255    |
4256    |.macro ins_arith, intins, fpins, fpcall
4257    |.if DUALNUM
4258    |  ins_arithdn intins, fpins, fpcall
4259    |.else
4260    |  ins_arithfp fpins
4261    |.endif
4262    |.endmacro
4263
4264  case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
4265    |.if GPR64
4266    |.macro addo32., y, a, b
4267    |  // Need to check overflow for (a<<32) + (b<<32).
4268    |  rldicr TMP0, a, 32, 31
4269    |  rldicr TMP1, b, 32, 31
4270    |  addo. TMP0, TMP0, TMP1
4271    |  add y, a, b
4272    |.endmacro
4273    |  ins_arith addo32., fadd, __adddf3
4274    |.else
4275    |  ins_arith addo., fadd, __adddf3
4276    |.endif
4277    break;
4278  case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
4279    |.if GPR64
4280    |.macro subo32., y, a, b
4281    |  // Need to check overflow for (a<<32) - (b<<32).
4282    |  rldicr TMP0, a, 32, 31
4283    |  rldicr TMP1, b, 32, 31
4284    |  subo. TMP0, TMP0, TMP1
4285    |  sub y, a, b
4286    |.endmacro
4287    |  ins_arith subo32., fsub, __subdf3
4288    |.else
4289    |  ins_arith subo., fsub, __subdf3
4290    |.endif
4291    break;
4292  case BC_MULVN: case BC_MULNV: case BC_MULVV:
4293    |  ins_arith mullwo., fmul, __muldf3
4294    break;
4295  case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
4296    |  ins_arithfp fdiv
4297    break;
4298  case BC_MODVN:
4299    |  ins_arith intmod, fpmod, sfpmod
4300    break;
4301  case BC_MODNV: case BC_MODVV:
4302    |  ins_arith intmod, fpmod_, sfpmod
4303    break;
4304  case BC_POW:
4305    |  // NYI: (partial) integer arithmetic.
4306    |  lwzx CARG1, BASE, RB
4307    |  lwzx CARG3, BASE, RC
4308    |.if FPU
4309    |   lfdx FARG1, BASE, RB
4310    |   lfdx FARG2, BASE, RC
4311    |.else
4312    |   add TMP1, BASE, RB
4313    |   add TMP2, BASE, RC
4314    |   lwz CARG2, 4(TMP1)
4315    |   lwz CARG4, 4(TMP2)
4316    |.endif
4317    |  checknum cr0, CARG1
4318    |  checknum cr1, CARG3
4319    |  crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
4320    |  bge ->vmeta_arith_vv
4321    |  blex pow
4322    |  ins_next1
4323    |.if FPU
4324    |  stfdx FARG1, BASE, RA
4325    |.else
4326    |  stwux CARG1, RA, BASE
4327    |  stw CARG2, 4(RA)
4328    |.endif
4329    |  ins_next2
4330    break;
4331
4332  case BC_CAT:
4333    |  // RA = dst*8, RB = src_start*8, RC = src_end*8
4334    |  sub CARG3, RC, RB
4335    |   stp BASE, L->base
4336    |  add CARG2, BASE, RC
4337    |  mr SAVE0, RB
4338    |->BC_CAT_Z:
4339    |   stw PC, SAVE_PC
4340    |  mr CARG1, L
4341    |  srwi CARG3, CARG3, 3
4342    |  bl extern lj_meta_cat		// (lua_State *L, TValue *top, int left)
4343    |  // Returns NULL (finished) or TValue * (metamethod).
4344    |  cmplwi CRET1, 0
4345    |   lp BASE, L->base
4346    |  bne ->vmeta_binop
4347    |  ins_next1
4348    |.if FPU
4349    |  lfdx f0, BASE, SAVE0		// Copy result from RB to RA.
4350    |  stfdx f0, BASE, RA
4351    |.else
4352    |  lwzux TMP0, SAVE0, BASE
4353    |  lwz TMP1, 4(SAVE0)
4354    |  stwux TMP0, RA, BASE
4355    |  stw TMP1, 4(RA)
4356    |.endif
4357    |  ins_next2
4358    break;
4359
4360  /* -- Constant ops ------------------------------------------------------ */
4361
4362  case BC_KSTR:
4363    |  // RA = dst*8, RD = str_const*8 (~)
4364    |  srwi TMP1, RD, 1
4365    |  subfic TMP1, TMP1, -4
4366    |  ins_next1
4367    |  lwzx TMP0, KBASE, TMP1		// KBASE-4-str_const*4
4368    |  li TMP2, LJ_TSTR
4369    |  stwux TMP2, RA, BASE
4370    |  stw TMP0, 4(RA)
4371    |  ins_next2
4372    break;
4373  case BC_KCDATA:
4374    |.if FFI
4375    |  // RA = dst*8, RD = cdata_const*8 (~)
4376    |  srwi TMP1, RD, 1
4377    |  subfic TMP1, TMP1, -4
4378    |  ins_next1
4379    |  lwzx TMP0, KBASE, TMP1		// KBASE-4-cdata_const*4
4380    |  li TMP2, LJ_TCDATA
4381    |  stwux TMP2, RA, BASE
4382    |  stw TMP0, 4(RA)
4383    |  ins_next2
4384    |.endif
4385    break;
4386  case BC_KSHORT:
4387    |  // RA = dst*8, RD = int16_literal*8
4388    |.if DUALNUM
4389    |  slwi RD, RD, 13
4390    |  srawi RD, RD, 16
4391    |  ins_next1
4392    |   stwux TISNUM, RA, BASE
4393    |   stw RD, 4(RA)
4394    |  ins_next2
4395    |.else
4396    |  // The soft-float approach is faster.
4397    |  slwi RD, RD, 13
4398    |  srawi TMP1, RD, 31
4399    |  xor TMP2, TMP1, RD
4400    |  sub TMP2, TMP2, TMP1		// TMP2 = abs(x)
4401    |  cntlzw TMP3, TMP2
4402    |  subfic TMP1, TMP3, 0x40d		// TMP1 = exponent-1
4403    |   slw TMP2, TMP2, TMP3		// TMP2 = left aligned mantissa
4404    |    subfic TMP3, RD, 0
4405    |  slwi TMP1, TMP1, 20
4406    |   rlwimi RD, TMP2, 21, 1, 31	// hi = sign(x) | (mantissa>>11)
4407    |    subfe TMP0, TMP0, TMP0
4408    |   add RD, RD, TMP1		// hi = hi + exponent-1
4409    |    and RD, RD, TMP0		// hi = x == 0 ? 0 : hi
4410    |  ins_next1
4411    |    stwux RD, RA, BASE
4412    |    stw ZERO, 4(RA)
4413    |  ins_next2
4414    |.endif
4415    break;
4416  case BC_KNUM:
4417    |  // RA = dst*8, RD = num_const*8
4418    |  ins_next1
4419    |.if FPU
4420    |  lfdx f0, KBASE, RD
4421    |  stfdx f0, BASE, RA
4422    |.else
4423    |  lwzux TMP0, RD, KBASE
4424    |  lwz TMP1, 4(RD)
4425    |  stwux TMP0, RA, BASE
4426    |  stw TMP1, 4(RA)
4427    |.endif
4428    |  ins_next2
4429    break;
4430  case BC_KPRI:
4431    |  // RA = dst*8, RD = primitive_type*8 (~)
4432    |  srwi TMP1, RD, 3
4433    |  not TMP0, TMP1
4434    |  ins_next1
4435    |  stwx TMP0, BASE, RA
4436    |  ins_next2
4437    break;
4438  case BC_KNIL:
4439    |  // RA = base*8, RD = end*8
4440    |  stwx TISNIL, BASE, RA
4441    |   addi RA, RA, 8
4442    |1:
4443    |  stwx TISNIL, BASE, RA
4444    |  cmpw RA, RD
4445    |   addi RA, RA, 8
4446    |  blt <1
4447    |  ins_next_
4448    break;
4449
4450  /* -- Upvalue and function ops ------------------------------------------ */
4451
4452  case BC_UGET:
4453    |  // RA = dst*8, RD = uvnum*8
4454    |  lwz LFUNC:RB, FRAME_FUNC(BASE)
4455    |   srwi RD, RD, 1
4456    |   addi RD, RD, offsetof(GCfuncL, uvptr)
4457    |  lwzx UPVAL:RB, LFUNC:RB, RD
4458    |  ins_next1
4459    |  lwz TMP1, UPVAL:RB->v
4460    |.if FPU
4461    |  lfd f0, 0(TMP1)
4462    |  stfdx f0, BASE, RA
4463    |.else
4464    |  lwz TMP2, 0(TMP1)
4465    |  lwz TMP3, 4(TMP1)
4466    |  stwux TMP2, RA, BASE
4467    |  stw TMP3, 4(RA)
4468    |.endif
4469    |  ins_next2
4470    break;
4471  case BC_USETV:
4472    |  // RA = uvnum*8, RD = src*8
4473    |  lwz LFUNC:RB, FRAME_FUNC(BASE)
4474    |    srwi RA, RA, 1
4475    |    addi RA, RA, offsetof(GCfuncL, uvptr)
4476    |.if FPU
4477    |   lfdux f0, RD, BASE
4478    |.else
4479    |   lwzux CARG1, RD, BASE
4480    |   lwz CARG3, 4(RD)
4481    |.endif
4482    |  lwzx UPVAL:RB, LFUNC:RB, RA
4483    |  lbz TMP3, UPVAL:RB->marked
4484    |   lwz CARG2, UPVAL:RB->v
4485    |  andix. TMP3, TMP3, LJ_GC_BLACK	// isblack(uv)
4486    |    lbz TMP0, UPVAL:RB->closed
4487    |   lwz TMP2, 0(RD)
4488    |.if FPU
4489    |   stfd f0, 0(CARG2)
4490    |.else
4491    |   stw CARG1, 0(CARG2)
4492    |   stw CARG3, 4(CARG2)
4493    |.endif
4494    |    cmplwi cr1, TMP0, 0
4495    |   lwz TMP1, 4(RD)
4496    |  cror 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
4497    |   subi TMP2, TMP2, (LJ_TNUMX+1)
4498    |  bne >2				// Upvalue is closed and black?
4499    |1:
4500    |  ins_next
4501    |
4502    |2:  // Check if new value is collectable.
4503    |  cmplwi TMP2, LJ_TISGCV - (LJ_TNUMX+1)
4504    |  bge <1				// tvisgcv(v)
4505    |  lbz TMP3, GCOBJ:TMP1->gch.marked
4506    |  andix. TMP3, TMP3, LJ_GC_WHITES	// iswhite(v)
4507    |   la CARG1, GG_DISP2G(DISPATCH)
4508    |  // Crossed a write barrier. Move the barrier forward.
4509    |  beq <1
4510    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
4511    |  b <1
4512    break;
4513  case BC_USETS:
4514    |  // RA = uvnum*8, RD = str_const*8 (~)
4515    |  lwz LFUNC:RB, FRAME_FUNC(BASE)
4516    |   srwi TMP1, RD, 1
4517    |    srwi RA, RA, 1
4518    |   subfic TMP1, TMP1, -4
4519    |    addi RA, RA, offsetof(GCfuncL, uvptr)
4520    |   lwzx STR:TMP1, KBASE, TMP1	// KBASE-4-str_const*4
4521    |  lwzx UPVAL:RB, LFUNC:RB, RA
4522    |  lbz TMP3, UPVAL:RB->marked
4523    |   lwz CARG2, UPVAL:RB->v
4524    |  andix. TMP3, TMP3, LJ_GC_BLACK	// isblack(uv)
4525    |   lbz TMP3, STR:TMP1->marked
4526    |   lbz TMP2, UPVAL:RB->closed
4527    |   li TMP0, LJ_TSTR
4528    |   stw STR:TMP1, 4(CARG2)
4529    |   stw TMP0, 0(CARG2)
4530    |  bne >2
4531    |1:
4532    |  ins_next
4533    |
4534    |2:  // Check if string is white and ensure upvalue is closed.
4535    |  andix. TMP3, TMP3, LJ_GC_WHITES	// iswhite(str)
4536    |   cmplwi cr1, TMP2, 0
4537    |  cror 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
4538    |   la CARG1, GG_DISP2G(DISPATCH)
4539    |  // Crossed a write barrier. Move the barrier forward.
4540    |  beq <1
4541    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
4542    |  b <1
4543    break;
4544  case BC_USETN:
4545    |  // RA = uvnum*8, RD = num_const*8
4546    |  lwz LFUNC:RB, FRAME_FUNC(BASE)
4547    |   srwi RA, RA, 1
4548    |   addi RA, RA, offsetof(GCfuncL, uvptr)
4549    |.if FPU
4550    |    lfdx f0, KBASE, RD
4551    |.else
4552    |    lwzux TMP2, RD, KBASE
4553    |    lwz TMP3, 4(RD)
4554    |.endif
4555    |  lwzx UPVAL:RB, LFUNC:RB, RA
4556    |  ins_next1
4557    |  lwz TMP1, UPVAL:RB->v
4558    |.if FPU
4559    |  stfd f0, 0(TMP1)
4560    |.else
4561    |  stw TMP2, 0(TMP1)
4562    |  stw TMP3, 4(TMP1)
4563    |.endif
4564    |  ins_next2
4565    break;
4566  case BC_USETP:
4567    |  // RA = uvnum*8, RD = primitive_type*8 (~)
4568    |  lwz LFUNC:RB, FRAME_FUNC(BASE)
4569    |   srwi RA, RA, 1
4570    |    srwi TMP0, RD, 3
4571    |   addi RA, RA, offsetof(GCfuncL, uvptr)
4572    |    not TMP0, TMP0
4573    |  lwzx UPVAL:RB, LFUNC:RB, RA
4574    |  ins_next1
4575    |  lwz TMP1, UPVAL:RB->v
4576    |  stw TMP0, 0(TMP1)
4577    |  ins_next2
4578    break;
4579
4580  case BC_UCLO:
4581    |  // RA = level*8, RD = target
4582    |  lwz TMP1, L->openupval
4583    |  branch_RD			// Do this first since RD is not saved.
4584    |   stp BASE, L->base
4585    |  cmplwi TMP1, 0
4586    |   mr CARG1, L
4587    |  beq >1
4588    |   add CARG2, BASE, RA
4589    |  bl extern lj_func_closeuv	// (lua_State *L, TValue *level)
4590    |  lp BASE, L->base
4591    |1:
4592    |  ins_next
4593    break;
4594
4595  case BC_FNEW:
4596    |  // RA = dst*8, RD = proto_const*8 (~) (holding function prototype)
4597    |  srwi TMP1, RD, 1
4598    |   stp BASE, L->base
4599    |  subfic TMP1, TMP1, -4
4600    |   stw PC, SAVE_PC
4601    |  lwzx CARG2, KBASE, TMP1		// KBASE-4-tab_const*4
4602    |   mr CARG1, L
4603    |  lwz CARG3, FRAME_FUNC(BASE)
4604    |  // (lua_State *L, GCproto *pt, GCfuncL *parent)
4605    |  bl extern lj_func_newL_gc
4606    |  // Returns GCfuncL *.
4607    |  lp BASE, L->base
4608    |   li TMP0, LJ_TFUNC
4609    |  stwux TMP0, RA, BASE
4610    |  stw LFUNC:CRET1, 4(RA)
4611    |  ins_next
4612    break;
4613
4614  /* -- Table ops --------------------------------------------------------- */
4615
4616  case BC_TNEW:
4617  case BC_TDUP:
4618    |  // RA = dst*8, RD = (hbits|asize)*8 | tab_const*8 (~)
4619    |  lwz TMP0, DISPATCH_GL(gc.total)(DISPATCH)
4620    |   mr CARG1, L
4621    |  lwz TMP1, DISPATCH_GL(gc.threshold)(DISPATCH)
4622    |   stp BASE, L->base
4623    |  cmplw TMP0, TMP1
4624    |   stw PC, SAVE_PC
4625    |  bge >5
4626    |1:
4627    if (op == BC_TNEW) {
4628      |  rlwinm CARG2, RD, 29, 21, 31
4629      |  rlwinm CARG3, RD, 18, 27, 31
4630      |  cmpwi CARG2, 0x7ff; beq >3
4631      |2:
4632      |  bl extern lj_tab_new  // (lua_State *L, int32_t asize, uint32_t hbits)
4633      |  // Returns Table *.
4634    } else {
4635      |  srwi TMP1, RD, 1
4636      |  subfic TMP1, TMP1, -4
4637      |  lwzx CARG2, KBASE, TMP1		// KBASE-4-tab_const*4
4638      |  bl extern lj_tab_dup  // (lua_State *L, Table *kt)
4639      |  // Returns Table *.
4640    }
4641    |  lp BASE, L->base
4642    |   li TMP0, LJ_TTAB
4643    |  stwux TMP0, RA, BASE
4644    |  stw TAB:CRET1, 4(RA)
4645    |  ins_next
4646    if (op == BC_TNEW) {
4647      |3:
4648      |  li CARG2, 0x801
4649      |  b <2
4650    }
4651    |5:
4652    |  mr SAVE0, RD
4653    |  bl extern lj_gc_step_fixtop  // (lua_State *L)
4654    |  mr RD, SAVE0
4655    |  mr CARG1, L
4656    |  b <1
4657    break;
4658
4659  case BC_GGET:
4660    |  // RA = dst*8, RD = str_const*8 (~)
4661  case BC_GSET:
4662    |  // RA = src*8, RD = str_const*8 (~)
4663    |  lwz LFUNC:TMP2, FRAME_FUNC(BASE)
4664    |   srwi TMP1, RD, 1
4665    |  lwz TAB:RB, LFUNC:TMP2->env
4666    |   subfic TMP1, TMP1, -4
4667    |   lwzx STR:RC, KBASE, TMP1	// KBASE-4-str_const*4
4668    if (op == BC_GGET) {
4669      |  b ->BC_TGETS_Z
4670    } else {
4671      |  b ->BC_TSETS_Z
4672    }
4673    break;
4674
4675  case BC_TGETV:
4676    |  // RA = dst*8, RB = table*8, RC = key*8
4677    |  lwzux CARG1, RB, BASE
4678    |  lwzux CARG2, RC, BASE
4679    |   lwz TAB:RB, 4(RB)
4680    |.if DUALNUM
4681    |   lwz RC, 4(RC)
4682    |.else
4683    |   lfd f0, 0(RC)
4684    |.endif
4685    |  checktab CARG1
4686    |   checknum cr1, CARG2
4687    |  bne ->vmeta_tgetv
4688    |.if DUALNUM
4689    |  lwz TMP0, TAB:RB->asize
4690    |   bne cr1, >5
4691    |   lwz TMP1, TAB:RB->array
4692    |  cmplw TMP0, RC
4693    |   slwi TMP2, RC, 3
4694    |.else
4695    |   bge cr1, >5
4696    |  // Convert number key to integer, check for integerness and range.
4697    |  fctiwz f1, f0
4698    |    fadd f2, f0, TOBIT
4699    |  stfd f1, TMPD
4700    |   lwz TMP0, TAB:RB->asize
4701    |    fsub f2, f2, TOBIT
4702    |  lwz TMP2, TMPD_LO
4703    |   lwz TMP1, TAB:RB->array
4704    |    fcmpu cr1, f0, f2
4705    |  cmplw cr0, TMP0, TMP2
4706    |  crand 4*cr0+gt, 4*cr0+gt, 4*cr1+eq
4707    |   slwi TMP2, TMP2, 3
4708    |.endif
4709    |  ble ->vmeta_tgetv		// Integer key and in array part?
4710    |  lwzx TMP0, TMP1, TMP2
4711    |.if FPU
4712    |   lfdx f14, TMP1, TMP2
4713    |.else
4714    |   lwzux SAVE0, TMP1, TMP2
4715    |   lwz SAVE1, 4(TMP1)
4716    |.endif
4717    |  checknil TMP0; beq >2
4718    |1:
4719    |  ins_next1
4720    |.if FPU
4721    |   stfdx f14, BASE, RA
4722    |.else
4723    |   stwux SAVE0, RA, BASE
4724    |   stw SAVE1, 4(RA)
4725    |.endif
4726    |  ins_next2
4727    |
4728    |2:  // Check for __index if table value is nil.
4729    |  lwz TAB:TMP2, TAB:RB->metatable
4730    |  cmplwi TAB:TMP2, 0
4731    |  beq <1				// No metatable: done.
4732    |  lbz TMP0, TAB:TMP2->nomm
4733    |  andix. TMP0, TMP0, 1<<MM_index
4734    |  bne <1				// 'no __index' flag set: done.
4735    |  b ->vmeta_tgetv
4736    |
4737    |5:
4738    |  checkstr CARG2; bne ->vmeta_tgetv
4739    |.if not DUALNUM
4740    |  lwz STR:RC, 4(RC)
4741    |.endif
4742    |  b ->BC_TGETS_Z			// String key?
4743    break;
4744  case BC_TGETS:
4745    |  // RA = dst*8, RB = table*8, RC = str_const*8 (~)
4746    |  lwzux CARG1, RB, BASE
4747    |   srwi TMP1, RC, 1
4748    |    lwz TAB:RB, 4(RB)
4749    |   subfic TMP1, TMP1, -4
4750    |  checktab CARG1
4751    |   lwzx STR:RC, KBASE, TMP1	// KBASE-4-str_const*4
4752    |  bne ->vmeta_tgets1
4753    |->BC_TGETS_Z:
4754    |  // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8
4755    |  lwz TMP0, TAB:RB->hmask
4756    |  lwz TMP1, STR:RC->sid
4757    |  lwz NODE:TMP2, TAB:RB->node
4758    |  and TMP1, TMP1, TMP0		// idx = str->sid & tab->hmask
4759    |  slwi TMP0, TMP1, 5
4760    |  slwi TMP1, TMP1, 3
4761    |  sub TMP1, TMP0, TMP1
4762    |  add NODE:TMP2, NODE:TMP2, TMP1	// node = tab->node + (idx*32-idx*8)
4763    |1:
4764    |  lwz CARG1, NODE:TMP2->key
4765    |   lwz TMP0, 4+offsetof(Node, key)(NODE:TMP2)
4766    |    lwz CARG2, NODE:TMP2->val
4767    |     lwz TMP1, 4+offsetof(Node, val)(NODE:TMP2)
4768    |  checkstr CARG1; bne >4
4769    |   cmpw TMP0, STR:RC; bne >4
4770    |    checknil CARG2; beq >5		// Key found, but nil value?
4771    |3:
4772    |    stwux CARG2, RA, BASE
4773    |     stw TMP1, 4(RA)
4774    |  ins_next
4775    |
4776    |4:  // Follow hash chain.
4777    |  lwz NODE:TMP2, NODE:TMP2->next
4778    |  cmplwi NODE:TMP2, 0
4779    |  bne <1
4780    |  // End of hash chain: key not found, nil result.
4781    |   li CARG2, LJ_TNIL
4782    |
4783    |5:  // Check for __index if table value is nil.
4784    |  lwz TAB:TMP2, TAB:RB->metatable
4785    |  cmplwi TAB:TMP2, 0
4786    |  beq <3				// No metatable: done.
4787    |  lbz TMP0, TAB:TMP2->nomm
4788    |  andix. TMP0, TMP0, 1<<MM_index
4789    |  bne <3				// 'no __index' flag set: done.
4790    |  b ->vmeta_tgets
4791    break;
4792  case BC_TGETB:
4793    |  // RA = dst*8, RB = table*8, RC = index*8
4794    |  lwzux CARG1, RB, BASE
4795    |   srwi TMP0, RC, 3
4796    |   lwz TAB:RB, 4(RB)
4797    |  checktab CARG1; bne ->vmeta_tgetb
4798    |  lwz TMP1, TAB:RB->asize
4799    |   lwz TMP2, TAB:RB->array
4800    |  cmplw TMP0, TMP1; bge ->vmeta_tgetb
4801    |.if FPU
4802    |  lwzx TMP1, TMP2, RC
4803    |   lfdx f0, TMP2, RC
4804    |.else
4805    |  lwzux TMP1, TMP2, RC
4806    |   lwz TMP3, 4(TMP2)
4807    |.endif
4808    |  checknil TMP1; beq >5
4809    |1:
4810    |  ins_next1
4811    |.if FPU
4812    |   stfdx f0, BASE, RA
4813    |.else
4814    |   stwux TMP1, RA, BASE
4815    |   stw TMP3, 4(RA)
4816    |.endif
4817    |  ins_next2
4818    |
4819    |5:  // Check for __index if table value is nil.
4820    |  lwz TAB:TMP2, TAB:RB->metatable
4821    |  cmplwi TAB:TMP2, 0
4822    |  beq <1				// No metatable: done.
4823    |  lbz TMP2, TAB:TMP2->nomm
4824    |  andix. TMP2, TMP2, 1<<MM_index
4825    |  bne <1				// 'no __index' flag set: done.
4826    |  b ->vmeta_tgetb			// Caveat: preserve TMP0!
4827    break;
4828  case BC_TGETR:
4829    |  // RA = dst*8, RB = table*8, RC = key*8
4830    |  add RB, BASE, RB
4831    |  lwz TAB:CARG1, 4(RB)
4832    |.if DUALNUM
4833    |  add RC, BASE, RC
4834    |  lwz TMP0, TAB:CARG1->asize
4835    |  lwz CARG2, 4(RC)
4836    |   lwz TMP1, TAB:CARG1->array
4837    |.else
4838    |  lfdx f0, BASE, RC
4839    |  lwz TMP0, TAB:CARG1->asize
4840    |  toint CARG2, f0
4841    |   lwz TMP1, TAB:CARG1->array
4842    |.endif
4843    |  cmplw TMP0, CARG2
4844    |   slwi TMP2, CARG2, 3
4845    |  ble ->vmeta_tgetr		// In array part?
4846    |.if FPU
4847    |   lfdx f14, TMP1, TMP2
4848    |.else
4849    |   lwzux SAVE0, TMP2, TMP1
4850    |   lwz SAVE1, 4(TMP2)
4851    |.endif
4852    |->BC_TGETR_Z:
4853    |  ins_next1
4854    |.if FPU
4855    |   stfdx f14, BASE, RA
4856    |.else
4857    |   stwux SAVE0, RA, BASE
4858    |   stw SAVE1, 4(RA)
4859    |.endif
4860    |  ins_next2
4861    break;
4862
4863  case BC_TSETV:
4864    |  // RA = src*8, RB = table*8, RC = key*8
4865    |  lwzux CARG1, RB, BASE
4866    |  lwzux CARG2, RC, BASE
4867    |   lwz TAB:RB, 4(RB)
4868    |.if DUALNUM
4869    |   lwz RC, 4(RC)
4870    |.else
4871    |   lfd f0, 0(RC)
4872    |.endif
4873    |  checktab CARG1
4874    |   checknum cr1, CARG2
4875    |  bne ->vmeta_tsetv
4876    |.if DUALNUM
4877    |  lwz TMP0, TAB:RB->asize
4878    |   bne cr1, >5
4879    |   lwz TMP1, TAB:RB->array
4880    |  cmplw TMP0, RC
4881    |   slwi TMP0, RC, 3
4882    |.else
4883    |   bge cr1, >5
4884    |  // Convert number key to integer, check for integerness and range.
4885    |  fctiwz f1, f0
4886    |    fadd f2, f0, TOBIT
4887    |  stfd f1, TMPD
4888    |   lwz TMP0, TAB:RB->asize
4889    |    fsub f2, f2, TOBIT
4890    |  lwz TMP2, TMPD_LO
4891    |   lwz TMP1, TAB:RB->array
4892    |    fcmpu cr1, f0, f2
4893    |  cmplw cr0, TMP0, TMP2
4894    |  crand 4*cr0+gt, 4*cr0+gt, 4*cr1+eq
4895    |   slwi TMP0, TMP2, 3
4896    |.endif
4897    |  ble ->vmeta_tsetv		// Integer key and in array part?
4898    |   lwzx TMP2, TMP1, TMP0
4899    |  lbz TMP3, TAB:RB->marked
4900    |.if FPU
4901    |    lfdx f14, BASE, RA
4902    |.else
4903    |    add SAVE1, BASE, RA
4904    |    lwz SAVE0, 0(SAVE1)
4905    |    lwz SAVE1, 4(SAVE1)
4906    |.endif
4907    |   checknil TMP2; beq >3
4908    |1:
4909    |  andix. TMP2, TMP3, LJ_GC_BLACK	// isblack(table)
4910    |.if FPU
4911    |    stfdx f14, TMP1, TMP0
4912    |.else
4913    |    stwux SAVE0, TMP1, TMP0
4914    |    stw SAVE1, 4(TMP1)
4915    |.endif
4916    |  bne >7
4917    |2:
4918    |  ins_next
4919    |
4920    |3:  // Check for __newindex if previous value is nil.
4921    |  lwz TAB:TMP2, TAB:RB->metatable
4922    |  cmplwi TAB:TMP2, 0
4923    |  beq <1				// No metatable: done.
4924    |  lbz TMP2, TAB:TMP2->nomm
4925    |  andix. TMP2, TMP2, 1<<MM_newindex
4926    |  bne <1				// 'no __newindex' flag set: done.
4927    |  b ->vmeta_tsetv
4928    |
4929    |5:
4930    |  checkstr CARG2; bne ->vmeta_tsetv
4931    |.if not DUALNUM
4932    |  lwz STR:RC, 4(RC)
4933    |.endif
4934    |  b ->BC_TSETS_Z			// String key?
4935    |
4936    |7:  // Possible table write barrier for the value. Skip valiswhite check.
4937    |  barrierback TAB:RB, TMP3, TMP0
4938    |  b <2
4939    break;
4940  case BC_TSETS:
4941    |  // RA = src*8, RB = table*8, RC = str_const*8 (~)
4942    |  lwzux CARG1, RB, BASE
4943    |   srwi TMP1, RC, 1
4944    |    lwz TAB:RB, 4(RB)
4945    |   subfic TMP1, TMP1, -4
4946    |  checktab CARG1
4947    |   lwzx STR:RC, KBASE, TMP1	// KBASE-4-str_const*4
4948    |  bne ->vmeta_tsets1
4949    |->BC_TSETS_Z:
4950    |  // TAB:RB = GCtab *, STR:RC = GCstr *, RA = src*8
4951    |  lwz TMP0, TAB:RB->hmask
4952    |  lwz TMP1, STR:RC->sid
4953    |  lwz NODE:TMP2, TAB:RB->node
4954    |    stb ZERO, TAB:RB->nomm		// Clear metamethod cache.
4955    |  and TMP1, TMP1, TMP0		// idx = str->sid & tab->hmask
4956    |.if FPU
4957    |    lfdx f14, BASE, RA
4958    |.else
4959    |    add CARG2, BASE, RA
4960    |    lwz SAVE0, 0(CARG2)
4961    |    lwz SAVE1, 4(CARG2)
4962    |.endif
4963    |  slwi TMP0, TMP1, 5
4964    |  slwi TMP1, TMP1, 3
4965    |  sub TMP1, TMP0, TMP1
4966    |    lbz TMP3, TAB:RB->marked
4967    |  add NODE:TMP2, NODE:TMP2, TMP1	// node = tab->node + (idx*32-idx*8)
4968    |1:
4969    |  lwz CARG1, NODE:TMP2->key
4970    |   lwz TMP0, 4+offsetof(Node, key)(NODE:TMP2)
4971    |    lwz CARG2, NODE:TMP2->val
4972    |     lwz NODE:TMP1, NODE:TMP2->next
4973    |  checkstr CARG1; bne >5
4974    |   cmpw TMP0, STR:RC; bne >5
4975    |    checknil CARG2; beq >4		// Key found, but nil value?
4976    |2:
4977    |  andix. TMP0, TMP3, LJ_GC_BLACK	// isblack(table)
4978    |.if FPU
4979    |    stfd f14, NODE:TMP2->val
4980    |.else
4981    |    stw SAVE0, NODE:TMP2->val.u32.hi
4982    |    stw SAVE1, NODE:TMP2->val.u32.lo
4983    |.endif
4984    |  bne >7
4985    |3:
4986    |  ins_next
4987    |
4988    |4:  // Check for __newindex if previous value is nil.
4989    |  lwz TAB:TMP1, TAB:RB->metatable
4990    |  cmplwi TAB:TMP1, 0
4991    |  beq <2				// No metatable: done.
4992    |  lbz TMP0, TAB:TMP1->nomm
4993    |  andix. TMP0, TMP0, 1<<MM_newindex
4994    |  bne <2				// 'no __newindex' flag set: done.
4995    |  b ->vmeta_tsets
4996    |
4997    |5:  // Follow hash chain.
4998    |  cmplwi NODE:TMP1, 0
4999    |   mr NODE:TMP2, NODE:TMP1
5000    |  bne <1
5001    |  // End of hash chain: key not found, add a new one.
5002    |
5003    |  // But check for __newindex first.
5004    |  lwz TAB:TMP1, TAB:RB->metatable
5005    |   la CARG3, DISPATCH_GL(tmptv)(DISPATCH)
5006    |   stw PC, SAVE_PC
5007    |   mr CARG1, L
5008    |  cmplwi TAB:TMP1, 0
5009    |   stp BASE, L->base
5010    |  beq >6				// No metatable: continue.
5011    |  lbz TMP0, TAB:TMP1->nomm
5012    |  andix. TMP0, TMP0, 1<<MM_newindex
5013    |  beq ->vmeta_tsets		// 'no __newindex' flag NOT set: check.
5014    |6:
5015    |  li TMP0, LJ_TSTR
5016    |   stw STR:RC, 4(CARG3)
5017    |   mr CARG2, TAB:RB
5018    |  stw TMP0, 0(CARG3)
5019    |  bl extern lj_tab_newkey		// (lua_State *L, GCtab *t, TValue *k)
5020    |  // Returns TValue *.
5021    |  lp BASE, L->base
5022    |.if FPU
5023    |  stfd f14, 0(CRET1)
5024    |.else
5025    |  stw SAVE0, 0(CRET1)
5026    |  stw SAVE1, 4(CRET1)
5027    |.endif
5028    |  b <3				// No 2nd write barrier needed.
5029    |
5030    |7:  // Possible table write barrier for the value. Skip valiswhite check.
5031    |  barrierback TAB:RB, TMP3, TMP0
5032    |  b <3
5033    break;
5034  case BC_TSETB:
5035    |  // RA = src*8, RB = table*8, RC = index*8
5036    |  lwzux CARG1, RB, BASE
5037    |   srwi TMP0, RC, 3
5038    |   lwz TAB:RB, 4(RB)
5039    |  checktab CARG1; bne ->vmeta_tsetb
5040    |  lwz TMP1, TAB:RB->asize
5041    |   lwz TMP2, TAB:RB->array
5042    |    lbz TMP3, TAB:RB->marked
5043    |  cmplw TMP0, TMP1
5044    |.if FPU
5045    |   lfdx f14, BASE, RA
5046    |.else
5047    |   add CARG2, BASE, RA
5048    |   lwz SAVE0, 0(CARG2)
5049    |   lwz SAVE1, 4(CARG2)
5050    |.endif
5051    |  bge ->vmeta_tsetb
5052    |  lwzx TMP1, TMP2, RC
5053    |  checknil TMP1; beq >5
5054    |1:
5055    |  andix. TMP0, TMP3, LJ_GC_BLACK	// isblack(table)
5056    |.if FPU
5057    |   stfdx f14, TMP2, RC
5058    |.else
5059    |   stwux SAVE0, RC, TMP2
5060    |   stw SAVE1, 4(RC)
5061    |.endif
5062    |  bne >7
5063    |2:
5064    |  ins_next
5065    |
5066    |5:  // Check for __newindex if previous value is nil.
5067    |  lwz TAB:TMP1, TAB:RB->metatable
5068    |  cmplwi TAB:TMP1, 0
5069    |  beq <1				// No metatable: done.
5070    |  lbz TMP1, TAB:TMP1->nomm
5071    |  andix. TMP1, TMP1, 1<<MM_newindex
5072    |  bne <1				// 'no __newindex' flag set: done.
5073    |  b ->vmeta_tsetb			// Caveat: preserve TMP0!
5074    |
5075    |7:  // Possible table write barrier for the value. Skip valiswhite check.
5076    |  barrierback TAB:RB, TMP3, TMP0
5077    |  b <2
5078    break;
5079  case BC_TSETR:
5080    |  // RA = dst*8, RB = table*8, RC = key*8
5081    |  add RB, BASE, RB
5082    |  lwz TAB:CARG2, 4(RB)
5083    |.if DUALNUM
5084    |  add RC, BASE, RC
5085    |    lbz TMP3, TAB:CARG2->marked
5086    |  lwz TMP0, TAB:CARG2->asize
5087    |  lwz CARG3, 4(RC)
5088    |   lwz TMP1, TAB:CARG2->array
5089    |.else
5090    |  lfdx f0, BASE, RC
5091    |    lbz TMP3, TAB:CARG2->marked
5092    |  lwz TMP0, TAB:CARG2->asize
5093    |  toint CARG3, f0
5094    |   lwz TMP1, TAB:CARG2->array
5095    |.endif
5096    |  andix. TMP2, TMP3, LJ_GC_BLACK	// isblack(table)
5097    |  bne >7
5098    |2:
5099    |  cmplw TMP0, CARG3
5100    |   slwi TMP2, CARG3, 3
5101    |.if FPU
5102    |   lfdx f14, BASE, RA
5103    |.else
5104    |  lwzux SAVE0, RA, BASE
5105    |  lwz SAVE1, 4(RA)
5106    |.endif
5107    |  ble ->vmeta_tsetr		// In array part?
5108    |  ins_next1
5109    |.if FPU
5110    |   stfdx f14, TMP1, TMP2
5111    |.else
5112    |   stwux SAVE0, TMP1, TMP2
5113    |   stw SAVE1, 4(TMP1)
5114    |.endif
5115    |  ins_next2
5116    |
5117    |7:  // Possible table write barrier for the value. Skip valiswhite check.
5118    |  barrierback TAB:CARG2, TMP3, TMP2
5119    |  b <2
5120    break;
5121
5122
5123  case BC_TSETM:
5124    |  // RA = base*8 (table at base-1), RD = num_const*8 (start index)
5125    |  add RA, BASE, RA
5126    |1:
5127    |   add TMP3, KBASE, RD
5128    |  lwz TAB:CARG2, -4(RA)		// Guaranteed to be a table.
5129    |    addic. TMP0, MULTRES, -8
5130    |   lwz TMP3, 4(TMP3)		// Integer constant is in lo-word.
5131    |    srwi CARG3, TMP0, 3
5132    |    beq >4				// Nothing to copy?
5133    |  add CARG3, CARG3, TMP3
5134    |  lwz TMP2, TAB:CARG2->asize
5135    |   slwi TMP1, TMP3, 3
5136    |    lbz TMP3, TAB:CARG2->marked
5137    |  cmplw CARG3, TMP2
5138    |   add TMP2, RA, TMP0
5139    |   lwz TMP0, TAB:CARG2->array
5140    |  bgt >5
5141    |   add TMP1, TMP1, TMP0
5142    |    andix. TMP0, TMP3, LJ_GC_BLACK	// isblack(table)
5143    |3:  // Copy result slots to table.
5144    |.if FPU
5145    |   lfd f0, 0(RA)
5146    |.else
5147    |   lwz SAVE0, 0(RA)
5148    |   lwz SAVE1, 4(RA)
5149    |.endif
5150    |  addi RA, RA, 8
5151    |  cmpw cr1, RA, TMP2
5152    |.if FPU
5153    |   stfd f0, 0(TMP1)
5154    |.else
5155    |   stw SAVE0, 0(TMP1)
5156    |   stw SAVE1, 4(TMP1)
5157    |.endif
5158    |    addi TMP1, TMP1, 8
5159    |  blt cr1, <3
5160    |  bne >7
5161    |4:
5162    |  ins_next
5163    |
5164    |5:  // Need to resize array part.
5165    |   stp BASE, L->base
5166    |  mr CARG1, L
5167    |   stw PC, SAVE_PC
5168    |  mr SAVE0, RD
5169    |  bl extern lj_tab_reasize		// (lua_State *L, GCtab *t, int nasize)
5170    |  // Must not reallocate the stack.
5171    |  mr RD, SAVE0
5172    |  b <1
5173    |
5174    |7:  // Possible table write barrier for any value. Skip valiswhite check.
5175    |  barrierback TAB:CARG2, TMP3, TMP0
5176    |  b <4
5177    break;
5178
5179  /* -- Calls and vararg handling ----------------------------------------- */
5180
5181  case BC_CALLM:
5182    |  // RA = base*8, (RB = (nresults+1)*8,) RC = extra_nargs*8
5183    |  add NARGS8:RC, NARGS8:RC, MULTRES
5184    |  // Fall through. Assumes BC_CALL follows.
5185    break;
5186  case BC_CALL:
5187    |  // RA = base*8, (RB = (nresults+1)*8,) RC = (nargs+1)*8
5188    |  mr TMP2, BASE
5189    |  lwzux TMP0, BASE, RA
5190    |   lwz LFUNC:RB, 4(BASE)
5191    |    subi NARGS8:RC, NARGS8:RC, 8
5192    |   addi BASE, BASE, 8
5193    |  checkfunc TMP0; bne ->vmeta_call
5194    |  ins_call
5195    break;
5196
5197  case BC_CALLMT:
5198    |  // RA = base*8, (RB = 0,) RC = extra_nargs*8
5199    |  add NARGS8:RC, NARGS8:RC, MULTRES
5200    |  // Fall through. Assumes BC_CALLT follows.
5201    break;
5202  case BC_CALLT:
5203    |  // RA = base*8, (RB = 0,) RC = (nargs+1)*8
5204    |  lwzux TMP0, RA, BASE
5205    |   lwz LFUNC:RB, 4(RA)
5206    |    subi NARGS8:RC, NARGS8:RC, 8
5207    |    lwz TMP1, FRAME_PC(BASE)
5208    |  checkfunc TMP0
5209    |   addi RA, RA, 8
5210    |  bne ->vmeta_callt
5211    |->BC_CALLT_Z:
5212    |  andix. TMP0, TMP1, FRAME_TYPE	// Caveat: preserve cr0 until the crand.
5213    |   lbz TMP3, LFUNC:RB->ffid
5214    |    xori TMP2, TMP1, FRAME_VARG
5215    |    cmplwi cr1, NARGS8:RC, 0
5216    |  bne >7
5217    |1:
5218    |  stw LFUNC:RB, FRAME_FUNC(BASE)	// Copy function down, but keep PC.
5219    |  li TMP2, 0
5220    |   cmplwi cr7, TMP3, 1		// (> FF_C) Calling a fast function?
5221    |    beq cr1, >3
5222    |2:
5223    |  addi TMP3, TMP2, 8
5224    |.if FPU
5225    |   lfdx f0, RA, TMP2
5226    |.else
5227    |   add CARG3, RA, TMP2
5228    |   lwz CARG1, 0(CARG3)
5229    |   lwz CARG2, 4(CARG3)
5230    |.endif
5231    |  cmplw cr1, TMP3, NARGS8:RC
5232    |.if FPU
5233    |   stfdx f0, BASE, TMP2
5234    |.else
5235    |   stwux CARG1, TMP2, BASE
5236    |   stw CARG2, 4(TMP2)
5237    |.endif
5238    |  mr TMP2, TMP3
5239    |  bne cr1, <2
5240    |3:
5241    |  crand 4*cr0+eq, 4*cr0+eq, 4*cr7+gt
5242    |  beq >5
5243    |4:
5244    |  ins_callt
5245    |
5246    |5:  // Tailcall to a fast function with a Lua frame below.
5247    |  lwz INS, -4(TMP1)
5248    |  decode_RA8 RA, INS
5249    |  sub TMP1, BASE, RA
5250    |  lwz LFUNC:TMP1, FRAME_FUNC-8(TMP1)
5251    |  lwz TMP1, LFUNC:TMP1->pc
5252    |  lwz KBASE, PC2PROTO(k)(TMP1)	// Need to prepare KBASE.
5253    |  b <4
5254    |
5255    |7:  // Tailcall from a vararg function.
5256    |  andix. TMP0, TMP2, FRAME_TYPEP
5257    |  bne <1				// Vararg frame below?
5258    |  sub BASE, BASE, TMP2		// Relocate BASE down.
5259    |  lwz TMP1, FRAME_PC(BASE)
5260    |  andix. TMP0, TMP1, FRAME_TYPE
5261    |  b <1
5262    break;
5263
5264  case BC_ITERC:
5265    |  // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 ((2+1)*8))
5266    |  mr TMP2, BASE
5267    |  add BASE, BASE, RA
5268    |  lwz TMP1, -24(BASE)
5269    |   lwz LFUNC:RB, -20(BASE)
5270    |.if FPU
5271    |    lfd f1, -8(BASE)
5272    |    lfd f0, -16(BASE)
5273    |.else
5274    |    lwz CARG1, -8(BASE)
5275    |    lwz CARG2, -4(BASE)
5276    |    lwz CARG3, -16(BASE)
5277    |    lwz CARG4, -12(BASE)
5278    |.endif
5279    |  stw TMP1, 0(BASE)		// Copy callable.
5280    |   stw LFUNC:RB, 4(BASE)
5281    |  checkfunc TMP1
5282    |     li NARGS8:RC, 16		// Iterators get 2 arguments.
5283    |.if FPU
5284    |    stfd f1, 16(BASE)		// Copy control var.
5285    |    stfdu f0, 8(BASE)		// Copy state.
5286    |.else
5287    |    stw CARG1, 16(BASE)		// Copy control var.
5288    |    stw CARG2, 20(BASE)
5289    |    stwu CARG3, 8(BASE)		// Copy state.
5290    |    stw CARG4, 4(BASE)
5291    |.endif
5292    |  bne ->vmeta_call
5293    |  ins_call
5294    break;
5295
5296  case BC_ITERN:
5297    |  // RA = base*8, (RB = (nresults+1)*8, RC = (nargs+1)*8 (2+1)*8)
5298    |.if JIT
5299    |  // NYI on big-endian
5300    |.endif
5301    |->vm_IITERN:
5302    |  add RA, BASE, RA
5303    |  lwz TAB:RB, -12(RA)
5304    |  lwz RC, -4(RA)			// Get index from control var.
5305    |  lwz TMP0, TAB:RB->asize
5306    |  lwz TMP1, TAB:RB->array
5307    |   addi PC, PC, 4
5308    |1:  // Traverse array part.
5309    |  cmplw RC, TMP0
5310    |   slwi TMP3, RC, 3
5311    |  bge >5				// Index points after array part?
5312    |  lwzx TMP2, TMP1, TMP3
5313    |.if FPU
5314    |   lfdx f0, TMP1, TMP3
5315    |.else
5316    |   lwzux CARG1, TMP3, TMP1
5317    |   lwz CARG2, 4(TMP3)
5318    |.endif
5319    |  checknil TMP2
5320    |     lwz INS, -4(PC)
5321    |  beq >4
5322    |.if DUALNUM
5323    |   stw RC, 4(RA)
5324    |   stw TISNUM, 0(RA)
5325    |.else
5326    |   tonum_u f1, RC
5327    |.endif
5328    |    addi RC, RC, 1
5329    |     addis TMP3, PC, -(BCBIAS_J*4 >> 16)
5330    |.if FPU
5331    |  stfd f0, 8(RA)
5332    |.else
5333    |  stw CARG1, 8(RA)
5334    |  stw CARG2, 12(RA)
5335    |.endif
5336    |     decode_RD4 TMP1, INS
5337    |    stw RC, -4(RA)			// Update control var.
5338    |     add PC, TMP1, TMP3
5339    |.if not DUALNUM
5340    |   stfd f1, 0(RA)
5341    |.endif
5342    |3:
5343    |  ins_next
5344    |
5345    |4:  // Skip holes in array part.
5346    |  addi RC, RC, 1
5347    |  b <1
5348    |
5349    |5:  // Traverse hash part.
5350    |  lwz TMP1, TAB:RB->hmask
5351    |  sub RC, RC, TMP0
5352    |   lwz TMP2, TAB:RB->node
5353    |6:
5354    |  cmplw RC, TMP1			// End of iteration? Branch to ITERL+1.
5355    |   slwi TMP3, RC, 5
5356    |  bgty <3
5357    |   slwi RB, RC, 3
5358    |   sub TMP3, TMP3, RB
5359    |  lwzx RB, TMP2, TMP3
5360    |.if FPU
5361    |  lfdx f0, TMP2, TMP3
5362    |.else
5363    |  add CARG3, TMP2, TMP3
5364    |  lwz CARG1, 0(CARG3)
5365    |  lwz CARG2, 4(CARG3)
5366    |.endif
5367    |   add NODE:TMP3, TMP2, TMP3
5368    |  checknil RB
5369    |     lwz INS, -4(PC)
5370    |  beq >7
5371    |.if FPU
5372    |   lfd f1, NODE:TMP3->key
5373    |.else
5374    |   lwz CARG3, NODE:TMP3->key.u32.hi
5375    |   lwz CARG4, NODE:TMP3->key.u32.lo
5376    |.endif
5377    |     addis TMP2, PC, -(BCBIAS_J*4 >> 16)
5378    |.if FPU
5379    |  stfd f0, 8(RA)
5380    |.else
5381    |  stw CARG1, 8(RA)
5382    |  stw CARG2, 12(RA)
5383    |.endif
5384    |    add RC, RC, TMP0
5385    |     decode_RD4 TMP1, INS
5386    |.if FPU
5387    |   stfd f1, 0(RA)
5388    |.else
5389    |   stw CARG3, 0(RA)
5390    |   stw CARG4, 4(RA)
5391    |.endif
5392    |    addi RC, RC, 1
5393    |     add PC, TMP1, TMP2
5394    |    stw RC, -4(RA)			// Update control var.
5395    |  b <3
5396    |
5397    |7:  // Skip holes in hash part.
5398    |  addi RC, RC, 1
5399    |  b <6
5400    break;
5401
5402  case BC_ISNEXT:
5403    |  // RA = base*8, RD = target (points to ITERN)
5404    |  add RA, BASE, RA
5405    |  lwz TMP0, -24(RA)
5406    |  lwz CFUNC:TMP1, -20(RA)
5407    |   lwz TMP2, -16(RA)
5408    |    lwz TMP3, -8(RA)
5409    |   cmpwi cr0, TMP2, LJ_TTAB
5410    |  cmpwi cr1, TMP0, LJ_TFUNC
5411    |    cmpwi cr6, TMP3, LJ_TNIL
5412    |  bne cr1, >5
5413    |  lbz TMP1, CFUNC:TMP1->ffid
5414    |   crand 4*cr0+eq, 4*cr0+eq, 4*cr6+eq
5415    |  cmpwi cr7, TMP1, FF_next_N
5416    |    srwi TMP0, RD, 1
5417    |  crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq
5418    |    add TMP3, PC, TMP0
5419    |  bne cr0, >5
5420    |  lus TMP1, (LJ_KEYINDEX >> 16)
5421    |  ori TMP1, TMP1, (LJ_KEYINDEX & 0xffff)
5422    |  stw ZERO, -4(RA)			// Initialize control var.
5423    |  stw TMP1, -8(RA)
5424    |    addis PC, TMP3, -(BCBIAS_J*4 >> 16)
5425    |1:
5426    |  ins_next
5427    |5:  // Despecialize bytecode if any of the checks fail.
5428    |  li TMP0, BC_JMP
5429    |   li TMP1, BC_ITERC
5430    |  stb TMP0, -1(PC)
5431    |    addis PC, TMP3, -(BCBIAS_J*4 >> 16)
5432    |  // NYI on big-endian: unpatch JLOOP.
5433    |   stb TMP1, 3(PC)
5434    |  b <1
5435    break;
5436
5437  case BC_VARG:
5438    |  // RA = base*8, RB = (nresults+1)*8, RC = numparams*8
5439    |  lwz TMP0, FRAME_PC(BASE)
5440    |  add RC, BASE, RC
5441    |   add RA, BASE, RA
5442    |  addi RC, RC, FRAME_VARG
5443    |   add TMP2, RA, RB
5444    |  subi TMP3, BASE, 8		// TMP3 = vtop
5445    |  sub RC, RC, TMP0			// RC = vbase
5446    |  // Note: RC may now be even _above_ BASE if nargs was < numparams.
5447    |  cmplwi cr1, RB, 0
5448    |.if PPE
5449    |   sub TMP1, TMP3, RC
5450    |   cmpwi TMP1, 0
5451    |.else
5452    |   sub. TMP1, TMP3, RC
5453    |.endif
5454    |  beq cr1, >5			// Copy all varargs?
5455    |   subi TMP2, TMP2, 16
5456    |   ble >2				// No vararg slots?
5457    |1:  // Copy vararg slots to destination slots.
5458    |.if FPU
5459    |  lfd f0, 0(RC)
5460    |.else
5461    |  lwz CARG1, 0(RC)
5462    |  lwz CARG2, 4(RC)
5463    |.endif
5464    |   addi RC, RC, 8
5465    |.if FPU
5466    |  stfd f0, 0(RA)
5467    |.else
5468    |  stw CARG1, 0(RA)
5469    |  stw CARG2, 4(RA)
5470    |.endif
5471    |  cmplw RA, TMP2
5472    |   cmplw cr1, RC, TMP3
5473    |  bge >3				// All destination slots filled?
5474    |    addi RA, RA, 8
5475    |   blt cr1, <1			// More vararg slots?
5476    |2:  // Fill up remainder with nil.
5477    |  stw TISNIL, 0(RA)
5478    |  cmplw RA, TMP2
5479    |   addi RA, RA, 8
5480    |  blt <2
5481    |3:
5482    |  ins_next
5483    |
5484    |5:  // Copy all varargs.
5485    |  lwz TMP0, L->maxstack
5486    |   li MULTRES, 8			// MULTRES = (0+1)*8
5487    |  bley <3				// No vararg slots?
5488    |  add TMP2, RA, TMP1
5489    |  cmplw TMP2, TMP0
5490    |   addi MULTRES, TMP1, 8
5491    |  bgt >7
5492    |6:
5493    |.if FPU
5494    |  lfd f0, 0(RC)
5495    |.else
5496    |  lwz CARG1, 0(RC)
5497    |  lwz CARG2, 4(RC)
5498    |.endif
5499    |   addi RC, RC, 8
5500    |.if FPU
5501    |  stfd f0, 0(RA)
5502    |.else
5503    |  stw CARG1, 0(RA)
5504    |  stw CARG2, 4(RA)
5505    |.endif
5506    |  cmplw RC, TMP3
5507    |   addi RA, RA, 8
5508    |  blt <6				// More vararg slots?
5509    |  b <3
5510    |
5511    |7:  // Grow stack for varargs.
5512    |  mr CARG1, L
5513    |   stp RA, L->top
5514    |  sub SAVE0, RC, BASE		// Need delta, because BASE may change.
5515    |   stp BASE, L->base
5516    |  sub RA, RA, BASE
5517    |   stw PC, SAVE_PC
5518    |  srwi CARG2, TMP1, 3
5519    |  bl extern lj_state_growstack	// (lua_State *L, int n)
5520    |  lp BASE, L->base
5521    |  add RA, BASE, RA
5522    |  add RC, BASE, SAVE0
5523    |  subi TMP3, BASE, 8
5524    |  b <6
5525    break;
5526
5527  /* -- Returns ----------------------------------------------------------- */
5528
5529  case BC_RETM:
5530    |  // RA = results*8, RD = extra_nresults*8
5531    |  add RD, RD, MULTRES		// MULTRES >= 8, so RD >= 8.
5532    |  // Fall through. Assumes BC_RET follows.
5533    break;
5534
5535  case BC_RET:
5536    |  // RA = results*8, RD = (nresults+1)*8
5537    |  lwz PC, FRAME_PC(BASE)
5538    |   add RA, BASE, RA
5539    |    mr MULTRES, RD
5540    |1:
5541    |  andix. TMP0, PC, FRAME_TYPE
5542    |   xori TMP1, PC, FRAME_VARG
5543    |  bne ->BC_RETV_Z
5544    |
5545    |->BC_RET_Z:
5546    |  // BASE = base, RA = resultptr, RD = (nresults+1)*8, PC = return
5547    |   lwz INS, -4(PC)
5548    |  cmpwi RD, 8
5549    |   subi TMP2, BASE, 8
5550    |   subi RC, RD, 8
5551    |   decode_RB8 RB, INS
5552    |  beq >3
5553    |   li TMP1, 0
5554    |2:
5555    |  addi TMP3, TMP1, 8
5556    |.if FPU
5557    |   lfdx f0, RA, TMP1
5558    |.else
5559    |   add CARG3, RA, TMP1
5560    |   lwz CARG1, 0(CARG3)
5561    |   lwz CARG2, 4(CARG3)
5562    |.endif
5563    |  cmpw TMP3, RC
5564    |.if FPU
5565    |   stfdx f0, TMP2, TMP1
5566    |.else
5567    |   add CARG3, TMP2, TMP1
5568    |   stw CARG1, 0(CARG3)
5569    |   stw CARG2, 4(CARG3)
5570    |.endif
5571    |  beq >3
5572    |  addi TMP1, TMP3, 8
5573    |.if FPU
5574    |   lfdx f1, RA, TMP3
5575    |.else
5576    |   add CARG3, RA, TMP3
5577    |   lwz CARG1, 0(CARG3)
5578    |   lwz CARG2, 4(CARG3)
5579    |.endif
5580    |  cmpw TMP1, RC
5581    |.if FPU
5582    |   stfdx f1, TMP2, TMP3
5583    |.else
5584    |   add CARG3, TMP2, TMP3
5585    |   stw CARG1, 0(CARG3)
5586    |   stw CARG2, 4(CARG3)
5587    |.endif
5588    |  bne <2
5589    |3:
5590    |5:
5591    |  cmplw RB, RD
5592    |   decode_RA8 RA, INS
5593    |  bgt >6
5594    |   sub BASE, TMP2, RA
5595    |  lwz LFUNC:TMP1, FRAME_FUNC(BASE)
5596    |  ins_next1
5597    |  lwz TMP1, LFUNC:TMP1->pc
5598    |  lwz KBASE, PC2PROTO(k)(TMP1)
5599    |  ins_next2
5600    |
5601    |6:  // Fill up results with nil.
5602    |  subi TMP1, RD, 8
5603    |   addi RD, RD, 8
5604    |  stwx TISNIL, TMP2, TMP1
5605    |  b <5
5606    |
5607    |->BC_RETV_Z:  // Non-standard return case.
5608    |  andix. TMP2, TMP1, FRAME_TYPEP
5609    |  bne ->vm_return
5610    |  // Return from vararg function: relocate BASE down.
5611    |  sub BASE, BASE, TMP1
5612    |  lwz PC, FRAME_PC(BASE)
5613    |  b <1
5614    break;
5615
5616  case BC_RET0: case BC_RET1:
5617    |  // RA = results*8, RD = (nresults+1)*8
5618    |  lwz PC, FRAME_PC(BASE)
5619    |   add RA, BASE, RA
5620    |    mr MULTRES, RD
5621    |  andix. TMP0, PC, FRAME_TYPE
5622    |   xori TMP1, PC, FRAME_VARG
5623    |  bney ->BC_RETV_Z
5624    |
5625    |  lwz INS, -4(PC)
5626    |   subi TMP2, BASE, 8
5627    |  decode_RB8 RB, INS
5628    if (op == BC_RET1) {
5629      |.if FPU
5630      |  lfd f0, 0(RA)
5631      |  stfd f0, 0(TMP2)
5632      |.else
5633      |  lwz CARG1, 0(RA)
5634      |  lwz CARG2, 4(RA)
5635      |  stw CARG1, 0(TMP2)
5636      |  stw CARG2, 4(TMP2)
5637      |.endif
5638    }
5639    |5:
5640    |  cmplw RB, RD
5641    |   decode_RA8 RA, INS
5642    |  bgt >6
5643    |   sub BASE, TMP2, RA
5644    |  lwz LFUNC:TMP1, FRAME_FUNC(BASE)
5645    |  ins_next1
5646    |  lwz TMP1, LFUNC:TMP1->pc
5647    |  lwz KBASE, PC2PROTO(k)(TMP1)
5648    |  ins_next2
5649    |
5650    |6:  // Fill up results with nil.
5651    |  subi TMP1, RD, 8
5652    |   addi RD, RD, 8
5653    |  stwx TISNIL, TMP2, TMP1
5654    |  b <5
5655    break;
5656
5657  /* -- Loops and branches ------------------------------------------------ */
5658
5659  case BC_FORL:
5660    |.if JIT
5661    |  hotloop
5662    |.endif
5663    |  // Fall through. Assumes BC_IFORL follows.
5664    break;
5665
5666  case BC_JFORI:
5667  case BC_JFORL:
5668#if !LJ_HASJIT
5669    break;
5670#endif
5671  case BC_FORI:
5672  case BC_IFORL:
5673    |  // RA = base*8, RD = target (after end of loop or start of loop)
5674    vk = (op == BC_IFORL || op == BC_JFORL);
5675    |.if DUALNUM
5676    |  // Integer loop.
5677    |  lwzux TMP1, RA, BASE
5678    |   lwz CARG1, FORL_IDX*8+4(RA)
5679    |  cmplw cr0, TMP1, TISNUM
5680    if (vk) {
5681      |   lwz CARG3, FORL_STEP*8+4(RA)
5682      |  bne >9
5683      |.if GPR64
5684      |  // Need to check overflow for (a<<32) + (b<<32).
5685      |  rldicr TMP0, CARG1, 32, 31
5686      |  rldicr TMP2, CARG3, 32, 31
5687      |  add CARG1, CARG1, CARG3
5688      |  addo. TMP0, TMP0, TMP2
5689      |.else
5690      |  addo. CARG1, CARG1, CARG3
5691      |.endif
5692      |    cmpwi cr6, CARG3, 0
5693      |   lwz CARG2, FORL_STOP*8+4(RA)
5694      |  bso >6
5695      |4:
5696      |  stw CARG1, FORL_IDX*8+4(RA)
5697    } else {
5698      |  lwz SAVE0, FORL_STEP*8(RA)
5699      |   lwz CARG3, FORL_STEP*8+4(RA)
5700      |  lwz TMP2, FORL_STOP*8(RA)
5701      |   lwz CARG2, FORL_STOP*8+4(RA)
5702      |  cmplw cr7, SAVE0, TISNUM
5703      |  cmplw cr1, TMP2, TISNUM
5704      |  crand 4*cr0+eq, 4*cr0+eq, 4*cr7+eq
5705      |  crand 4*cr0+eq, 4*cr0+eq, 4*cr1+eq
5706      |    cmpwi cr6, CARG3, 0
5707      |  bne >9
5708    }
5709    |    blt cr6, >5
5710    |  cmpw CARG1, CARG2
5711    |1:
5712    |   stw TISNUM, FORL_EXT*8(RA)
5713    if (op != BC_JFORL) {
5714      |  srwi RD, RD, 1
5715    }
5716    |   stw CARG1, FORL_EXT*8+4(RA)
5717    if (op != BC_JFORL) {
5718      |  add RD, PC, RD
5719    }
5720    if (op == BC_FORI) {
5721      |  bgt >3  // See FP loop below.
5722    } else if (op == BC_JFORI) {
5723      |  addis PC, RD, -(BCBIAS_J*4 >> 16)
5724      |  bley >7
5725    } else if (op == BC_IFORL) {
5726      |  bgt >2
5727      |  addis PC, RD, -(BCBIAS_J*4 >> 16)
5728    } else {
5729      |  bley =>BC_JLOOP
5730    }
5731    |2:
5732    |  ins_next
5733    |5:  // Invert check for negative step.
5734    |  cmpw CARG2, CARG1
5735    |  b <1
5736    if (vk) {
5737      |6:  // Potential overflow.
5738      |  checkov TMP0, <4		// Ignore unrelated overflow.
5739      |  b <2
5740    }
5741    |.endif
5742    if (vk) {
5743      |.if DUALNUM
5744      |9:  // FP loop.
5745      |.if FPU
5746      |  lfd f1, FORL_IDX*8(RA)
5747      |.else
5748      |  lwz CARG1, FORL_IDX*8(RA)
5749      |  lwz CARG2, FORL_IDX*8+4(RA)
5750      |.endif
5751      |.else
5752      |  lfdux f1, RA, BASE
5753      |.endif
5754      |.if FPU
5755      |  lfd f3, FORL_STEP*8(RA)
5756      |  lfd f2, FORL_STOP*8(RA)
5757      |  fadd f1, f1, f3
5758      |  stfd f1, FORL_IDX*8(RA)
5759      |.else
5760      |  lwz CARG3, FORL_STEP*8(RA)
5761      |  lwz CARG4, FORL_STEP*8+4(RA)
5762      |  mr SAVE1, RD
5763      |  blex __adddf3
5764      |  mr RD, SAVE1
5765      |  stw CRET1, FORL_IDX*8(RA)
5766      |  stw CRET2, FORL_IDX*8+4(RA)
5767      |  lwz CARG3, FORL_STOP*8(RA)
5768      |  lwz CARG4, FORL_STOP*8+4(RA)
5769      |.endif
5770      |   lwz SAVE0, FORL_STEP*8(RA)
5771    } else {
5772      |.if DUALNUM
5773      |9:  // FP loop.
5774      |.else
5775      |  lwzux TMP1, RA, BASE
5776      |  lwz SAVE0, FORL_STEP*8(RA)
5777      |  lwz TMP2, FORL_STOP*8(RA)
5778      |  cmplw cr0, TMP1, TISNUM
5779      |  cmplw cr7, SAVE0, TISNUM
5780      |  cmplw cr1, TMP2, TISNUM
5781      |.endif
5782      |.if FPU
5783      |   lfd f1, FORL_IDX*8(RA)
5784      |.else
5785      |   lwz CARG1, FORL_IDX*8(RA)
5786      |   lwz CARG2, FORL_IDX*8+4(RA)
5787      |.endif
5788      |  crand 4*cr0+lt, 4*cr0+lt, 4*cr7+lt
5789      |  crand 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
5790      |.if FPU
5791      |   lfd f2, FORL_STOP*8(RA)
5792      |.else
5793      |   lwz CARG3, FORL_STOP*8(RA)
5794      |   lwz CARG4, FORL_STOP*8+4(RA)
5795      |.endif
5796      |  bge ->vmeta_for
5797    }
5798    |  cmpwi cr6, SAVE0, 0
5799    if (op != BC_JFORL) {
5800      |  srwi RD, RD, 1
5801    }
5802    |.if FPU
5803    |   stfd f1, FORL_EXT*8(RA)
5804    |.else
5805    |   stw CARG1, FORL_EXT*8(RA)
5806    |   stw CARG2, FORL_EXT*8+4(RA)
5807    |.endif
5808    if (op != BC_JFORL) {
5809      |  add RD, PC, RD
5810    }
5811    |.if FPU
5812    |  fcmpu cr0, f1, f2
5813    |.else
5814    |  mr SAVE1, RD
5815    |  blex __ledf2
5816    |  cmpwi CRET1, 0
5817    |  mr RD, SAVE1
5818    |.endif
5819    if (op == BC_JFORI) {
5820      |  addis PC, RD, -(BCBIAS_J*4 >> 16)
5821    }
5822    |  blt cr6, >5
5823    if (op == BC_FORI) {
5824      |  bgt >3
5825    } else if (op == BC_IFORL) {
5826      |.if DUALNUM
5827      |  bgty <2
5828      |.else
5829      |  bgt >2
5830      |.endif
5831      |1:
5832      |  addis PC, RD, -(BCBIAS_J*4 >> 16)
5833    } else if (op == BC_JFORI) {
5834      |  bley >7
5835    } else {
5836      |  bley =>BC_JLOOP
5837    }
5838    |.if DUALNUM
5839    |  b <2
5840    |.else
5841    |2:
5842    |  ins_next
5843    |.endif
5844    |5:  // Negative step.
5845    if (op == BC_FORI) {
5846      |  bge <2
5847      |3:  // Used by integer loop, too.
5848      |  addis PC, RD, -(BCBIAS_J*4 >> 16)
5849    } else if (op == BC_IFORL) {
5850      |  bgey <1
5851    } else if (op == BC_JFORI) {
5852      |  bgey >7
5853    } else {
5854      |  bgey =>BC_JLOOP
5855    }
5856    |  b <2
5857    if (op == BC_JFORI) {
5858      |7:
5859      |  lwz INS, -4(PC)
5860      |  decode_RD8 RD, INS
5861      |  b =>BC_JLOOP
5862    }
5863    break;
5864
5865  case BC_ITERL:
5866    |.if JIT
5867    |  hotloop
5868    |.endif
5869    |  // Fall through. Assumes BC_IITERL follows.
5870    break;
5871
5872  case BC_JITERL:
5873#if !LJ_HASJIT
5874    break;
5875#endif
5876  case BC_IITERL:
5877    |  // RA = base*8, RD = target
5878    |  lwzux TMP1, RA, BASE
5879    |   lwz TMP2, 4(RA)
5880    |  checknil TMP1; beq >1		// Stop if iterator returned nil.
5881    if (op == BC_JITERL) {
5882      |  stw TMP1, -8(RA)
5883      |   stw TMP2, -4(RA)
5884      |  b =>BC_JLOOP
5885    } else {
5886      |  branch_RD			// Otherwise save control var + branch.
5887      |  stw TMP1, -8(RA)
5888      |   stw TMP2, -4(RA)
5889    }
5890    |1:
5891    |  ins_next
5892    break;
5893
5894  case BC_LOOP:
5895    |  // RA = base*8, RD = target (loop extent)
5896    |  // Note: RA/RD is only used by trace recorder to determine scope/extent
5897    |  // This opcode does NOT jump, it's only purpose is to detect a hot loop.
5898    |.if JIT
5899    |  hotloop
5900    |.endif
5901    |  // Fall through. Assumes BC_ILOOP follows.
5902    break;
5903
5904  case BC_ILOOP:
5905    |  // RA = base*8, RD = target (loop extent)
5906    |  ins_next
5907    break;
5908
5909  case BC_JLOOP:
5910    |.if JIT
5911    |  // RA = base*8 (ignored), RD = traceno*8
5912    |  lwz TMP1, DISPATCH_J(trace)(DISPATCH)
5913    |  srwi RD, RD, 1
5914    |  // Traces on PPC don't store the trace number, so use 0.
5915    |   stw ZERO, DISPATCH_GL(vmstate)(DISPATCH)
5916    |  lwzx TRACE:TMP2, TMP1, RD
5917    |  clrso TMP1
5918    |  lp TMP2, TRACE:TMP2->mcode
5919    |   stw BASE, DISPATCH_GL(jit_base)(DISPATCH)
5920    |  mtctr TMP2
5921    |   addi JGL, DISPATCH, GG_DISP2G+32768
5922    |   stw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
5923    |  bctr
5924    |.endif
5925    break;
5926
5927  case BC_JMP:
5928    |  // RA = base*8 (only used by trace recorder), RD = target
5929    |  branch_RD
5930    |  ins_next
5931    break;
5932
5933  /* -- Function headers -------------------------------------------------- */
5934
5935  case BC_FUNCF:
5936    |.if JIT
5937    |  hotcall
5938    |.endif
5939  case BC_FUNCV:  /* NYI: compiled vararg functions. */
5940    |  // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
5941    break;
5942
5943  case BC_JFUNCF:
5944#if !LJ_HASJIT
5945    break;
5946#endif
5947  case BC_IFUNCF:
5948    |  // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
5949    |  lwz TMP2, L->maxstack
5950    |   lbz TMP1, -4+PC2PROTO(numparams)(PC)
5951    |    lwz KBASE, -4+PC2PROTO(k)(PC)
5952    |  cmplw RA, TMP2
5953    |   slwi TMP1, TMP1, 3
5954    |  bgt ->vm_growstack_l
5955    if (op != BC_JFUNCF) {
5956      |  ins_next1
5957    }
5958    |2:
5959    |  cmplw NARGS8:RC, TMP1		// Check for missing parameters.
5960    |  blt >3
5961    if (op == BC_JFUNCF) {
5962      |  decode_RD8 RD, INS
5963      |  b =>BC_JLOOP
5964    } else {
5965      |  ins_next2
5966    }
5967    |
5968    |3:  // Clear missing parameters.
5969    |  stwx TISNIL, BASE, NARGS8:RC
5970    |  addi NARGS8:RC, NARGS8:RC, 8
5971    |  b <2
5972    break;
5973
5974  case BC_JFUNCV:
5975#if !LJ_HASJIT
5976    break;
5977#endif
5978    |  NYI  // NYI: compiled vararg functions
5979    break;  /* NYI: compiled vararg functions. */
5980
5981  case BC_IFUNCV:
5982    |  // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
5983    |  lwz TMP2, L->maxstack
5984    |   add TMP1, BASE, RC
5985    |  add TMP0, RA, RC
5986    |   stw LFUNC:RB, 4(TMP1)		// Store copy of LFUNC.
5987    |   addi TMP3, RC, 8+FRAME_VARG
5988    |    lwz KBASE, -4+PC2PROTO(k)(PC)
5989    |  cmplw TMP0, TMP2
5990    |   stw TMP3, 0(TMP1)		// Store delta + FRAME_VARG.
5991    |  bge ->vm_growstack_l
5992    |  lbz TMP2, -4+PC2PROTO(numparams)(PC)
5993    |   mr RA, BASE
5994    |   mr RC, TMP1
5995    |  ins_next1
5996    |  cmpwi TMP2, 0
5997    |   addi BASE, TMP1, 8
5998    |  beq >3
5999    |1:
6000    |  cmplw RA, RC			// Less args than parameters?
6001    |   lwz TMP0, 0(RA)
6002    |   lwz TMP3, 4(RA)
6003    |  bge >4
6004    |    stw TISNIL, 0(RA)		// Clear old fixarg slot (help the GC).
6005    |    addi RA, RA, 8
6006    |2:
6007    |  addic. TMP2, TMP2, -1
6008    |   stw TMP0, 8(TMP1)
6009    |   stw TMP3, 12(TMP1)
6010    |    addi TMP1, TMP1, 8
6011    |  bne <1
6012    |3:
6013    |  ins_next2
6014    |
6015    |4:  // Clear missing parameters.
6016    |  li TMP0, LJ_TNIL
6017    |  b <2
6018    break;
6019
6020  case BC_FUNCC:
6021  case BC_FUNCCW:
6022    |  // BASE = new base, RA = BASE+framesize*8, RB = CFUNC, RC = nargs*8
6023    if (op == BC_FUNCC) {
6024      |  lp RD, CFUNC:RB->f
6025    } else {
6026      |  lp RD, DISPATCH_GL(wrapf)(DISPATCH)
6027    }
6028    |   add TMP1, RA, NARGS8:RC
6029    |   lwz TMP2, L->maxstack
6030    |  .toc lp TMP3, 0(RD)
6031    |    add RC, BASE, NARGS8:RC
6032    |   stp BASE, L->base
6033    |   cmplw TMP1, TMP2
6034    |    stp RC, L->top
6035    |     li_vmstate C
6036    |.if TOC
6037    |  mtctr TMP3
6038    |.else
6039    |  mtctr RD
6040    |.endif
6041    if (op == BC_FUNCCW) {
6042      |  lp CARG2, CFUNC:RB->f
6043    }
6044    |  mr CARG1, L
6045    |   bgt ->vm_growstack_c		// Need to grow stack.
6046    |  .toc lp TOCREG, TOC_OFS(RD)
6047    |  .tocenv lp ENVREG, ENV_OFS(RD)
6048    |     st_vmstate
6049    |  bctrl				// (lua_State *L [, lua_CFunction f])
6050    |  // Returns nresults.
6051    |  lp BASE, L->base
6052    |  .toc ld TOCREG, SAVE_TOC
6053    |   slwi RD, CRET1, 3
6054    |  lp TMP1, L->top
6055    |    li_vmstate INTERP
6056    |  lwz PC, FRAME_PC(BASE)		// Fetch PC of caller.
6057    |    stw L, DISPATCH_GL(cur_L)(DISPATCH)
6058    |   sub RA, TMP1, RD		// RA = L->top - nresults*8
6059    |    st_vmstate
6060    |  b ->vm_returnc
6061    break;
6062
6063  /* ---------------------------------------------------------------------- */
6064
6065  default:
6066    fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
6067    exit(2);
6068    break;
6069  }
6070}
6071
6072static int build_backend(BuildCtx *ctx)
6073{
6074  int op;
6075
6076  dasm_growpc(Dst, BC__MAX);
6077
6078  build_subroutines(ctx);
6079
6080  |.code_op
6081  for (op = 0; op < BC__MAX; op++)
6082    build_ins(ctx, (BCOp)op, op);
6083
6084  return BC__MAX;
6085}
6086
6087/* Emit pseudo frame-info for all assembler functions. */
6088static void emit_asm_debug(BuildCtx *ctx)
6089{
6090  int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
6091  int i;
6092  switch (ctx->mode) {
6093  case BUILD_elfasm:
6094    fprintf(ctx->fp, "\t.section .debug_frame,\"\",@progbits\n");
6095    fprintf(ctx->fp,
6096	".Lframe0:\n"
6097	"\t.long .LECIE0-.LSCIE0\n"
6098	".LSCIE0:\n"
6099	"\t.long 0xffffffff\n"
6100	"\t.byte 0x1\n"
6101	"\t.string \"\"\n"
6102	"\t.uleb128 0x1\n"
6103	"\t.sleb128 -4\n"
6104	"\t.byte 65\n"
6105	"\t.byte 0xc\n\t.uleb128 1\n\t.uleb128 0\n"
6106	"\t.align 2\n"
6107	".LECIE0:\n\n");
6108    fprintf(ctx->fp,
6109	".LSFDE0:\n"
6110	"\t.long .LEFDE0-.LASFDE0\n"
6111	".LASFDE0:\n"
6112	"\t.long .Lframe0\n"
6113	"\t.long .Lbegin\n"
6114	"\t.long %d\n"
6115	"\t.byte 0xe\n\t.uleb128 %d\n"
6116	"\t.byte 0x11\n\t.uleb128 65\n\t.sleb128 -1\n"
6117	"\t.byte 0x5\n\t.uleb128 70\n\t.uleb128 55\n",
6118	fcofs, CFRAME_SIZE);
6119    for (i = 14; i <= 31; i++)
6120      fprintf(ctx->fp,
6121	"\t.byte %d\n\t.uleb128 %d\n"
6122	"\t.byte %d\n\t.uleb128 %d\n",
6123	0x80+i, 37+(31-i), 0x80+32+i, 2+2*(31-i));
6124    fprintf(ctx->fp,
6125	"\t.align 2\n"
6126	".LEFDE0:\n\n");
6127#if LJ_HASFFI
6128    fprintf(ctx->fp,
6129	".LSFDE1:\n"
6130	"\t.long .LEFDE1-.LASFDE1\n"
6131	".LASFDE1:\n"
6132	"\t.long .Lframe0\n"
6133#if LJ_TARGET_PS3
6134	"\t.long .lj_vm_ffi_call\n"
6135#else
6136	"\t.long lj_vm_ffi_call\n"
6137#endif
6138	"\t.long %d\n"
6139	"\t.byte 0x11\n\t.uleb128 65\n\t.sleb128 -1\n"
6140	"\t.byte 0x8e\n\t.uleb128 2\n"
6141	"\t.byte 0xd\n\t.uleb128 0xe\n"
6142	"\t.align 2\n"
6143	".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
6144#endif
6145#if !LJ_NO_UNWIND
6146    fprintf(ctx->fp, "\t.section .eh_frame,\"a\",@progbits\n");
6147    fprintf(ctx->fp,
6148	".Lframe1:\n"
6149	"\t.long .LECIE1-.LSCIE1\n"
6150	".LSCIE1:\n"
6151	"\t.long 0\n"
6152	"\t.byte 0x1\n"
6153	"\t.string \"zPR\"\n"
6154	"\t.uleb128 0x1\n"
6155	"\t.sleb128 -4\n"
6156	"\t.byte 65\n"
6157	"\t.uleb128 6\n"			/* augmentation length */
6158	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
6159	"\t.long lj_err_unwind_dwarf-.\n"
6160	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
6161	"\t.byte 0xc\n\t.uleb128 1\n\t.uleb128 0\n"
6162	"\t.align 2\n"
6163	".LECIE1:\n\n");
6164    fprintf(ctx->fp,
6165	".LSFDE2:\n"
6166	"\t.long .LEFDE2-.LASFDE2\n"
6167	".LASFDE2:\n"
6168	"\t.long .LASFDE2-.Lframe1\n"
6169	"\t.long .Lbegin-.\n"
6170	"\t.long %d\n"
6171	"\t.uleb128 0\n"			/* augmentation length */
6172	"\t.byte 0xe\n\t.uleb128 %d\n"
6173	"\t.byte 0x11\n\t.uleb128 65\n\t.sleb128 -1\n"
6174	"\t.byte 0x5\n\t.uleb128 70\n\t.uleb128 55\n",
6175	fcofs, CFRAME_SIZE);
6176    for (i = 14; i <= 31; i++)
6177      fprintf(ctx->fp,
6178	"\t.byte %d\n\t.uleb128 %d\n"
6179	"\t.byte %d\n\t.uleb128 %d\n",
6180	0x80+i, 37+(31-i), 0x80+32+i, 2+2*(31-i));
6181    fprintf(ctx->fp,
6182	"\t.align 2\n"
6183	".LEFDE2:\n\n");
6184#if LJ_HASFFI
6185    fprintf(ctx->fp,
6186	".Lframe2:\n"
6187	"\t.long .LECIE2-.LSCIE2\n"
6188	".LSCIE2:\n"
6189	"\t.long 0\n"
6190	"\t.byte 0x1\n"
6191	"\t.string \"zR\"\n"
6192	"\t.uleb128 0x1\n"
6193	"\t.sleb128 -4\n"
6194	"\t.byte 65\n"
6195	"\t.uleb128 1\n"			/* augmentation length */
6196	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
6197	"\t.byte 0xc\n\t.uleb128 1\n\t.uleb128 0\n"
6198	"\t.align 2\n"
6199	".LECIE2:\n\n");
6200    fprintf(ctx->fp,
6201	".LSFDE3:\n"
6202	"\t.long .LEFDE3-.LASFDE3\n"
6203	".LASFDE3:\n"
6204	"\t.long .LASFDE3-.Lframe2\n"
6205	"\t.long lj_vm_ffi_call-.\n"
6206	"\t.long %d\n"
6207	"\t.uleb128 0\n"			/* augmentation length */
6208	"\t.byte 0x11\n\t.uleb128 65\n\t.sleb128 -1\n"
6209	"\t.byte 0x8e\n\t.uleb128 2\n"
6210	"\t.byte 0xd\n\t.uleb128 0xe\n"
6211	"\t.align 2\n"
6212	".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
6213#endif
6214#endif
6215    break;
6216  default:
6217    break;
6218  }
6219}
6220