luna - luajit/src/vm

   1|// Low-level VM code for ARM CPUs.
   2|// Bytecode interpreter, fast functions and helper functions.
   3|// Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4|
   5|.arch arm
   6|.section code_op, code_sub
   7|
   8|.actionlist build_actionlist
   9|.globals GLOB_
  10|.globalnames globnames
  11|.externnames extnames
  12|
  13|// Note: The ragged indentation of the instructions is intentional.
  14|//       The starting columns indicate data dependencies.
  15|
  16|//-----------------------------------------------------------------------
  17|
  18|// Fixed register assignments for the interpreter.
  19|
  20|// The following must be C callee-save.
  21|.define MASKR8,	r4	// 255*8 constant for fast bytecode decoding.
  22|.define KBASE,		r5	// Constants of current Lua function.
  23|.define PC,		r6	// Next PC.
  24|.define DISPATCH,	r7	// Opcode dispatch table.
  25|.define LREG,		r8	// Register holding lua_State (also in SAVE_L).
  26|
  27|// C callee-save in EABI, but often refetched. Temporary in iOS 3.0+.
  28|.define BASE,		r9	// Base of current Lua stack frame.
  29|
  30|// The following temporaries are not saved across C calls, except for RA/RC.
  31|.define RA,		r10	// Callee-save.
  32|.define RC,		r11	// Callee-save.
  33|.define RB,		r12
  34|.define OP,		r12	// Overlaps RB, must not be lr.
  35|.define INS,		lr
  36|
  37|// Calling conventions. Also used as temporaries.
  38|.define CARG1,		r0
  39|.define CARG2,		r1
  40|.define CARG3,		r2
  41|.define CARG4,		r3
  42|.define CARG12,	r0	// For 1st soft-fp double.
  43|.define CARG34,	r2	// For 2nd soft-fp double.
  44|
  45|.define CRET1,		r0
  46|.define CRET2,		r1
  47|
  48|// Stack layout while in interpreter. Must match with lj_frame.h.
  49|.define SAVE_R4,	[sp, #28]
  50|.define CFRAME_SPACE,	#28
  51|.define SAVE_ERRF,	[sp, #24]
  52|.define SAVE_NRES,	[sp, #20]
  53|.define SAVE_CFRAME,	[sp, #16]
  54|.define SAVE_L,	[sp, #12]
  55|.define SAVE_PC,	[sp, #8]
  56|.define SAVE_MULTRES,	[sp, #4]
  57|.define ARG5,		[sp]
  58|
  59|.define TMPDhi,	[sp, #4]
  60|.define TMPDlo,	[sp]
  61|.define TMPD,		[sp]
  62|.define TMPDp,		sp
  63|
  64|.if FPU
  65|.macro saveregs
  66|  push {r5, r6, r7, r8, r9, r10, r11, lr}
  67|  vpush {d8-d15}
  68|  sub sp, sp, CFRAME_SPACE+4
  69|  str r4, SAVE_R4
  70|.endmacro
  71|.macro restoreregs_ret
  72|  ldr r4, SAVE_R4
  73|  add sp, sp, CFRAME_SPACE+4
  74|  vpop {d8-d15}
  75|  pop {r5, r6, r7, r8, r9, r10, r11, pc}
  76|.endmacro
  77|.else
  78|.macro saveregs
  79|  push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
  80|  sub sp, sp, CFRAME_SPACE
  81|.endmacro
  82|.macro restoreregs_ret
  83|  add sp, sp, CFRAME_SPACE
  84|  pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
  85|.endmacro
  86|.endif
  87|
  88|// Type definitions. Some of these are only used for documentation.
  89|.type L,		lua_State,	LREG
  90|.type GL,		global_State
  91|.type TVALUE,		TValue
  92|.type GCOBJ,		GCobj
  93|.type STR,		GCstr
  94|.type TAB,		GCtab
  95|.type LFUNC,		GCfuncL
  96|.type CFUNC,		GCfuncC
  97|.type PROTO,		GCproto
  98|.type UPVAL,		GCupval
  99|.type NODE,		Node
 100|.type NARGS8,		int
 101|.type TRACE,		GCtrace
 102|.type SBUF,		SBuf
 103|
 104|//-----------------------------------------------------------------------
 105|
 106|// Trap for not-yet-implemented parts.
 107|.macro NYI; ud; .endmacro
 108|
 109|//-----------------------------------------------------------------------
 110|
 111|// Access to frame relative to BASE.
 112|.define FRAME_FUNC,	#-8
 113|.define FRAME_PC,	#-4
 114|
 115|.macro decode_RA8, dst, ins; and dst, MASKR8, ins, lsr #5; .endmacro
 116|.macro decode_RB8, dst, ins; and dst, MASKR8, ins, lsr #21; .endmacro
 117|.macro decode_RC8, dst, ins; and dst, MASKR8, ins, lsr #13; .endmacro
 118|.macro decode_RD, dst, ins; lsr dst, ins, #16; .endmacro
 119|.macro decode_OP, dst, ins; and dst, ins, #255; .endmacro
 120|
 121|// Instruction fetch.
 122|.macro ins_NEXT1
 123|  ldrb OP, [PC]
 124|.endmacro
 125|.macro ins_NEXT2
 126|   ldr INS, [PC], #4
 127|.endmacro
 128|// Instruction decode+dispatch.
 129|.macro ins_NEXT3
 130|  ldr OP, [DISPATCH, OP, lsl #2]
 131|   decode_RA8 RA, INS
 132|   decode_RD RC, INS
 133|  bx OP
 134|.endmacro
 135|.macro ins_NEXT
 136|  ins_NEXT1
 137|  ins_NEXT2
 138|  ins_NEXT3
 139|.endmacro
 140|
 141|// Instruction footer.
 142|.if 1
 143|  // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
 144|  .define ins_next, ins_NEXT
 145|  .define ins_next_, ins_NEXT
 146|  .define ins_next1, ins_NEXT1
 147|  .define ins_next2, ins_NEXT2
 148|  .define ins_next3, ins_NEXT3
 149|.else
 150|  // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
 151|  // Affects only certain kinds of benchmarks (and only with -j off).
 152|  .macro ins_next
 153|    b ->ins_next
 154|  .endmacro
 155|  .macro ins_next1
 156|  .endmacro
 157|  .macro ins_next2
 158|  .endmacro
 159|  .macro ins_next3
 160|    b ->ins_next
 161|  .endmacro
 162|  .macro ins_next_
 163|  ->ins_next:
 164|    ins_NEXT
 165|  .endmacro
 166|.endif
 167|
 168|// Avoid register name substitution for field name.
 169#define field_pc	pc
 170|
 171|// Call decode and dispatch.
 172|.macro ins_callt
 173|  // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
 174|  ldr PC, LFUNC:CARG3->field_pc
 175|  ldrb OP, [PC]  // STALL: load PC. early PC.
 176|   ldr INS, [PC], #4
 177|  ldr OP, [DISPATCH, OP, lsl #2]  // STALL: load OP. early OP.
 178|   decode_RA8 RA, INS
 179|   add RA, RA, BASE
 180|  bx OP
 181|.endmacro
 182|
 183|.macro ins_call
 184|  // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
 185|  str PC, [BASE, FRAME_PC]
 186|  ins_callt  // STALL: locked PC.
 187|.endmacro
 188|
 189|//-----------------------------------------------------------------------
 190|
 191|// Macros to test operand types.
 192|.macro checktp, reg, tp; cmn reg, #-tp; .endmacro
 193|.macro checktpeq, reg, tp; cmneq reg, #-tp; .endmacro
 194|.macro checktpne, reg, tp; cmnne reg, #-tp; .endmacro
 195|.macro checkstr, reg, target; checktp reg, LJ_TSTR; bne target; .endmacro
 196|.macro checktab, reg, target; checktp reg, LJ_TTAB; bne target; .endmacro
 197|.macro checkfunc, reg, target; checktp reg, LJ_TFUNC; bne target; .endmacro
 198|
 199|// Assumes DISPATCH is relative to GL.
 200#define DISPATCH_GL(field)	(GG_DISP2G + (int)offsetof(global_State, field))
 201#define DISPATCH_J(field)	(GG_DISP2J + (int)offsetof(jit_State, field))
 202|
 203#define PC2PROTO(field)  ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
 204|
 205|.macro hotcheck, delta
 206|  lsr CARG1, PC, #1
 207|  and CARG1, CARG1, #126
 208|  sub CARG1, CARG1, #-GG_DISP2HOT
 209|  ldrh CARG2, [DISPATCH, CARG1]
 210|  subs CARG2, CARG2, #delta
 211|  strh CARG2, [DISPATCH, CARG1]
 212|.endmacro
 213|
 214|.macro hotloop
 215|  hotcheck HOTCOUNT_LOOP
 216|  blo ->vm_hotloop
 217|.endmacro
 218|
 219|.macro hotcall
 220|  hotcheck HOTCOUNT_CALL
 221|  blo ->vm_hotcall
 222|.endmacro
 223|
 224|// Set current VM state.
 225|.macro mv_vmstate, reg, st; mvn reg, #LJ_VMST_..st; .endmacro
 226|.macro st_vmstate, reg; str reg, [DISPATCH, #DISPATCH_GL(vmstate)]; .endmacro
 227|
 228|// Move table write barrier back. Overwrites mark and tmp.
 229|.macro barrierback, tab, mark, tmp
 230|  ldr tmp, [DISPATCH, #DISPATCH_GL(gc.grayagain)]
 231|   bic mark, mark, #LJ_GC_BLACK		// black2gray(tab)
 232|  str tab, [DISPATCH, #DISPATCH_GL(gc.grayagain)]
 233|   strb mark, tab->marked
 234|  str tmp, tab->gclist
 235|.endmacro
 236|
 237|.macro .IOS, a, b
 238|.if IOS
 239|  a, b
 240|.endif
 241|.endmacro
 242|
 243|//-----------------------------------------------------------------------
 244
 245#if !LJ_DUALNUM
 246#error "Only dual-number mode supported for ARM target"
 247#endif
 248
 249/* Generate subroutines used by opcodes and other parts of the VM. */
 250/* The .code_sub section should be last to help static branch prediction. */
 251static void build_subroutines(BuildCtx *ctx)
 252{
 253  |.code_sub
 254  |
 255  |//-----------------------------------------------------------------------
 256  |//-- Return handling ----------------------------------------------------
 257  |//-----------------------------------------------------------------------
 258  |
 259  |->vm_returnp:
 260  |  // See vm_return. Also: RB = previous base.
 261  |  tst PC, #FRAME_P
 262  |  beq ->cont_dispatch
 263  |
 264  |  // Return from pcall or xpcall fast func.
 265  |  ldr PC, [RB, FRAME_PC]		// Fetch PC of previous frame.
 266  |   mvn CARG2, #~LJ_TTRUE
 267  |  mov BASE, RB
 268  |  // Prepending may overwrite the pcall frame, so do it at the end.
 269  |   str CARG2, [RA, FRAME_PC]		// Prepend true to results.
 270  |  sub RA, RA, #8
 271  |
 272  |->vm_returnc:
 273  |  adds RC, RC, #8			// RC = (nresults+1)*8.
 274  |  mov CRET1, #LUA_YIELD
 275  |  beq ->vm_unwind_c_eh
 276  |  str RC, SAVE_MULTRES
 277  |  ands CARG1, PC, #FRAME_TYPE
 278  |  beq ->BC_RET_Z			// Handle regular return to Lua.
 279  |
 280  |->vm_return:
 281  |  // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
 282  |  // CARG1 = PC & FRAME_TYPE
 283  |  bic RB, PC, #FRAME_TYPEP
 284  |   cmp CARG1, #FRAME_C
 285  |  sub RB, BASE, RB			// RB = previous base.
 286  |   bne ->vm_returnp
 287  |
 288  |  str RB, L->base
 289  |   ldr KBASE, SAVE_NRES
 290  |    mv_vmstate CARG4, C
 291  |   sub BASE, BASE, #8
 292  |  subs CARG3, RC, #8
 293  |   lsl KBASE, KBASE, #3		// KBASE = (nresults_wanted+1)*8
 294  |    st_vmstate CARG4
 295  |  beq >2
 296  |1:
 297  |  subs CARG3, CARG3, #8
 298  |   ldrd CARG12, [RA], #8
 299  |   strd CARG12, [BASE], #8
 300  |  bne <1
 301  |2:
 302  |  cmp KBASE, RC			// More/less results wanted?
 303  |  bne >6
 304  |3:
 305  |  str BASE, L->top			// Store new top.
 306  |
 307  |->vm_leave_cp:
 308  |  ldr RC, SAVE_CFRAME		// Restore previous C frame.
 309  |   mov CRET1, #0			// Ok return status for vm_pcall.
 310  |  str RC, L->cframe
 311  |
 312  |->vm_leave_unw:
 313  |  restoreregs_ret
 314  |
 315  |6:
 316  |  blt >7				// Less results wanted?
 317  |  // More results wanted. Check stack size and fill up results with nil.
 318  |  ldr CARG3, L->maxstack
 319  |   mvn CARG2, #~LJ_TNIL
 320  |  cmp BASE, CARG3
 321  |  bhs >8
 322  |   str CARG2, [BASE, #4]
 323  |  add RC, RC, #8
 324  |  add BASE, BASE, #8
 325  |  b <2
 326  |
 327  |7:  // Less results wanted.
 328  |  sub CARG1, RC, KBASE
 329  |  cmp KBASE, #0			// LUA_MULTRET+1 case?
 330  |  subne BASE, BASE, CARG1		// Either keep top or shrink it.
 331  |  b <3
 332  |
 333  |8:  // Corner case: need to grow stack for filling up results.
 334  |  // This can happen if:
 335  |  // - A C function grows the stack (a lot).
 336  |  // - The GC shrinks the stack in between.
 337  |  // - A return back from a lua_call() with (high) nresults adjustment.
 338  |  str BASE, L->top			// Save current top held in BASE (yes).
 339  |  lsr CARG2, KBASE, #3
 340  |  mov CARG1, L
 341  |  bl extern lj_state_growstack	// (lua_State *L, int n)
 342  |  ldr BASE, L->top			// Need the (realloced) L->top in BASE.
 343  |  b <2
 344  |
 345  |->vm_unwind_c:			// Unwind C stack, return from vm_pcall.
 346  |  // (void *cframe, int errcode)
 347  |  mov sp, CARG1
 348  |  mov CRET1, CARG2
 349  |->vm_unwind_c_eh:			// Landing pad for external unwinder.
 350  |  ldr L, SAVE_L
 351  |   mv_vmstate CARG4, C
 352  |  ldr GL:CARG3, L->glref
 353  |   str CARG4, GL:CARG3->vmstate
 354  |  b ->vm_leave_unw
 355  |
 356  |->vm_unwind_ff:			// Unwind C stack, return from ff pcall.
 357  |  // (void *cframe)
 358  |  bic CARG1, CARG1, #~CFRAME_RAWMASK	// Use two steps: bic sp is deprecated.
 359  |  mov sp, CARG1
 360  |->vm_unwind_ff_eh:			// Landing pad for external unwinder.
 361  |  ldr L, SAVE_L
 362  |   mov MASKR8, #255
 363  |    mov RC, #16			// 2 results: false + error message.
 364  |   lsl MASKR8, MASKR8, #3		// MASKR8 = 255*8.
 365  |  ldr BASE, L->base
 366  |   ldr DISPATCH, L->glref		// Setup pointer to dispatch table.
 367  |    mvn CARG1, #~LJ_TFALSE
 368  |  sub RA, BASE, #8			// Results start at BASE-8.
 369  |  ldr PC, [BASE, FRAME_PC]		// Fetch PC of previous frame.
 370  |   add DISPATCH, DISPATCH, #GG_G2DISP
 371  |   mv_vmstate CARG2, INTERP
 372  |    str CARG1, [BASE, #-4]		// Prepend false to error message.
 373  |   st_vmstate CARG2
 374  |  b ->vm_returnc
 375  |
 376  |->vm_unwind_ext:			// Complete external unwind.
 377#if !LJ_NO_UNWIND
 378  |  push {r0, r1, r2, lr}
 379  |  bl extern _Unwind_Complete
 380  |  ldr r0, [sp]
 381  |  bl extern _Unwind_DeleteException
 382  |  pop {r0, r1, r2, lr}
 383  |  mov r0, r1
 384  |  bx r2
 385#endif
 386  |
 387  |//-----------------------------------------------------------------------
 388  |//-- Grow stack for calls -----------------------------------------------
 389  |//-----------------------------------------------------------------------
 390  |
 391  |->vm_growstack_c:			// Grow stack for C function.
 392  |  // CARG1 = L
 393  |  mov CARG2, #LUA_MINSTACK
 394  |  b >2
 395  |
 396  |->vm_growstack_l:			// Grow stack for Lua function.
 397  |  // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
 398  |  add RC, BASE, RC
 399  |   sub RA, RA, BASE
 400  |    mov CARG1, L
 401  |  str BASE, L->base
 402  |   add PC, PC, #4			// Must point after first instruction.
 403  |  str RC, L->top
 404  |   lsr CARG2, RA, #3
 405  |2:
 406  |  // L->base = new base, L->top = top
 407  |  str PC, SAVE_PC
 408  |  bl extern lj_state_growstack	// (lua_State *L, int n)
 409  |  ldr BASE, L->base
 410  |   ldr RC, L->top
 411  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
 412  |   sub NARGS8:RC, RC, BASE
 413  |  // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
 414  |  ins_callt				// Just retry the call.
 415  |
 416  |//-----------------------------------------------------------------------
 417  |//-- Entry points into the assembler VM ---------------------------------
 418  |//-----------------------------------------------------------------------
 419  |
 420  |->vm_resume:				// Setup C frame and resume thread.
 421  |  // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
 422  |  saveregs
 423  |  mov L, CARG1
 424  |    ldr DISPATCH, L:CARG1->glref	// Setup pointer to dispatch table.
 425  |  mov BASE, CARG2
 426  |    add DISPATCH, DISPATCH, #GG_G2DISP
 427  |   str L, SAVE_L
 428  |  mov PC, #FRAME_CP
 429  |   str CARG3, SAVE_NRES
 430  |    add CARG2, sp, #CFRAME_RESUME
 431  |  ldrb CARG1, L->status
 432  |   str CARG3, SAVE_ERRF
 433  |   str L, SAVE_PC			// Any value outside of bytecode is ok.
 434  |   str CARG3, SAVE_CFRAME
 435  |  cmp CARG1, #0
 436  |    str CARG2, L->cframe
 437  |  beq >3
 438  |
 439  |  // Resume after yield (like a return).
 440  |  str L, [DISPATCH, #DISPATCH_GL(cur_L)]
 441  |  mov RA, BASE
 442  |   ldr BASE, L->base
 443  |   ldr CARG1, L->top
 444  |    mov MASKR8, #255
 445  |     strb CARG3, L->status
 446  |   sub RC, CARG1, BASE
 447  |  ldr PC, [BASE, FRAME_PC]
 448  |    lsl MASKR8, MASKR8, #3		// MASKR8 = 255*8.
 449  |     mv_vmstate CARG2, INTERP
 450  |   add RC, RC, #8
 451  |  ands CARG1, PC, #FRAME_TYPE
 452  |     st_vmstate CARG2
 453  |   str RC, SAVE_MULTRES
 454  |  beq ->BC_RET_Z
 455  |  b ->vm_return
 456  |
 457  |->vm_pcall:				// Setup protected C frame and enter VM.
 458  |  // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
 459  |  saveregs
 460  |  mov PC, #FRAME_CP
 461  |  str CARG4, SAVE_ERRF
 462  |  b >1
 463  |
 464  |->vm_call:				// Setup C frame and enter VM.
 465  |  // (lua_State *L, TValue *base, int nres1)
 466  |  saveregs
 467  |  mov PC, #FRAME_C
 468  |
 469  |1:  // Entry point for vm_pcall above (PC = ftype).
 470  |  ldr RC, L:CARG1->cframe
 471  |   str CARG3, SAVE_NRES
 472  |    mov L, CARG1
 473  |   str CARG1, SAVE_L
 474  |    ldr DISPATCH, L->glref		// Setup pointer to dispatch table.
 475  |     mov BASE, CARG2
 476  |   str CARG1, SAVE_PC		// Any value outside of bytecode is ok.
 477  |  str RC, SAVE_CFRAME
 478  |    add DISPATCH, DISPATCH, #GG_G2DISP
 479  |  str sp, L->cframe			// Add our C frame to cframe chain.
 480  |
 481  |3:  // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
 482  |  str L, [DISPATCH, #DISPATCH_GL(cur_L)]
 483  |  ldr RB, L->base			// RB = old base (for vmeta_call).
 484  |   ldr CARG1, L->top
 485  |    mov MASKR8, #255
 486  |  add PC, PC, BASE
 487  |    lsl MASKR8, MASKR8, #3		// MASKR8 = 255*8.
 488  |  sub PC, PC, RB			// PC = frame delta + frame type
 489  |    mv_vmstate CARG2, INTERP
 490  |   sub NARGS8:RC, CARG1, BASE
 491  |    st_vmstate CARG2
 492  |
 493  |->vm_call_dispatch:
 494  |  // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
 495  |  ldrd CARG34, [BASE, FRAME_FUNC]
 496  |  checkfunc CARG4, ->vmeta_call
 497  |
 498  |->vm_call_dispatch_f:
 499  |  ins_call
 500  |  // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
 501  |
 502  |->vm_cpcall:				// Setup protected C frame, call C.
 503  |  // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
 504  |  saveregs
 505  |  mov L, CARG1
 506  |   ldr RA, L:CARG1->stack
 507  |  str CARG1, SAVE_L
 508  |    ldr DISPATCH, L->glref		// Setup pointer to dispatch table.
 509  |   ldr RB, L->top
 510  |  str CARG1, SAVE_PC			// Any value outside of bytecode is ok.
 511  |  ldr RC, L->cframe
 512  |    add DISPATCH, DISPATCH, #GG_G2DISP
 513  |   sub RA, RA, RB			// Compute -savestack(L, L->top).
 514  |  mov RB, #0
 515  |   str RA, SAVE_NRES			// Neg. delta means cframe w/o frame.
 516  |  str RB, SAVE_ERRF			// No error function.
 517  |  str RC, SAVE_CFRAME
 518  |  str sp, L->cframe			// Add our C frame to cframe chain.
 519  |    str L, [DISPATCH, #DISPATCH_GL(cur_L)]
 520  |  blx CARG4			// (lua_State *L, lua_CFunction func, void *ud)
 521  |  movs BASE, CRET1
 522  |   mov PC, #FRAME_CP
 523  |  bne <3				// Else continue with the call.
 524  |  b ->vm_leave_cp			// No base? Just remove C frame.
 525  |
 526  |//-----------------------------------------------------------------------
 527  |//-- Metamethod handling ------------------------------------------------
 528  |//-----------------------------------------------------------------------
 529  |
 530  |//-- Continuation dispatch ----------------------------------------------
 531  |
 532  |->cont_dispatch:
 533  |  // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
 534  |  ldr LFUNC:CARG3, [RB, FRAME_FUNC]
 535  |    ldr CARG1, [BASE, #-16]		// Get continuation.
 536  |   mov CARG4, BASE
 537  |   mov BASE, RB			// Restore caller BASE.
 538  |.if FFI
 539  |    cmp CARG1, #1
 540  |.endif
 541  |   ldr PC, [CARG4, #-12]		// Restore PC from [cont|PC].
 542  |    mvn INS, #~LJ_TNIL
 543  |    add CARG2, RA, RC
 544  |    str INS, [CARG2, #-4]		// Ensure one valid arg.
 545  |.if FFI
 546  |    bls >1
 547  |.endif
 548  |  ldr CARG3, LFUNC:CARG3->field_pc
 549  |  ldr KBASE, [CARG3, #PC2PROTO(k)]
 550  |  // BASE = base, RA = resultptr, CARG4 = meta base
 551  |    bx CARG1
 552  |
 553  |.if FFI
 554  |1:
 555  |  beq ->cont_ffi_callback		// cont = 1: return from FFI callback.
 556  |  // cont = 0: tailcall from C function.
 557  |  sub CARG4, CARG4, #16
 558  |  sub RC, CARG4, BASE
 559  |  b ->vm_call_tail
 560  |.endif
 561  |
 562  |->cont_cat:				// RA = resultptr, CARG4 = meta base
 563  |  ldr INS, [PC, #-4]
 564  |   sub CARG2, CARG4, #16
 565  |   ldrd CARG34, [RA]
 566  |     str BASE, L->base
 567  |  decode_RB8 RC, INS
 568  |   decode_RA8 RA, INS
 569  |  add CARG1, BASE, RC
 570  |  subs CARG1, CARG2, CARG1
 571  |   strdne CARG34, [CARG2]
 572  |   movne CARG3, CARG1
 573  |  bne ->BC_CAT_Z
 574  |   strd CARG34, [BASE, RA]
 575  |  b ->cont_nop
 576  |
 577  |//-- Table indexing metamethods -----------------------------------------
 578  |
 579  |->vmeta_tgets1:
 580  |  add CARG2, BASE, RB
 581  |  b >2
 582  |
 583  |->vmeta_tgets:
 584  |  sub CARG2, DISPATCH, #-DISPATCH_GL(tmptv)
 585  |   mvn CARG4, #~LJ_TTAB
 586  |  str TAB:RB, [CARG2]
 587  |   str CARG4, [CARG2, #4]
 588  |2:
 589  |   mvn CARG4, #~LJ_TSTR
 590  |  str STR:RC, TMPDlo
 591  |   str CARG4, TMPDhi
 592  |  mov CARG3, TMPDp
 593  |  b >1
 594  |
 595  |->vmeta_tgetb:			// RC = index
 596  |  decode_RB8 RB, INS
 597  |   str RC, TMPDlo
 598  |   mvn CARG4, #~LJ_TISNUM
 599  |  add CARG2, BASE, RB
 600  |   str CARG4, TMPDhi
 601  |  mov CARG3, TMPDp
 602  |  b >1
 603  |
 604  |->vmeta_tgetv:
 605  |  add CARG2, BASE, RB
 606  |   add CARG3, BASE, RC
 607  |1:
 608  |   str BASE, L->base
 609  |  mov CARG1, L
 610  |   str PC, SAVE_PC
 611  |  bl extern lj_meta_tget		// (lua_State *L, TValue *o, TValue *k)
 612  |  // Returns TValue * (finished) or NULL (metamethod).
 613  |  .IOS ldr BASE, L->base
 614  |  cmp CRET1, #0
 615  |  beq >3
 616  |  ldrd CARG34, [CRET1]
 617  |   ins_next1
 618  |   ins_next2
 619  |  strd CARG34, [BASE, RA]
 620  |   ins_next3
 621  |
 622  |3:  // Call __index metamethod.
 623  |  // BASE = base, L->top = new base, stack = cont/func/t/k
 624  |   rsb CARG1, BASE, #FRAME_CONT
 625  |  ldr BASE, L->top
 626  |    mov NARGS8:RC, #16		// 2 args for func(t, k).
 627  |    str PC, [BASE, #-12]		// [cont|PC]
 628  |   add PC, CARG1, BASE
 629  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
 630  |  b ->vm_call_dispatch_f
 631  |
 632  |->vmeta_tgetr:
 633  |  .IOS mov RC, BASE
 634  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
 635  |  // Returns cTValue * or NULL.
 636  |  .IOS mov BASE, RC
 637  |  cmp CRET1, #0
 638  |  ldrdne CARG12, [CRET1]
 639  |  mvneq CARG2, #~LJ_TNIL
 640  |  b ->BC_TGETR_Z
 641  |
 642  |//-----------------------------------------------------------------------
 643  |
 644  |->vmeta_tsets1:
 645  |  add CARG2, BASE, RB
 646  |  b >2
 647  |
 648  |->vmeta_tsets:
 649  |  sub CARG2, DISPATCH, #-DISPATCH_GL(tmptv)
 650  |   mvn CARG4, #~LJ_TTAB
 651  |  str TAB:RB, [CARG2]
 652  |   str CARG4, [CARG2, #4]
 653  |2:
 654  |   mvn CARG4, #~LJ_TSTR
 655  |  str STR:RC, TMPDlo
 656  |   str CARG4, TMPDhi
 657  |  mov CARG3, TMPDp
 658  |  b >1
 659  |
 660  |->vmeta_tsetb:			// RC = index
 661  |  decode_RB8 RB, INS
 662  |   str RC, TMPDlo
 663  |   mvn CARG4, #~LJ_TISNUM
 664  |  add CARG2, BASE, RB
 665  |   str CARG4, TMPDhi
 666  |  mov CARG3, TMPDp
 667  |  b >1
 668  |
 669  |->vmeta_tsetv:
 670  |  add CARG2, BASE, RB
 671  |   add CARG3, BASE, RC
 672  |1:
 673  |   str BASE, L->base
 674  |  mov CARG1, L
 675  |   str PC, SAVE_PC
 676  |  bl extern lj_meta_tset		// (lua_State *L, TValue *o, TValue *k)
 677  |  // Returns TValue * (finished) or NULL (metamethod).
 678  |  .IOS ldr BASE, L->base
 679  |  cmp CRET1, #0
 680  |   ldrd CARG34, [BASE, RA]
 681  |  beq >3
 682  |   ins_next1
 683  |  // NOBARRIER: lj_meta_tset ensures the table is not black.
 684  |  strd CARG34, [CRET1]
 685  |   ins_next2
 686  |   ins_next3
 687  |
 688  |3:  // Call __newindex metamethod.
 689  |  // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
 690  |   rsb CARG1, BASE, #FRAME_CONT
 691  |  ldr BASE, L->top
 692  |    mov NARGS8:RC, #24		// 3 args for func(t, k, v).
 693  |   strd CARG34, [BASE, #16]		// Copy value to third argument.
 694  |    str PC, [BASE, #-12]		// [cont|PC]
 695  |   add PC, CARG1, BASE
 696  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
 697  |  b ->vm_call_dispatch_f
 698  |
 699  |->vmeta_tsetr:
 700  |  str BASE, L->base
 701  |  .IOS mov RC, BASE
 702  |  mov CARG1, L
 703  |  str PC, SAVE_PC
 704  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
 705  |  // Returns TValue *.
 706  |  .IOS mov BASE, RC
 707  |  b ->BC_TSETR_Z
 708  |
 709  |//-- Comparison metamethods ---------------------------------------------
 710  |
 711  |->vmeta_comp:
 712  |  mov CARG1, L
 713  |   sub PC, PC, #4
 714  |  mov CARG2, RA
 715  |   str BASE, L->base
 716  |  mov CARG3, RC
 717  |   str PC, SAVE_PC
 718  |  decode_OP CARG4, INS
 719  |  bl extern lj_meta_comp  // (lua_State *L, TValue *o1, *o2, int op)
 720  |  // Returns 0/1 or TValue * (metamethod).
 721  |3:
 722  |  .IOS ldr BASE, L->base
 723  |  cmp CRET1, #1
 724  |  bhi ->vmeta_binop
 725  |4:
 726  |  ldrh RB, [PC, #2]
 727  |   add PC, PC, #4
 728  |  add RB, PC, RB, lsl #2
 729  |  subhs PC, RB, #0x20000
 730  |->cont_nop:
 731  |  ins_next
 732  |
 733  |->cont_ra:				// RA = resultptr
 734  |  ldr INS, [PC, #-4]
 735  |   ldrd CARG12, [RA]
 736  |  decode_RA8 CARG3, INS
 737  |   strd CARG12, [BASE, CARG3]
 738  |  b ->cont_nop
 739  |
 740  |->cont_condt:			// RA = resultptr
 741  |  ldr CARG2, [RA, #4]
 742  |   mvn CARG1, #~LJ_TTRUE
 743  |  cmp CARG1, CARG2			// Branch if result is true.
 744  |  b <4
 745  |
 746  |->cont_condf:			// RA = resultptr
 747  |  ldr CARG2, [RA, #4]
 748  |  checktp CARG2, LJ_TFALSE		// Branch if result is false.
 749  |  b <4
 750  |
 751  |->vmeta_equal:
 752  |  // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
 753  |  sub PC, PC, #4
 754  |   str BASE, L->base
 755  |   mov CARG1, L
 756  |  str PC, SAVE_PC
 757  |  bl extern lj_meta_equal  // (lua_State *L, GCobj *o1, *o2, int ne)
 758  |  // Returns 0/1 or TValue * (metamethod).
 759  |  b <3
 760  |
 761  |->vmeta_equal_cd:
 762  |.if FFI
 763  |  sub PC, PC, #4
 764  |   str BASE, L->base
 765  |   mov CARG1, L
 766  |   mov CARG2, INS
 767  |  str PC, SAVE_PC
 768  |  bl extern lj_meta_equal_cd		// (lua_State *L, BCIns op)
 769  |  // Returns 0/1 or TValue * (metamethod).
 770  |  b <3
 771  |.endif
 772  |
 773  |->vmeta_istype:
 774  |  sub PC, PC, #4
 775  |   str BASE, L->base
 776  |   mov CARG1, L
 777  |   lsr CARG2, RA, #3
 778  |   mov CARG3, RC
 779  |  str PC, SAVE_PC
 780  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
 781  |  .IOS ldr BASE, L->base
 782  |  b ->cont_nop
 783  |
 784  |//-- Arithmetic metamethods ---------------------------------------------
 785  |
 786  |->vmeta_arith_vn:
 787  |  decode_RB8 RB, INS
 788  |   decode_RC8 RC, INS
 789  |  add CARG3, BASE, RB
 790  |   add CARG4, KBASE, RC
 791  |  b >1
 792  |
 793  |->vmeta_arith_nv:
 794  |  decode_RB8 RB, INS
 795  |   decode_RC8 RC, INS
 796  |  add CARG4, BASE, RB
 797  |   add CARG3, KBASE, RC
 798  |  b >1
 799  |
 800  |->vmeta_unm:
 801  |  ldr INS, [PC, #-8]
 802  |   sub PC, PC, #4
 803  |  add CARG3, BASE, RC
 804  |  add CARG4, BASE, RC
 805  |  b >1
 806  |
 807  |->vmeta_arith_vv:
 808  |  decode_RB8 RB, INS
 809  |   decode_RC8 RC, INS
 810  |  add CARG3, BASE, RB
 811  |   add CARG4, BASE, RC
 812  |1:
 813  |  decode_OP OP, INS
 814  |   add CARG2, BASE, RA
 815  |    str BASE, L->base
 816  |   mov CARG1, L
 817  |    str PC, SAVE_PC
 818  |  str OP, ARG5
 819  |  bl extern lj_meta_arith  // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
 820  |  // Returns NULL (finished) or TValue * (metamethod).
 821  |  .IOS ldr BASE, L->base
 822  |  cmp CRET1, #0
 823  |  beq ->cont_nop
 824  |
 825  |  // Call metamethod for binary op.
 826  |->vmeta_binop:
 827  |  // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
 828  |  sub CARG2, CRET1, BASE
 829  |   str PC, [CRET1, #-12]		// [cont|PC]
 830  |  add PC, CARG2, #FRAME_CONT
 831  |   mov BASE, CRET1
 832  |    mov NARGS8:RC, #16		// 2 args for func(o1, o2).
 833  |  b ->vm_call_dispatch
 834  |
 835  |->vmeta_len:
 836  |  add CARG2, BASE, RC
 837  |   str BASE, L->base
 838  |  mov CARG1, L
 839  |   str PC, SAVE_PC
 840  |  bl extern lj_meta_len		// (lua_State *L, TValue *o)
 841  |  // Returns NULL (retry) or TValue * (metamethod base).
 842  |  .IOS ldr BASE, L->base
 843#if LJ_52
 844  |  cmp CRET1, #0
 845  |  bne ->vmeta_binop			// Binop call for compatibility.
 846  |  ldr TAB:CARG1, [BASE, RC]
 847  |  b ->BC_LEN_Z
 848#else
 849  |  b ->vmeta_binop			// Binop call for compatibility.
 850#endif
 851  |
 852  |//-- Call metamethod ----------------------------------------------------
 853  |
 854  |->vmeta_call:			// Resolve and call __call metamethod.
 855  |  // RB = old base, BASE = new base, RC = nargs*8
 856  |  mov CARG1, L
 857  |   str RB, L->base			// This is the callers base!
 858  |  sub CARG2, BASE, #8
 859  |   str PC, SAVE_PC
 860  |  add CARG3, BASE, NARGS8:RC
 861  |  .IOS mov RA, BASE
 862  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
 863  |  .IOS mov BASE, RA
 864  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
 865  |   add NARGS8:RC, NARGS8:RC, #8	// Got one more argument now.
 866  |  ins_call
 867  |
 868  |->vmeta_callt:			// Resolve __call for BC_CALLT.
 869  |  // BASE = old base, RA = new base, RC = nargs*8
 870  |  mov CARG1, L
 871  |   str BASE, L->base
 872  |  sub CARG2, RA, #8
 873  |   str PC, SAVE_PC
 874  |  add CARG3, RA, NARGS8:RC
 875  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
 876  |  .IOS ldr BASE, L->base
 877  |  ldr LFUNC:CARG3, [RA, FRAME_FUNC]  // Guaranteed to be a function here.
 878  |   ldr PC, [BASE, FRAME_PC]
 879  |    add NARGS8:RC, NARGS8:RC, #8	// Got one more argument now.
 880  |  b ->BC_CALLT2_Z
 881  |
 882  |//-- Argument coercion for 'for' statement ------------------------------
 883  |
 884  |->vmeta_for:
 885  |  mov CARG1, L
 886  |   str BASE, L->base
 887  |  mov CARG2, RA
 888  |   str PC, SAVE_PC
 889  |  bl extern lj_meta_for	// (lua_State *L, TValue *base)
 890  |  .IOS ldr BASE, L->base
 891  |.if JIT
 892  |   ldrb OP, [PC, #-4]
 893  |.endif
 894  |  ldr INS, [PC, #-4]
 895  |.if JIT
 896  |   cmp OP, #BC_JFORI
 897  |.endif
 898  |  decode_RA8 RA, INS
 899  |  decode_RD RC, INS
 900  |.if JIT
 901  |   beq =>BC_JFORI
 902  |.endif
 903  |  b =>BC_FORI
 904  |
 905  |//-----------------------------------------------------------------------
 906  |//-- Fast functions -----------------------------------------------------
 907  |//-----------------------------------------------------------------------
 908  |
 909  |.macro .ffunc, name
 910  |->ff_ .. name:
 911  |.endmacro
 912  |
 913  |.macro .ffunc_1, name
 914  |->ff_ .. name:
 915  |  ldrd CARG12, [BASE]
 916  |   cmp NARGS8:RC, #8
 917  |   blo ->fff_fallback
 918  |.endmacro
 919  |
 920  |.macro .ffunc_2, name
 921  |->ff_ .. name:
 922  |  ldrd CARG12, [BASE]
 923  |   ldrd CARG34, [BASE, #8]
 924  |    cmp NARGS8:RC, #16
 925  |    blo ->fff_fallback
 926  |.endmacro
 927  |
 928  |.macro .ffunc_n, name
 929  |  .ffunc_1 name
 930  |  checktp CARG2, LJ_TISNUM
 931  |  bhs ->fff_fallback
 932  |.endmacro
 933  |
 934  |.macro .ffunc_nn, name
 935  |  .ffunc_2 name
 936  |  checktp CARG2, LJ_TISNUM
 937  |  cmnlo CARG4, #-LJ_TISNUM
 938  |  bhs ->fff_fallback
 939  |.endmacro
 940  |
 941  |.macro .ffunc_d, name
 942  |  .ffunc name
 943  |  ldr CARG2, [BASE, #4]
 944  |   cmp NARGS8:RC, #8
 945  |  vldr d0, [BASE]
 946  |   blo ->fff_fallback
 947  |  checktp CARG2, LJ_TISNUM
 948  |  bhs ->fff_fallback
 949  |.endmacro
 950  |
 951  |.macro .ffunc_dd, name
 952  |  .ffunc name
 953  |  ldr CARG2, [BASE, #4]
 954  |  ldr CARG4, [BASE, #12]
 955  |   cmp NARGS8:RC, #16
 956  |  vldr d0, [BASE]
 957  |  vldr d1, [BASE, #8]
 958  |   blo ->fff_fallback
 959  |  checktp CARG2, LJ_TISNUM
 960  |  cmnlo CARG4, #-LJ_TISNUM
 961  |  bhs ->fff_fallback
 962  |.endmacro
 963  |
 964  |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
 965  |.macro ffgccheck
 966  |  ldr CARG1, [DISPATCH, #DISPATCH_GL(gc.total)]
 967  |  ldr CARG2, [DISPATCH, #DISPATCH_GL(gc.threshold)]
 968  |  cmp CARG1, CARG2
 969  |  blge ->fff_gcstep
 970  |.endmacro
 971  |
 972  |//-- Base library: checks -----------------------------------------------
 973  |
 974  |.ffunc_1 assert
 975  |  checktp CARG2, LJ_TTRUE
 976  |  bhi ->fff_fallback
 977  |   ldr PC, [BASE, FRAME_PC]
 978  |  strd CARG12, [BASE, #-8]
 979  |  mov RB, BASE
 980  |  subs RA, NARGS8:RC, #8
 981  |   add RC, NARGS8:RC, #8		// Compute (nresults+1)*8.
 982  |  beq ->fff_res			// Done if exactly 1 argument.
 983  |1:
 984  |   ldrd CARG12, [RB, #8]
 985  |  subs RA, RA, #8
 986  |   strd CARG12, [RB], #8
 987  |  bne <1
 988  |  b ->fff_res
 989  |
 990  |.ffunc type
 991  |  ldr CARG2, [BASE, #4]
 992  |   cmp NARGS8:RC, #8
 993  |   blo ->fff_fallback
 994  |  checktp CARG2, LJ_TISNUM
 995  |  mvnlo CARG2, #~LJ_TISNUM
 996  |  rsb CARG4, CARG2, #(int)(offsetof(GCfuncC, upvalue)>>3)-1
 997  |  lsl CARG4, CARG4, #3
 998  |  ldrd CARG12, [CFUNC:CARG3, CARG4]
 999  |  b ->fff_restv
1000  |
1001  |//-- Base library: getters and setters ---------------------------------
1002  |
1003  |.ffunc_1 getmetatable
1004  |  checktp CARG2, LJ_TTAB
1005  |  cmnne CARG2, #-LJ_TUDATA
1006  |  bne >6
1007  |1:  // Field metatable must be at same offset for GCtab and GCudata!
1008  |  ldr TAB:RB, TAB:CARG1->metatable
1009  |2:
1010  |   mvn CARG2, #~LJ_TNIL
1011  |   ldr STR:RC, [DISPATCH, #DISPATCH_GL(gcroot[GCROOT_MMNAME+MM_metatable])]
1012  |  cmp TAB:RB, #0
1013  |  beq ->fff_restv
1014  |  ldr CARG3, TAB:RB->hmask
1015  |   ldr CARG4, STR:RC->sid
1016  |    ldr NODE:INS, TAB:RB->node
1017  |  and CARG3, CARG3, CARG4		// idx = str->sid & tab->hmask
1018  |  add CARG3, CARG3, CARG3, lsl #1
1019  |    add NODE:INS, NODE:INS, CARG3, lsl #3	// node = tab->node + idx*3*8
1020  |3:  // Rearranged logic, because we expect _not_ to find the key.
1021  |  ldrd CARG34, NODE:INS->key  // STALL: early NODE:INS.
1022  |   ldrd CARG12, NODE:INS->val
1023  |    ldr NODE:INS, NODE:INS->next
1024  |  checktp CARG4, LJ_TSTR
1025  |  cmpeq CARG3, STR:RC
1026  |  beq >5
1027  |  cmp NODE:INS, #0
1028  |  bne <3
1029  |4:
1030  |  mov CARG1, RB			// Use metatable as default result.
1031  |  mvn CARG2, #~LJ_TTAB
1032  |  b ->fff_restv
1033  |5:
1034  |  checktp CARG2, LJ_TNIL
1035  |  bne ->fff_restv
1036  |  b <4
1037  |
1038  |6:
1039  |  checktp CARG2, LJ_TISNUM
1040  |  mvnhs CARG2, CARG2
1041  |  movlo CARG2, #~LJ_TISNUM
1042  |  add CARG4, DISPATCH, CARG2, lsl #2
1043  |  ldr TAB:RB, [CARG4, #DISPATCH_GL(gcroot[GCROOT_BASEMT])]
1044  |  b <2
1045  |
1046  |.ffunc_2 setmetatable
1047  |  // Fast path: no mt for table yet and not clearing the mt.
1048  |  checktp CARG2, LJ_TTAB
1049  |   ldreq TAB:RB, TAB:CARG1->metatable
1050  |  checktpeq CARG4, LJ_TTAB
1051  |    ldrbeq CARG4, TAB:CARG1->marked
1052  |   cmpeq TAB:RB, #0
1053  |  bne ->fff_fallback
1054  |    tst CARG4, #LJ_GC_BLACK		// isblack(table)
1055  |     str TAB:CARG3, TAB:CARG1->metatable
1056  |    beq ->fff_restv
1057  |  barrierback TAB:CARG1, CARG4, CARG3
1058  |  b ->fff_restv
1059  |
1060  |.ffunc rawget
1061  |  ldrd CARG34, [BASE]
1062  |   cmp NARGS8:RC, #16
1063  |   blo ->fff_fallback
1064  |   mov CARG2, CARG3
1065  |  checktab CARG4, ->fff_fallback
1066  |   mov CARG1, L
1067  |   add CARG3, BASE, #8
1068  |  .IOS mov RA, BASE
1069  |  bl extern lj_tab_get  // (lua_State *L, GCtab *t, cTValue *key)
1070  |  // Returns cTValue *.
1071  |  .IOS mov BASE, RA
1072  |  ldrd CARG12, [CRET1]
1073  |  b ->fff_restv
1074  |
1075  |//-- Base library: conversions ------------------------------------------
1076  |
1077  |.ffunc tonumber
1078  |  // Only handles the number case inline (without a base argument).
1079  |  ldrd CARG12, [BASE]
1080  |   cmp NARGS8:RC, #8
1081  |   bne ->fff_fallback
1082  |  checktp CARG2, LJ_TISNUM
1083  |  bls ->fff_restv
1084  |  b ->fff_fallback
1085  |
1086  |.ffunc_1 tostring
1087  |  // Only handles the string or number case inline.
1088  |  checktp CARG2, LJ_TSTR
1089  |  // A __tostring method in the string base metatable is ignored.
1090  |  beq ->fff_restv
1091  |  // Handle numbers inline, unless a number base metatable is present.
1092  |  ldr CARG4, [DISPATCH, #DISPATCH_GL(gcroot[GCROOT_BASEMT_NUM])]
1093  |   str BASE, L->base
1094  |  checktp CARG2, LJ_TISNUM
1095  |  cmpls CARG4, #0
1096  |   str PC, SAVE_PC			// Redundant (but a defined value).
1097  |  bhi ->fff_fallback
1098  |  ffgccheck
1099  |  mov CARG1, L
1100  |  mov CARG2, BASE
1101  |  bl extern lj_strfmt_number		// (lua_State *L, cTValue *o)
1102  |  // Returns GCstr *.
1103  |  ldr BASE, L->base
1104  |  mvn CARG2, #~LJ_TSTR
1105  |  b ->fff_restv
1106  |
1107  |//-- Base library: iterators -------------------------------------------
1108  |
1109  |.ffunc_1 next
1110  |   mvn CARG4, #~LJ_TNIL
1111  |  checktab CARG2, ->fff_fallback
1112  |   strd CARG34, [BASE, NARGS8:RC]	// Set missing 2nd arg to nil.
1113  |   ldr PC, [BASE, FRAME_PC]
1114  |  add CARG2, BASE, #8
1115  |  sub CARG3, BASE, #8
1116  |  bl extern lj_tab_next		// (GCtab *t, cTValue *key, TValue *o)
1117  |  // Returns 1=found, 0=end, -1=error.
1118  |  .IOS ldr BASE, L->base
1119  |  cmp CRET1, #0
1120  |   mov RC, #(2+1)*8
1121  |  bgt ->fff_res			// Found key/value.
1122  |  bmi ->fff_fallback			// Invalid key.
1123  |  // End of traversal: return nil.
1124  |  mvn CRET2, #~LJ_TNIL
1125  |  b ->fff_restv
1126  |
1127  |.ffunc_1 pairs
1128  |  checktab CARG2, ->fff_fallback
1129#if LJ_52
1130  |  ldr TAB:RB, TAB:CARG1->metatable
1131#endif
1132  |   ldrd CFUNC:CARG34, CFUNC:CARG3->upvalue[0]
1133  |    ldr PC, [BASE, FRAME_PC]
1134#if LJ_52
1135  |  cmp TAB:RB, #0
1136  |  bne ->fff_fallback
1137#endif
1138  |  mvn CARG2, #~LJ_TNIL
1139  |    mov RC, #(3+1)*8
1140  |   strd CFUNC:CARG34, [BASE, #-8]
1141  |  str CARG2, [BASE, #12]
1142  |  b ->fff_res
1143  |
1144  |.ffunc_2 ipairs_aux
1145  |  checktp CARG2, LJ_TTAB
1146  |  checktpeq CARG4, LJ_TISNUM
1147  |  bne ->fff_fallback
1148  |  ldr RB, TAB:CARG1->asize
1149  |   ldr RC, TAB:CARG1->array
1150  |  add CARG3, CARG3, #1
1151  |    ldr PC, [BASE, FRAME_PC]
1152  |  cmp CARG3, RB
1153  |   add RC, RC, CARG3, lsl #3
1154  |  strd CARG34, [BASE, #-8]
1155  |   ldrdlo CARG12, [RC]
1156  |   mov RC, #(0+1)*8
1157  |  bhs >2				// Not in array part?
1158  |1:
1159  |   checktp CARG2, LJ_TNIL
1160  |   movne RC, #(2+1)*8
1161  |   strdne CARG12, [BASE]
1162  |  b ->fff_res
1163  |2:  // Check for empty hash part first. Otherwise call C function.
1164  |  ldr RB, TAB:CARG1->hmask
1165  |   mov CARG2, CARG3
1166  |  cmp RB, #0
1167  |  beq ->fff_res
1168  |  .IOS mov RA, BASE
1169  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
1170  |  // Returns cTValue * or NULL.
1171  |  .IOS mov BASE, RA
1172  |  cmp CRET1, #0
1173  |  beq ->fff_res
1174  |  ldrd CARG12, [CRET1]
1175  |  b <1
1176  |
1177  |.ffunc_1 ipairs
1178  |  checktab CARG2, ->fff_fallback
1179#if LJ_52
1180  |  ldr TAB:RB, TAB:CARG1->metatable
1181#endif
1182  |   ldrd CFUNC:CARG34, CFUNC:CARG3->upvalue[0]
1183  |    ldr PC, [BASE, FRAME_PC]
1184#if LJ_52
1185  |  cmp TAB:RB, #0
1186  |  bne ->fff_fallback
1187#endif
1188  |  mov CARG1, #0
1189  |  mvn CARG2, #~LJ_TISNUM
1190  |    mov RC, #(3+1)*8
1191  |   strd CFUNC:CARG34, [BASE, #-8]
1192  |  strd CARG12, [BASE, #8]
1193  |  b ->fff_res
1194  |
1195  |//-- Base library: catch errors ----------------------------------------
1196  |
1197  |.ffunc pcall
1198  |   ldr RB, L->maxstack
1199  |   add INS, BASE, NARGS8:RC
1200  |  ldrb RA, [DISPATCH, #DISPATCH_GL(hookmask)]
1201  |   cmp NARGS8:RC, #8
1202  |   cmphs RB, INS
1203  |   blo ->fff_fallback
1204  |  tst RA, #HOOK_ACTIVE		// Remember active hook before pcall.
1205  |   mov RB, BASE
1206  |   add BASE, BASE, #8
1207  |  moveq PC, #8+FRAME_PCALL
1208  |  movne PC, #8+FRAME_PCALLH
1209  |   sub NARGS8:RC, NARGS8:RC, #8
1210  |  b ->vm_call_dispatch
1211  |
1212  |.ffunc_2 xpcall
1213  |   ldr RB, L->maxstack
1214  |   add INS, BASE, NARGS8:RC
1215  |  ldrb RA, [DISPATCH, #DISPATCH_GL(hookmask)]
1216  |   cmp RB, INS
1217  |   blo ->fff_fallback
1218  |  checkfunc CARG4, ->fff_fallback	// Traceback must be a function.
1219  |   mov RB, BASE
1220  |  strd CARG12, [BASE, #8]		// Swap function and traceback.
1221  |   strd CARG34, [BASE]
1222  |  tst RA, #HOOK_ACTIVE		// Remember active hook before pcall.
1223  |   add BASE, BASE, #16
1224  |  moveq PC, #16+FRAME_PCALL
1225  |  movne PC, #16+FRAME_PCALLH
1226  |   sub NARGS8:RC, NARGS8:RC, #16
1227  |  b ->vm_call_dispatch
1228  |
1229  |//-- Coroutine library --------------------------------------------------
1230  |
1231  |.macro coroutine_resume_wrap, resume
1232  |.if resume
1233  |.ffunc_1 coroutine_resume
1234  |  checktp CARG2, LJ_TTHREAD
1235  |  bne ->fff_fallback
1236  |.else
1237  |.ffunc coroutine_wrap_aux
1238  |  ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1239  |.endif
1240  |   ldr PC, [BASE, FRAME_PC]
1241  |     str BASE, L->base
1242  |  ldr CARG2, L:CARG1->top
1243  |   ldrb RA, L:CARG1->status
1244  |    ldr RB, L:CARG1->base
1245  |  add CARG3, CARG2, NARGS8:RC
1246  |  add CARG4, CARG2, RA
1247  |   str PC, SAVE_PC
1248  |  cmp CARG4, RB
1249  |  beq ->fff_fallback
1250  |   ldr CARG4, L:CARG1->maxstack
1251  |    ldr RB, L:CARG1->cframe
1252  |   cmp RA, #LUA_YIELD
1253  |   cmpls CARG3, CARG4
1254  |    cmpls RB, #0
1255  |    bhi ->fff_fallback
1256  |1:
1257  |.if resume
1258  |  sub CARG3, CARG3, #8		// Keep resumed thread in stack for GC.
1259  |  add BASE, BASE, #8
1260  |  sub NARGS8:RC, NARGS8:RC, #8
1261  |.endif
1262  |  str CARG3, L:CARG1->top
1263  |  str BASE, L->top
1264  |2:  // Move args to coroutine.
1265  |   ldrd CARG34, [BASE, RB]
1266  |  cmp RB, NARGS8:RC
1267  |   strdne CARG34, [CARG2, RB]
1268  |  add RB, RB, #8
1269  |  bne <2
1270  |
1271  |  mov CARG3, #0
1272  |   mov L:RA, L:CARG1
1273  |  mov CARG4, #0
1274  |  bl ->vm_resume			// (lua_State *L, TValue *base, 0, 0)
1275  |  // Returns thread status.
1276  |4:
1277  |  ldr CARG3, L:RA->base
1278  |    mv_vmstate CARG2, INTERP
1279  |  ldr CARG4, L:RA->top
1280  |   cmp CRET1, #LUA_YIELD
1281  |  ldr BASE, L->base
1282  |    str L, [DISPATCH, #DISPATCH_GL(cur_L)]
1283  |    st_vmstate CARG2
1284  |   bhi >8
1285  |  subs RC, CARG4, CARG3
1286  |   ldr CARG1, L->maxstack
1287  |   add CARG2, BASE, RC
1288  |  beq >6				// No results?
1289  |  cmp CARG2, CARG1
1290  |   mov RB, #0
1291  |  bhi >9				// Need to grow stack?
1292  |
1293  |  sub CARG4, RC, #8
1294  |   str CARG3, L:RA->top		// Clear coroutine stack.
1295  |5:  // Move results from coroutine.
1296  |   ldrd CARG12, [CARG3, RB]
1297  |  cmp RB, CARG4
1298  |   strd CARG12, [BASE, RB]
1299  |  add RB, RB, #8
1300  |  bne <5
1301  |6:
1302  |.if resume
1303  |  mvn CARG3, #~LJ_TTRUE
1304  |   add RC, RC, #16
1305  |7:
1306  |  str CARG3, [BASE, #-4]		// Prepend true/false to results.
1307  |   sub RA, BASE, #8
1308  |.else
1309  |   mov RA, BASE
1310  |   add RC, RC, #8
1311  |.endif
1312  |  ands CARG1, PC, #FRAME_TYPE
1313  |   str PC, SAVE_PC
1314  |   str RC, SAVE_MULTRES
1315  |  beq ->BC_RET_Z
1316  |  b ->vm_return
1317  |
1318  |8:  // Coroutine returned with error (at co->top-1).
1319  |.if resume
1320  |  ldrd CARG12, [CARG4, #-8]!
1321  |   mvn CARG3, #~LJ_TFALSE
1322  |    mov RC, #(2+1)*8
1323  |  str CARG4, L:RA->top		// Remove error from coroutine stack.
1324  |  strd CARG12, [BASE]		// Copy error message.
1325  |  b <7
1326  |.else
1327  |  mov CARG1, L
1328  |  mov CARG2, L:RA
1329  |  bl extern lj_ffh_coroutine_wrap_err  // (lua_State *L, lua_State *co)
1330  |  // Never returns.
1331  |.endif
1332  |
1333  |9:  // Handle stack expansion on return from yield.
1334  |  mov CARG1, L
1335  |  lsr CARG2, RC, #3
1336  |  bl extern lj_state_growstack	// (lua_State *L, int n)
1337  |  mov CRET1, #0
1338  |  b <4
1339  |.endmacro
1340  |
1341  |  coroutine_resume_wrap 1		// coroutine.resume
1342  |  coroutine_resume_wrap 0		// coroutine.wrap
1343  |
1344  |.ffunc coroutine_yield
1345  |  ldr CARG1, L->cframe
1346  |   add CARG2, BASE, NARGS8:RC
1347  |   str BASE, L->base
1348  |  tst CARG1, #CFRAME_RESUME
1349  |   str CARG2, L->top
1350  |    mov CRET1, #LUA_YIELD
1351  |   mov CARG3, #0
1352  |  beq ->fff_fallback
1353  |   str CARG3, L->cframe
1354  |    strb CRET1, L->status
1355  |  b ->vm_leave_unw
1356  |
1357  |//-- Math library -------------------------------------------------------
1358  |
1359  |.macro math_round, func
1360  |  .ffunc_1 math_ .. func
1361  |  checktp CARG2, LJ_TISNUM
1362  |  beq ->fff_restv
1363  |  bhi ->fff_fallback
1364  |  // Round FP value and normalize result.
1365  |  lsl CARG3, CARG2, #1
1366  |  adds RB, CARG3, #0x00200000
1367  |  bpl >2				// |x| < 1?
1368  |  mvn CARG4, #0x3e0
1369  |    subs RB, CARG4, RB, asr #21
1370  |  lsl CARG4, CARG2, #11
1371  |   lsl CARG3, CARG1, #11
1372  |  orr CARG4, CARG4, #0x80000000
1373  |   rsb INS, RB, #32
1374  |  orr CARG4, CARG4, CARG1, lsr #21
1375  |    bls >3				// |x| >= 2^31?
1376  |   orr CARG3, CARG3, CARG4, lsl INS
1377  |  lsr CARG1, CARG4, RB
1378  |.if "func" == "floor"
1379  |   tst CARG3, CARG2, asr #31
1380  |   addne CARG1, CARG1, #1
1381  |.else
1382  |   bics CARG3, CARG3, CARG2, asr #31
1383  |   addsne CARG1, CARG1, #1
1384  |   ldrdvs CARG12, >9
1385  |   bvs ->fff_restv
1386  |.endif
1387  |    cmp CARG2, #0
1388  |    rsblt CARG1, CARG1, #0
1389  |1:
1390  |   mvn CARG2, #~LJ_TISNUM
1391  |  b ->fff_restv
1392  |
1393  |2:  // |x| < 1
1394  |  bcs ->fff_restv			// |x| is not finite.
1395  |  orr CARG3, CARG3, CARG1		// ztest = abs(hi) | lo
1396  |.if "func" == "floor"
1397  |  tst CARG3, CARG2, asr #31		// return (ztest & sign) == 0 ? 0 : -1
1398  |  moveq CARG1, #0
1399  |  mvnne CARG1, #0
1400  |.else
1401  |  bics CARG3, CARG3, CARG2, asr #31	// return (ztest & ~sign) == 0 ? 0 : 1
1402  |  moveq CARG1, #0
1403  |  movne CARG1, #1
1404  |.endif
1405  |  mvn CARG2, #~LJ_TISNUM
1406  |  b ->fff_restv
1407  |
1408  |3:  // |x| >= 2^31. Check for x == -(2^31).
1409  |  cmpeq CARG4, #0x80000000
1410  |.if "func" == "floor"
1411  |  cmpeq CARG3, #0
1412  |.endif
1413  |  bne >4
1414  |  cmp CARG2, #0
1415  |  movmi CARG1, #0x80000000
1416  |  bmi <1
1417  |4:
1418  |  bl ->vm_..func.._sf
1419  |  b ->fff_restv
1420  |.endmacro
1421  |
1422  |  math_round floor
1423  |  math_round ceil
1424  |
1425  |.align 8
1426  |9:
1427  |  .long 0x00000000, 0x41e00000	// 2^31.
1428  |
1429  |.ffunc_1 math_abs
1430  |  checktp CARG2, LJ_TISNUM
1431  |  bhi ->fff_fallback
1432  |  bicne CARG2, CARG2, #0x80000000
1433  |  bne ->fff_restv
1434  |  cmp CARG1, #0
1435  |  rsbslt CARG1, CARG1, #0
1436  |  ldrdvs CARG12, <9
1437  |  // Fallthrough.
1438  |
1439  |->fff_restv:
1440  |  // CARG12 = TValue result.
1441  |  ldr PC, [BASE, FRAME_PC]
1442  |  strd CARG12, [BASE, #-8]
1443  |->fff_res1:
1444  |  // PC = return.
1445  |  mov RC, #(1+1)*8
1446  |->fff_res:
1447  |  // RC = (nresults+1)*8, PC = return.
1448  |  ands CARG1, PC, #FRAME_TYPE
1449  |  ldreq INS, [PC, #-4]
1450  |   str RC, SAVE_MULTRES
1451  |  sub RA, BASE, #8
1452  |  bne ->vm_return
1453  |  decode_RB8 RB, INS
1454  |5:
1455  |  cmp RB, RC				// More results expected?
1456  |  bhi >6
1457  |  decode_RA8 CARG1, INS
1458  |   ins_next1
1459  |   ins_next2
1460  |  // Adjust BASE. KBASE is assumed to be set for the calling frame.
1461  |  sub BASE, RA, CARG1
1462  |   ins_next3
1463  |
1464  |6:  // Fill up results with nil.
1465  |  add CARG2, RA, RC
1466  |  mvn CARG1, #~LJ_TNIL
1467  |   add RC, RC, #8
1468  |  str CARG1, [CARG2, #-4]
1469  |  b <5
1470  |
1471  |.macro math_extern, func
1472  |.if HFABI
1473  |  .ffunc_d math_ .. func
1474  |.else
1475  |  .ffunc_n math_ .. func
1476  |.endif
1477  |  .IOS mov RA, BASE
1478  |  bl extern func
1479  |  .IOS mov BASE, RA
1480  |.if HFABI
1481  |  b ->fff_resd
1482  |.else
1483  |  b ->fff_restv
1484  |.endif
1485  |.endmacro
1486  |
1487  |.macro math_extern2, func
1488  |.if HFABI
1489  |  .ffunc_dd math_ .. func
1490  |.else
1491  |  .ffunc_nn math_ .. func
1492  |.endif
1493  |  .IOS mov RA, BASE
1494  |  bl extern func
1495  |  .IOS mov BASE, RA
1496  |.if HFABI
1497  |  b ->fff_resd
1498  |.else
1499  |  b ->fff_restv
1500  |.endif
1501  |.endmacro
1502  |
1503  |.if FPU
1504  |  .ffunc_d math_sqrt
1505  |  vsqrt.f64 d0, d0
1506  |->fff_resd:
1507  |  ldr PC, [BASE, FRAME_PC]
1508  |  vstr d0, [BASE, #-8]
1509  |  b ->fff_res1
1510  |.else
1511  |  math_extern sqrt
1512  |.endif
1513  |
1514  |.ffunc math_log
1515  |.if HFABI
1516  |  ldr CARG2, [BASE, #4]
1517  |   cmp NARGS8:RC, #8			// Need exactly 1 argument.
1518  |  vldr d0, [BASE]
1519  |   bne ->fff_fallback
1520  |.else
1521  |  ldrd CARG12, [BASE]
1522  |   cmp NARGS8:RC, #8			// Need exactly 1 argument.
1523  |   bne ->fff_fallback
1524  |.endif
1525  |  checktp CARG2, LJ_TISNUM
1526  |  bhs ->fff_fallback
1527  |  .IOS mov RA, BASE
1528  |  bl extern log
1529  |  .IOS mov BASE, RA
1530  |.if HFABI
1531  |  b ->fff_resd
1532  |.else
1533  |  b ->fff_restv
1534  |.endif
1535  |
1536  |  math_extern log10
1537  |  math_extern exp
1538  |  math_extern sin
1539  |  math_extern cos
1540  |  math_extern tan
1541  |  math_extern asin
1542  |  math_extern acos
1543  |  math_extern atan
1544  |  math_extern sinh
1545  |  math_extern cosh
1546  |  math_extern tanh
1547  |  math_extern2 pow
1548  |  math_extern2 atan2
1549  |  math_extern2 fmod
1550  |
1551  |.if HFABI
1552  |  .ffunc math_ldexp
1553  |  ldr CARG4, [BASE, #4]
1554  |  ldrd CARG12, [BASE, #8]
1555  |   cmp NARGS8:RC, #16
1556  |   blo ->fff_fallback
1557  |  vldr d0, [BASE]
1558  |  checktp CARG4, LJ_TISNUM
1559  |  bhs ->fff_fallback
1560  |  checktp CARG2, LJ_TISNUM
1561  |  bne ->fff_fallback
1562  |  .IOS mov RA, BASE
1563  |  bl extern ldexp			// (double x, int exp)
1564  |  .IOS mov BASE, RA
1565  |  b ->fff_resd
1566  |.else
1567  |.ffunc_2 math_ldexp
1568  |  checktp CARG2, LJ_TISNUM
1569  |  bhs ->fff_fallback
1570  |  checktp CARG4, LJ_TISNUM
1571  |  bne ->fff_fallback
1572  |  .IOS mov RA, BASE
1573  |  bl extern ldexp			// (double x, int exp)
1574  |  .IOS mov BASE, RA
1575  |  b ->fff_restv
1576  |.endif
1577  |
1578  |.if HFABI
1579  |.ffunc_d math_frexp
1580  |  mov CARG1, sp
1581  |  .IOS mov RA, BASE
1582  |  bl extern frexp
1583  |  .IOS mov BASE, RA
1584  |   ldr CARG3, [sp]
1585  |   mvn CARG4, #~LJ_TISNUM
1586  |    ldr PC, [BASE, FRAME_PC]
1587  |  vstr d0, [BASE, #-8]
1588  |    mov RC, #(2+1)*8
1589  |   strd CARG34, [BASE]
1590  |  b ->fff_res
1591  |.else
1592  |.ffunc_n math_frexp
1593  |  mov CARG3, sp
1594  |  .IOS mov RA, BASE
1595  |  bl extern frexp
1596  |  .IOS mov BASE, RA
1597  |   ldr CARG3, [sp]
1598  |   mvn CARG4, #~LJ_TISNUM
1599  |    ldr PC, [BASE, FRAME_PC]
1600  |  strd CARG12, [BASE, #-8]
1601  |    mov RC, #(2+1)*8
1602  |   strd CARG34, [BASE]
1603  |  b ->fff_res
1604  |.endif
1605  |
1606  |.if HFABI
1607  |.ffunc_d math_modf
1608  |  sub CARG1, BASE, #8
1609  |   ldr PC, [BASE, FRAME_PC]
1610  |  .IOS mov RA, BASE
1611  |  bl extern modf
1612  |  .IOS mov BASE, RA
1613  |   mov RC, #(2+1)*8
1614  |  vstr d0, [BASE]
1615  |  b ->fff_res
1616  |.else
1617  |.ffunc_n math_modf
1618  |  sub CARG3, BASE, #8
1619  |   ldr PC, [BASE, FRAME_PC]
1620  |  .IOS mov RA, BASE
1621  |  bl extern modf
1622  |  .IOS mov BASE, RA
1623  |   mov RC, #(2+1)*8
1624  |  strd CARG12, [BASE]
1625  |  b ->fff_res
1626  |.endif
1627  |
1628  |.macro math_minmax, name, cond, fcond
1629  |.if FPU
1630  |  .ffunc_1 name
1631  |   add RB, BASE, RC
1632  |  checktp CARG2, LJ_TISNUM
1633  |   add RA, BASE, #8
1634  |  bne >4
1635  |1:  // Handle integers.
1636  |  ldrd CARG34, [RA]
1637  |   cmp RA, RB
1638  |   bhs ->fff_restv
1639  |  checktp CARG4, LJ_TISNUM
1640  |  bne >3
1641  |  cmp CARG1, CARG3
1642  |   add RA, RA, #8
1643  |  mov..cond CARG1, CARG3
1644  |  b <1
1645  |3:  // Convert intermediate result to number and continue below.
1646  |  vmov s4, CARG1
1647  |  bhi ->fff_fallback
1648  |  vldr d1, [RA]
1649  |  vcvt.f64.s32 d0, s4
1650  |  b >6
1651  |
1652  |4:
1653  |  vldr d0, [BASE]
1654  |  bhi ->fff_fallback
1655  |5:  // Handle numbers.
1656  |  ldrd CARG34, [RA]
1657  |  vldr d1, [RA]
1658  |   cmp RA, RB
1659  |   bhs ->fff_resd
1660  |  checktp CARG4, LJ_TISNUM
1661  |  bhs >7
1662  |6:
1663  |  vcmp.f64 d0, d1
1664  |  vmrs
1665  |   add RA, RA, #8
1666  |  vmov..fcond.f64 d0, d1
1667  |  b <5
1668  |7:  // Convert integer to number and continue above.
1669  |  vmov s4, CARG3
1670  |  bhi ->fff_fallback
1671  |  vcvt.f64.s32 d1, s4
1672  |  b <6
1673  |
1674  |.else
1675  |
1676  |  .ffunc_1 name
1677  |  checktp CARG2, LJ_TISNUM
1678  |   mov RA, #8
1679  |  bne >4
1680  |1:  // Handle integers.
1681  |  ldrd CARG34, [BASE, RA]
1682  |   cmp RA, RC
1683  |   bhs ->fff_restv
1684  |  checktp CARG4, LJ_TISNUM
1685  |  bne >3
1686  |  cmp CARG1, CARG3
1687  |   add RA, RA, #8
1688  |  mov..cond CARG1, CARG3
1689  |  b <1
1690  |3:  // Convert intermediate result to number and continue below.
1691  |  bhi ->fff_fallback
1692  |  bl extern __aeabi_i2d
1693  |  ldrd CARG34, [BASE, RA]
1694  |  b >6
1695  |
1696  |4:
1697  |  bhi ->fff_fallback
1698  |5:  // Handle numbers.
1699  |  ldrd CARG34, [BASE, RA]
1700  |   cmp RA, RC
1701  |   bhs ->fff_restv
1702  |  checktp CARG4, LJ_TISNUM
1703  |  bhs >7
1704  |6:
1705  |  bl extern __aeabi_cdcmple
1706  |   add RA, RA, #8
1707  |  mov..fcond CARG1, CARG3
1708  |  mov..fcond CARG2, CARG4
1709  |  b <5
1710  |7:  // Convert integer to number and continue above.
1711  |  bhi ->fff_fallback
1712  |  strd CARG12, TMPD
1713  |  mov CARG1, CARG3
1714  |  bl extern __aeabi_i2d
1715  |  ldrd CARG34, TMPD
1716  |  b <6
1717  |.endif
1718  |.endmacro
1719  |
1720  |  math_minmax math_min, gt, hs
1721  |  math_minmax math_max, lt, ls
1722  |
1723  |//-- String library -----------------------------------------------------
1724  |
1725  |.ffunc string_byte			// Only handle the 1-arg case here.
1726  |  ldrd CARG12, [BASE]
1727  |    ldr PC, [BASE, FRAME_PC]
1728  |   cmp NARGS8:RC, #8
1729  |   checktpeq CARG2, LJ_TSTR		// Need exactly 1 argument.
1730  |   bne ->fff_fallback
1731  |  ldr CARG3, STR:CARG1->len
1732  |   ldrb CARG1, STR:CARG1[1]		// Access is always ok (NUL at end).
1733  |   mvn CARG2, #~LJ_TISNUM
1734  |  cmp CARG3, #0
1735  |  moveq RC, #(0+1)*8
1736  |  movne RC, #(1+1)*8
1737  |   strd CARG12, [BASE, #-8]
1738  |  b ->fff_res
1739  |
1740  |.ffunc string_char			// Only handle the 1-arg case here.
1741  |  ffgccheck
1742  |  ldrd CARG12, [BASE]
1743  |    ldr PC, [BASE, FRAME_PC]
1744  |   cmp NARGS8:RC, #8			// Need exactly 1 argument.
1745  |   checktpeq CARG2, LJ_TISNUM
1746  |   bicseq CARG4, CARG1, #255
1747  |  mov CARG3, #1
1748  |   bne ->fff_fallback
1749  |  str CARG1, TMPD
1750  |  mov CARG2, TMPDp			// Points to stack. Little-endian.
1751  |->fff_newstr:
1752  |  // CARG2 = str, CARG3 = len.
1753  |   str BASE, L->base
1754  |  mov CARG1, L
1755  |   str PC, SAVE_PC
1756  |  bl extern lj_str_new		// (lua_State *L, char *str, size_t l)
1757  |->fff_resstr:
1758  |  // Returns GCstr *.
1759  |  ldr BASE, L->base
1760  |   mvn CARG2, #~LJ_TSTR
1761  |  b ->fff_restv
1762  |
1763  |.ffunc string_sub
1764  |  ffgccheck
1765  |  ldrd CARG12, [BASE]
1766  |   ldrd CARG34, [BASE, #16]
1767  |    cmp NARGS8:RC, #16
1768  |     mvn RB, #0
1769  |    beq >1
1770  |    blo ->fff_fallback
1771  |   checktp CARG4, LJ_TISNUM
1772  |    mov RB, CARG3
1773  |   bne ->fff_fallback
1774  |1:
1775  |  ldrd CARG34, [BASE, #8]
1776  |  checktp CARG2, LJ_TSTR
1777  |   ldreq CARG2, STR:CARG1->len
1778  |  checktpeq CARG4, LJ_TISNUM
1779  |  bne ->fff_fallback
1780  |  // CARG1 = str, CARG2 = str->len, CARG3 = start, RB = end
1781  |  add CARG4, CARG2, #1
1782  |  cmp CARG3, #0			// if (start < 0) start += len+1
1783  |  addlt CARG3, CARG3, CARG4
1784  |  cmp CARG3, #1			// if (start < 1) start = 1
1785  |  movlt CARG3, #1
1786  |  cmp RB, #0				// if (end < 0) end += len+1
1787  |  addlt RB, RB, CARG4
1788  |  bic RB, RB, RB, asr #31		// if (end < 0) end = 0
1789  |  cmp RB, CARG2			// if (end > len) end = len
1790  |   add CARG1, STR:CARG1, #sizeof(GCstr)-1
1791  |  movgt RB, CARG2
1792  |   add CARG2, CARG1, CARG3
1793  |  subs CARG3, RB, CARG3		// len = end - start
1794  |   add CARG3, CARG3, #1		// len += 1
1795  |  bge ->fff_newstr
1796  |->fff_emptystr:
1797  |  sub STR:CARG1, DISPATCH, #-DISPATCH_GL(strempty)
1798  |  mvn CARG2, #~LJ_TSTR
1799  |  b ->fff_restv
1800  |
1801  |.macro ffstring_op, name
1802  |  .ffunc string_ .. name
1803  |  ffgccheck
1804  |  ldr CARG3, [BASE, #4]
1805  |   cmp NARGS8:RC, #8
1806  |  ldr STR:CARG2, [BASE]
1807  |   blo ->fff_fallback
1808  |  sub SBUF:CARG1, DISPATCH, #-DISPATCH_GL(tmpbuf)
1809  |  checkstr CARG3, ->fff_fallback
1810  |  ldr CARG4, SBUF:CARG1->b
1811  |   str BASE, L->base
1812  |   str PC, SAVE_PC
1813  |   str L, SBUF:CARG1->L
1814  |  str CARG4, SBUF:CARG1->w
1815  |  bl extern lj_buf_putstr_ .. name
1816  |  bl extern lj_buf_tostr
1817  |  b ->fff_resstr
1818  |.endmacro
1819  |
1820  |ffstring_op reverse
1821  |ffstring_op lower
1822  |ffstring_op upper
1823  |
1824  |//-- Bit library --------------------------------------------------------
1825  |
1826  |->vm_tobit_fb:
1827  |  bhi ->fff_fallback
1828  |.if FPU
1829  |// FP number to bit conversion for hard-float. Clobbers r0, d0-d1.
1830  |  vldr d1, >9
1831  |  vmov d0, CARG1, CARG2
1832  |  vadd.f64 d0, d0, d1
1833  |  vmov CARG1, s0
1834  |  bx lr
1835  |9:
1836  |  .long 0, 0x43380000		// (double)(2^52 + 2^51).
1837  |.else
1838  |// FP number to bit conversion for soft-float. Clobbers r0-r3.
1839  |->vm_tobit:
1840  |  lsl RB, CARG2, #1
1841  |  adds RB, RB, #0x00200000
1842  |  movpl CARG1, #0			// |x| < 1?
1843  |  bxpl lr
1844  |  mvn CARG4, #0x3e0
1845  |  subs RB, CARG4, RB, asr #21
1846  |  bmi >1				// |x| >= 2^32?
1847  |  lsl CARG4, CARG2, #11
1848  |  orr CARG4, CARG4, #0x80000000
1849  |  orr CARG4, CARG4, CARG1, lsr #21
1850  |   cmp CARG2, #0
1851  |  lsr CARG1, CARG4, RB
1852  |   rsblt CARG1, CARG1, #0
1853  |  bx lr
1854  |1:
1855  |  add RB, RB, #21
1856  |  lsr CARG4, CARG1, RB
1857  |  rsb RB, RB, #20
1858  |  lsl CARG1, CARG2, #12
1859  |   cmp CARG2, #0
1860  |  orr CARG1, CARG4, CARG1, lsl RB
1861  |   rsblt CARG1, CARG1, #0
1862  |  bx lr
1863  |.endif
1864  |
1865  |.macro .ffunc_bit, name
1866  |  .ffunc_1 bit_..name
1867  |  checktp CARG2, LJ_TISNUM
1868  |  blne ->vm_tobit_fb
1869  |.endmacro
1870  |
1871  |.ffunc_bit tobit
1872  |  mvn CARG2, #~LJ_TISNUM
1873  |  b ->fff_restv
1874  |
1875  |.macro .ffunc_bit_op, name, ins
1876  |  .ffunc_bit name
1877  |  mov CARG3, CARG1
1878  |  mov RA, #8
1879  |1:
1880  |  ldrd CARG12, [BASE, RA]
1881  |   cmp RA, NARGS8:RC
1882  |    add RA, RA, #8
1883  |   bge >2
1884  |  checktp CARG2, LJ_TISNUM
1885  |  blne ->vm_tobit_fb
1886  |  ins CARG3, CARG3, CARG1
1887  |  b <1
1888  |.endmacro
1889  |
1890  |.ffunc_bit_op band, and
1891  |.ffunc_bit_op bor, orr
1892  |.ffunc_bit_op bxor, eor
1893  |
1894  |2:
1895  |  mvn CARG4, #~LJ_TISNUM
1896  |   ldr PC, [BASE, FRAME_PC]
1897  |  strd CARG34, [BASE, #-8]
1898  |  b ->fff_res1
1899  |
1900  |.ffunc_bit bswap
1901  |  eor CARG3, CARG1, CARG1, ror #16
1902  |  bic CARG3, CARG3, #0x00ff0000
1903  |  ror CARG1, CARG1, #8
1904  |   mvn CARG2, #~LJ_TISNUM
1905  |  eor CARG1, CARG1, CARG3, lsr #8
1906  |  b ->fff_restv
1907  |
1908  |.ffunc_bit bnot
1909  |  mvn CARG1, CARG1
1910  |  mvn CARG2, #~LJ_TISNUM
1911  |  b ->fff_restv
1912  |
1913  |.macro .ffunc_bit_sh, name, ins, shmod
1914  |  .ffunc bit_..name
1915  |  ldrd CARG12, [BASE, #8]
1916  |   cmp NARGS8:RC, #16
1917  |   blo ->fff_fallback
1918  |  checktp CARG2, LJ_TISNUM
1919  |  blne ->vm_tobit_fb
1920  |.if shmod == 0
1921  |  and RA, CARG1, #31
1922  |.else
1923  |  rsb RA, CARG1, #0
1924  |.endif
1925  |  ldrd CARG12, [BASE]
1926  |  checktp CARG2, LJ_TISNUM
1927  |  blne ->vm_tobit_fb
1928  |  ins CARG1, CARG1, RA
1929  |  mvn CARG2, #~LJ_TISNUM
1930  |  b ->fff_restv
1931  |.endmacro
1932  |
1933  |.ffunc_bit_sh lshift, lsl, 0
1934  |.ffunc_bit_sh rshift, lsr, 0
1935  |.ffunc_bit_sh arshift, asr, 0
1936  |.ffunc_bit_sh rol, ror, 1
1937  |.ffunc_bit_sh ror, ror, 0
1938  |
1939  |//-----------------------------------------------------------------------
1940  |
1941  |->fff_fallback:			// Call fast function fallback handler.
1942  |  // BASE = new base, RC = nargs*8
1943  |   ldr CARG3, [BASE, FRAME_FUNC]
1944  |  ldr CARG2, L->maxstack
1945  |  add CARG1, BASE, NARGS8:RC
1946  |    ldr PC, [BASE, FRAME_PC]		// Fallback may overwrite PC.
1947  |  str CARG1, L->top
1948  |   ldr CARG3, CFUNC:CARG3->f
1949  |    str BASE, L->base
1950  |  add CARG1, CARG1, #8*LUA_MINSTACK
1951  |    str PC, SAVE_PC			// Redundant (but a defined value).
1952  |  cmp CARG1, CARG2
1953  |   mov CARG1, L
1954  |  bhi >5				// Need to grow stack.
1955  |   blx CARG3				// (lua_State *L)
1956  |  // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1957  |   ldr BASE, L->base
1958  |  cmp CRET1, #0
1959  |   lsl RC, CRET1, #3
1960  |   sub RA, BASE, #8
1961  |  bgt ->fff_res			// Returned nresults+1?
1962  |1:  // Returned 0 or -1: retry fast path.
1963  |   ldr CARG1, L->top
1964  |    ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1965  |   sub NARGS8:RC, CARG1, BASE
1966  |  bne ->vm_call_tail			// Returned -1?
1967  |  ins_callt				// Returned 0: retry fast path.
1968  |
1969  |// Reconstruct previous base for vmeta_call during tailcall.
1970  |->vm_call_tail:
1971  |  ands CARG1, PC, #FRAME_TYPE
1972  |   bic CARG2, PC, #FRAME_TYPEP
1973  |  ldreq INS, [PC, #-4]
1974  |  andeq CARG2, MASKR8, INS, lsr #5	// Conditional decode_RA8.
1975  |  addeq CARG2, CARG2, #8
1976  |  sub RB, BASE, CARG2
1977  |  b ->vm_call_dispatch		// Resolve again for tailcall.
1978  |
1979  |5:  // Grow stack for fallback handler.
1980  |  mov CARG2, #LUA_MINSTACK
1981  |  bl extern lj_state_growstack	// (lua_State *L, int n)
1982  |  ldr BASE, L->base
1983  |  cmp CARG1, CARG1			// Set zero-flag to force retry.
1984  |  b <1
1985  |
1986  |->fff_gcstep:			// Call GC step function.
1987  |  // BASE = new base, RC = nargs*8
1988  |  mov RA, lr
1989  |   str BASE, L->base
1990  |  add CARG2, BASE, NARGS8:RC
1991  |   str PC, SAVE_PC			// Redundant (but a defined value).
1992  |  str CARG2, L->top
1993  |  mov CARG1, L
1994  |  bl extern lj_gc_step		// (lua_State *L)
1995  |   ldr BASE, L->base
1996  |  mov lr, RA				// Help return address predictor.
1997  |   ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1998  |  bx lr
1999  |
2000  |//-----------------------------------------------------------------------
2001  |//-- Special dispatch targets -------------------------------------------
2002  |//-----------------------------------------------------------------------
2003  |
2004  |->vm_record:				// Dispatch target for recording phase.
2005  |.if JIT
2006  |  ldrb CARG1, [DISPATCH, #DISPATCH_GL(hookmask)]
2007  |  tst CARG1, #HOOK_VMEVENT		// No recording while in vmevent.
2008  |  bne >5
2009  |  // Decrement the hookcount for consistency, but always do the call.
2010  |   ldr CARG2, [DISPATCH, #DISPATCH_GL(hookcount)]
2011  |  tst CARG1, #HOOK_ACTIVE
2012  |  bne >1
2013  |   sub CARG2, CARG2, #1
2014  |  tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
2015  |   strne CARG2, [DISPATCH, #DISPATCH_GL(hookcount)]
2016  |  b >1
2017  |.endif
2018  |
2019  |->vm_rethook:			// Dispatch target for return hooks.
2020  |  ldrb CARG1, [DISPATCH, #DISPATCH_GL(hookmask)]
2021  |  tst CARG1, #HOOK_ACTIVE		// Hook already active?
2022  |  beq >1
2023  |5:  // Re-dispatch to static ins.
2024  |  decode_OP OP, INS
2025  |  add OP, DISPATCH, OP, lsl #2
2026  |  ldr pc, [OP, #GG_DISP2STATIC]
2027  |
2028  |->vm_inshook:			// Dispatch target for instr/line hooks.
2029  |  ldrb CARG1, [DISPATCH, #DISPATCH_GL(hookmask)]
2030  |   ldr CARG2, [DISPATCH, #DISPATCH_GL(hookcount)]
2031  |  tst CARG1, #HOOK_ACTIVE		// Hook already active?
2032  |  bne <5
2033  |  tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
2034  |  beq <5
2035  |   subs CARG2, CARG2, #1
2036  |   str CARG2, [DISPATCH, #DISPATCH_GL(hookcount)]
2037  |   beq >1
2038  |  tst CARG1, #LUA_MASKLINE
2039  |  beq <5
2040  |1:
2041  |  mov CARG1, L
2042  |   str BASE, L->base
2043  |  mov CARG2, PC
2044  |  // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
2045  |  bl extern lj_dispatch_ins		// (lua_State *L, const BCIns *pc)
2046  |3:
2047  |  ldr BASE, L->base
2048  |4:  // Re-dispatch to static ins.
2049  |  ldrb OP, [PC, #-4]
2050  |   ldr INS, [PC, #-4]
2051  |  add OP, DISPATCH, OP, lsl #2
2052  |  ldr OP, [OP, #GG_DISP2STATIC]
2053  |   decode_RA8 RA, INS
2054  |   decode_RD RC, INS
2055  |  bx OP
2056  |
2057  |->cont_hook:				// Continue from hook yield.
2058  |  ldr CARG1, [CARG4, #-24]
2059  |   add PC, PC, #4
2060  |  str CARG1, SAVE_MULTRES		// Restore MULTRES for *M ins.
2061  |  b <4
2062  |
2063  |->vm_hotloop:			// Hot loop counter underflow.
2064  |.if JIT
2065  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Same as curr_topL(L).
2066  |   sub CARG1, DISPATCH, #-GG_DISP2J
2067  |   str PC, SAVE_PC
2068  |  ldr CARG3, LFUNC:CARG3->field_pc
2069  |   mov CARG2, PC
2070  |   str L, [DISPATCH, #DISPATCH_J(L)]
2071  |  ldrb CARG3, [CARG3, #PC2PROTO(framesize)]
2072  |   str BASE, L->base
2073  |  add CARG3, BASE, CARG3, lsl #3
2074  |  str CARG3, L->top
2075  |  bl extern lj_trace_hot		// (jit_State *J, const BCIns *pc)
2076  |  b <3
2077  |.endif
2078  |
2079  |->vm_callhook:			// Dispatch target for call hooks.
2080  |  mov CARG2, PC
2081  |.if JIT
2082  |  b >1
2083  |.endif
2084  |
2085  |->vm_hotcall:			// Hot call counter underflow.
2086  |.if JIT
2087  |  orr CARG2, PC, #1
2088  |1:
2089  |.endif
2090  |  add CARG4, BASE, RC
2091  |   str PC, SAVE_PC
2092  |    mov CARG1, L
2093  |   str BASE, L->base
2094  |    sub RA, RA, BASE
2095  |  str CARG4, L->top
2096  |  bl extern lj_dispatch_call		// (lua_State *L, const BCIns *pc)
2097  |  // Returns ASMFunction.
2098  |  ldr BASE, L->base
2099  |   ldr CARG4, L->top
2100  |    mov CARG2, #0
2101  |  add RA, BASE, RA
2102  |   sub NARGS8:RC, CARG4, BASE
2103  |    str CARG2, SAVE_PC		// Invalidate for subsequent line hook.
2104  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2105  |   ldr INS, [PC, #-4]
2106  |  bx CRET1
2107  |
2108  |->cont_stitch:			// Trace stitching.
2109  |.if JIT
2110  |  // RA = resultptr, CARG4 = meta base
2111  |   ldr RB, SAVE_MULTRES
2112  |  ldr INS, [PC, #-4]
2113  |    ldr TRACE:CARG3, [CARG4, #-24]	// Save previous trace.
2114  |   subs RB, RB, #8
2115  |  decode_RA8 RC, INS			// Call base.
2116  |   beq >2
2117  |1:  // Move results down.
2118  |  ldrd CARG12, [RA]
2119  |    add RA, RA, #8
2120  |   subs RB, RB, #8
2121  |  strd CARG12, [BASE, RC]
2122  |    add RC, RC, #8
2123  |   bne <1
2124  |2:
2125  |   decode_RA8 RA, INS
2126  |   decode_RB8 RB, INS
2127  |   add RA, RA, RB
2128  |3:
2129  |   cmp RA, RC
2130  |  mvn CARG2, #~LJ_TNIL
2131  |   bhi >9				// More results wanted?
2132  |
2133  |  ldrh RA, TRACE:CARG3->traceno
2134  |  ldrh RC, TRACE:CARG3->link
2135  |  cmp RC, RA
2136  |  beq ->cont_nop			// Blacklisted.
2137  |  cmp RC, #0
2138  |  bne =>BC_JLOOP			// Jump to stitched trace.
2139  |
2140  |  // Stitch a new trace to the previous trace.
2141  |  str RA, [DISPATCH, #DISPATCH_J(exitno)]
2142  |  str L, [DISPATCH, #DISPATCH_J(L)]
2143  |  str BASE, L->base
2144  |  sub CARG1, DISPATCH, #-GG_DISP2J
2145  |  mov CARG2, PC
2146  |  bl extern lj_dispatch_stitch	// (jit_State *J, const BCIns *pc)
2147  |  ldr BASE, L->base
2148  |  b ->cont_nop
2149  |
2150  |9:  // Fill up results with nil.
2151  |  strd CARG12, [BASE, RC]
2152  |  add RC, RC, #8
2153  |  b <3
2154  |.endif
2155  |
2156  |->vm_profhook:			// Dispatch target for profiler hook.
2157#if LJ_HASPROFILE
2158  |  mov CARG1, L
2159  |   str BASE, L->base
2160  |  mov CARG2, PC
2161  |  bl extern lj_dispatch_profile	// (lua_State *L, const BCIns *pc)
2162  |  // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2163  |  ldr BASE, L->base
2164  |  sub PC, PC, #4
2165  |  b ->cont_nop
2166#endif
2167  |
2168  |//-----------------------------------------------------------------------
2169  |//-- Trace exit handler -------------------------------------------------
2170  |//-----------------------------------------------------------------------
2171  |
2172  |->vm_exit_handler:
2173  |.if JIT
2174  |  sub sp, sp, #12
2175  |  push {r0,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11,r12}
2176  |  ldr CARG1, [sp, #64]	// Load original value of lr.
2177  |   ldr DISPATCH, [lr]	// Load DISPATCH.
2178  |    add CARG3, sp, #64	// Recompute original value of sp.
2179  |   mv_vmstate CARG4, EXIT
2180  |    str CARG3, [sp, #52]	// Store sp in RID_SP
2181  |   st_vmstate CARG4
2182  |  ldr CARG2, [CARG1, #-4]!	// Get exit instruction.
2183  |   str CARG1, [sp, #56]	// Store exit pc in RID_LR and RID_PC.
2184  |   str CARG1, [sp, #60]
2185  |.if FPU
2186  |  vpush {d0-d15}
2187  |.endif
2188  |  lsl CARG2, CARG2, #8
2189  |  add CARG1, CARG1, CARG2, asr #6
2190  |   ldr CARG2, [lr, #4]	// Load exit stub group offset.
2191  |   sub CARG1, CARG1, lr
2192  |  ldr L, [DISPATCH, #DISPATCH_GL(cur_L)]
2193  |   add CARG1, CARG2, CARG1, lsr #2	// Compute exit number.
2194  |    ldr BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
2195  |   str CARG1, [DISPATCH, #DISPATCH_J(exitno)]
2196  |   mov CARG4, #0
2197  |    str BASE, L->base
2198  |  str L, [DISPATCH, #DISPATCH_J(L)]
2199  |   str CARG4, [DISPATCH, #DISPATCH_GL(jit_base)]
2200  |  sub CARG1, DISPATCH, #-GG_DISP2J
2201  |  mov CARG2, sp
2202  |  bl extern lj_trace_exit		// (jit_State *J, ExitState *ex)
2203  |  // Returns MULTRES (unscaled) or negated error code.
2204  |  ldr CARG2, L->cframe
2205  |   ldr BASE, L->base
2206  |  bic CARG2, CARG2, #~CFRAME_RAWMASK	// Use two steps: bic sp is deprecated.
2207  |  mov sp, CARG2
2208  |   ldr PC, SAVE_PC			// Get SAVE_PC.
2209  |  str L, SAVE_L			// Set SAVE_L (on-trace resume/yield).
2210  |  b >1
2211  |.endif
2212  |->vm_exit_interp:
2213  |  // CARG1 = MULTRES or negated error code, BASE, PC and DISPATCH set.
2214  |.if JIT
2215  |  ldr L, SAVE_L
2216  |1:
2217  |  cmn CARG1, #LUA_ERRERR
2218  |  bhs >9				// Check for error from exit.
2219  |   lsl RC, CARG1, #3
2220  |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2221  |   str RC, SAVE_MULTRES
2222  |   mov CARG3, #0
2223  |   str BASE, L->base
2224  |  ldr CARG2, LFUNC:CARG2->field_pc
2225  |   str CARG3, [DISPATCH, #DISPATCH_GL(jit_base)]
2226  |    mv_vmstate CARG4, INTERP
2227  |  ldr KBASE, [CARG2, #PC2PROTO(k)]
2228  |  // Modified copy of ins_next which handles function header dispatch, too.
2229  |  ldrb OP, [PC]
2230  |     mov MASKR8, #255
2231  |   ldr INS, [PC], #4
2232  |     lsl MASKR8, MASKR8, #3		// MASKR8 = 255*8.
2233  |    st_vmstate CARG4
2234  |  cmn CARG1, #17			// Static dispatch?
2235  |  beq >5
2236  |  cmp OP, #BC_FUNCC+2		// Fast function?
2237  |  bhs >4
2238  |2:
2239  |  cmp OP, #BC_FUNCF			// Function header?
2240  |  ldr OP, [DISPATCH, OP, lsl #2]
2241  |   decode_RA8 RA, INS
2242  |   lsrlo RC, INS, #16	// No: Decode operands A*8 and D.
2243  |   subhs RC, RC, #8
2244  |   addhs RA, RA, BASE	// Yes: RA = BASE+framesize*8, RC = nargs*8
2245  |   ldrhs CARG3, [BASE, FRAME_FUNC]
2246  |  bx OP
2247  |
2248  |4:  // Check frame below fast function.
2249  |  ldr CARG1, [BASE, FRAME_PC]
2250  |  ands CARG2, CARG1, #FRAME_TYPE
2251  |  bne <2			// Trace stitching continuation?
2252  |  // Otherwise set KBASE for Lua function below fast function.
2253  |  ldr CARG3, [CARG1, #-4]
2254  |  decode_RA8 CARG1, CARG3
2255  |  sub CARG2, BASE, CARG1
2256  |  ldr LFUNC:CARG3, [CARG2, #-16]
2257  |  ldr CARG3, LFUNC:CARG3->field_pc
2258  |  ldr KBASE, [CARG3, #PC2PROTO(k)]
2259  |  b <2
2260  |
2261  |5:  // Dispatch to static entry of original ins replaced by BC_JLOOP.
2262  |  ldr CARG1, [DISPATCH, #DISPATCH_J(trace)]
2263  |  decode_RD RC, INS
2264  |  ldr TRACE:CARG1, [CARG1, RC, lsl #2]
2265  |  ldr INS, TRACE:CARG1->startins
2266  |  decode_OP OP, INS
2267  |   decode_RA8 RA, INS
2268  |  add OP, DISPATCH, OP, lsl #2
2269  |   decode_RD RC, INS
2270  |  ldr pc, [OP, #GG_DISP2STATIC]
2271  |
2272  |9:  // Rethrow error from the right C frame.
2273  |  rsb CARG2, CARG1, #0
2274  |  mov CARG1, L
2275  |  bl extern lj_err_trace		// (lua_State *L, int errcode)
2276  |.endif
2277  |
2278  |//-----------------------------------------------------------------------
2279  |//-- Math helper functions ----------------------------------------------
2280  |//-----------------------------------------------------------------------
2281  |
2282  |// FP value rounding. Called from JIT code.
2283  |//
2284  |// double lj_vm_floor/ceil/trunc(double x);
2285  |.macro vm_round, func, hf
2286  |.if hf == 1
2287  |  vmov CARG1, CARG2, d0
2288  |.endif
2289  |  lsl CARG3, CARG2, #1
2290  |  adds RB, CARG3, #0x00200000
2291  |  bpl >2				// |x| < 1?
2292  |  mvn CARG4, #0x3cc
2293  |  subs RB, CARG4, RB, asr #21	// 2^0: RB = 51, 2^51: RB = 0.
2294  |  bxlo lr				// |x| >= 2^52: done.
2295  |  mvn CARG4, #1
2296  |   bic CARG3, CARG1, CARG4, lsl RB	// ztest = lo & ~lomask
2297  |  and CARG1, CARG1, CARG4, lsl RB	// lo &= lomask
2298  |  subs RB, RB, #32
2299  |   bicpl CARG4, CARG2, CARG4, lsl RB	// |x| <= 2^20: ztest |= hi & ~himask
2300  |   orrpl CARG3, CARG3, CARG4
2301  |   mvnpl CARG4, #1
2302  |  andpl CARG2, CARG2, CARG4, lsl RB	// |x| <= 2^20: hi &= himask
2303  |.if "func" == "floor"
2304  |   tst CARG3, CARG2, asr #31		// iszero = ((ztest & signmask) == 0)
2305  |.else
2306  |   bics CARG3, CARG3, CARG2, asr #31	// iszero = ((ztest & ~signmask) == 0)
2307  |.endif
2308  |.if hf == 1
2309  |  vmoveq d0, CARG1, CARG2
2310  |.endif
2311  |  bxeq lr				// iszero: done.
2312  |  mvn CARG4, #1
2313  |  cmp RB, #0
2314  |  lslpl CARG3, CARG4, RB
2315  |  mvnmi CARG3, #0
2316  |  add RB, RB, #32
2317  |  subs CARG1, CARG1, CARG4, lsl RB	// lo = lo-lomask
2318  |  sbc CARG2, CARG2, CARG3		// hi = hi-himask+carry
2319  |.if hf == 1
2320  |  vmov d0, CARG1, CARG2
2321  |.endif
2322  |  bx lr
2323  |
2324  |2:  // |x| < 1:
2325  |  bxcs lr				// |x| is not finite.
2326  |  orr CARG3, CARG3, CARG1		// ztest = (2*hi) | lo
2327  |.if "func" == "floor"
2328  |  tst CARG3, CARG2, asr #31		// iszero = ((ztest & signmask) == 0)
2329  |.else
2330  |  bics CARG3, CARG3, CARG2, asr #31	// iszero = ((ztest & ~signmask) == 0)
2331  |.endif
2332  |  mov CARG1, #0			// lo = 0
2333  |  and CARG2, CARG2, #0x80000000
2334  |  ldrne CARG4, <9			// hi = sign(x) | (iszero ? 0.0 : 1.0)
2335  |  orrne CARG2, CARG2, CARG4
2336  |.if hf == 1
2337  |  vmov d0, CARG1, CARG2
2338  |.endif
2339  |  bx lr
2340  |.endmacro
2341  |
2342  |9:
2343  |  .long 0x3ff00000			// hiword(+1.0)
2344  |
2345  |->vm_floor:
2346  |.if HFABI
2347  |  vm_round floor, 1
2348  |.endif
2349  |->vm_floor_sf:
2350  |  vm_round floor, 0
2351  |
2352  |->vm_ceil:
2353  |.if HFABI
2354  |  vm_round ceil, 1
2355  |.endif
2356  |->vm_ceil_sf:
2357  |  vm_round ceil, 0
2358  |
2359  |.macro vm_trunc, hf
2360  |.if JIT
2361  |.if hf == 1
2362  |  vmov CARG1, CARG2, d0
2363  |.endif
2364  |  lsl CARG3, CARG2, #1
2365  |  adds RB, CARG3, #0x00200000
2366  |  andpl CARG2, CARG2, #0x80000000	// |x| < 1? hi = sign(x), lo = 0.
2367  |  movpl CARG1, #0
2368  |.if hf == 1
2369  |  vmovpl d0, CARG1, CARG2
2370  |.endif
2371  |  bxpl lr
2372  |  mvn CARG4, #0x3cc
2373  |  subs RB, CARG4, RB, asr #21	// 2^0: RB = 51, 2^51: RB = 0.
2374  |  bxlo lr				// |x| >= 2^52: already done.
2375  |  mvn CARG4, #1
2376  |  and CARG1, CARG1, CARG4, lsl RB	// lo &= lomask
2377  |  subs RB, RB, #32
2378  |  andpl CARG2, CARG2, CARG4, lsl RB	// |x| <= 2^20: hi &= himask
2379  |.if hf == 1
2380  |  vmov d0, CARG1, CARG2
2381  |.endif
2382  |  bx lr
2383  |.endif
2384  |.endmacro
2385  |
2386  |->vm_trunc:
2387  |.if HFABI
2388  |  vm_trunc 1
2389  |.endif
2390  |->vm_trunc_sf:
2391  |  vm_trunc 0
2392  |
2393  |  // double lj_vm_mod(double dividend, double divisor);
2394  |->vm_mod:
2395  |.if FPU
2396  |  // Special calling convention. Also, RC (r11) is not preserved.
2397  |  vdiv.f64 d0, d6, d7
2398  |   mov RC, lr
2399  |  vmov CARG1, CARG2, d0
2400  |  bl ->vm_floor_sf
2401  |  vmov d0, CARG1, CARG2
2402  |  vmul.f64 d0, d0, d7
2403  |   mov lr, RC
2404  |  vsub.f64 d6, d6, d0
2405  |  bx lr
2406  |.else
2407  |  push {r0, r1, r2, r3, r4, lr}
2408  |  bl extern __aeabi_ddiv
2409  |  bl ->vm_floor_sf
2410  |  ldrd CARG34, [sp, #8]
2411  |  bl extern __aeabi_dmul
2412  |  ldrd CARG34, [sp]
2413  |  eor CARG2, CARG2, #0x80000000
2414  |  bl extern __aeabi_dadd
2415  |  add sp, sp, #20
2416  |  pop {pc}
2417  |.endif
2418  |
2419  |  // int lj_vm_modi(int dividend, int divisor);
2420  |->vm_modi:
2421  |  ands RB, CARG1, #0x80000000
2422  |  rsbmi CARG1, CARG1, #0		// a = |dividend|
2423  |  eor RB, RB, CARG2, asr #1		// Keep signdiff and sign(divisor).
2424  |  cmp CARG2, #0
2425  |  rsbmi CARG2, CARG2, #0		// b = |divisor|
2426  |  subs CARG4, CARG2, #1
2427  |  cmpne CARG1, CARG2
2428  |  moveq CARG1, #0			// if (b == 1 || a == b) a = 0
2429  |  tsthi CARG2, CARG4
2430  |  andeq CARG1, CARG1, CARG4		// else if ((b & (b-1)) == 0) a &= b-1
2431  |  bls >1
2432  |  // Use repeated subtraction to get the remainder.
2433  |  clz CARG3, CARG1
2434  |  clz CARG4, CARG2
2435  |  sub CARG4, CARG4, CARG3
2436  |  rsbs CARG3, CARG4, #31		// entry = (31-(clz(b)-clz(a)))*8
2437  |  addne pc, pc, CARG3, lsl #3	// Duff's device.
2438  |  nop
2439  {
2440    int i;
2441    for (i = 31; i >= 0; i--) {
2442      |  cmp CARG1, CARG2, lsl #i
2443      |  subhs CARG1, CARG1, CARG2, lsl #i
2444    }
2445  }
2446  |1:
2447  |  cmp CARG1, #0
2448  |  cmpne RB, #0
2449  |  submi CARG1, CARG1, CARG2		// if (y != 0 && signdiff) y = y - b
2450  |  eors CARG2, CARG1, RB, lsl #1
2451  |  rsbmi CARG1, CARG1, #0		// if (sign(divisor) != sign(y)) y = -y
2452  |  bx lr
2453  |
2454  |//-----------------------------------------------------------------------
2455  |//-- Number conversion functions ----------------------------------------
2456  |//-----------------------------------------------------------------------
2457  |
2458  |// int64_t lj_vm_num2int_check(double x)
2459  |->vm_num2int_check:
2460  |.if FPU
2461  |.if not HFABI
2462  |  vmov d0, CARG1, CARG2
2463  |.endif
2464  |  vcvt.s32.f64 s4, d0
2465  |  vcvt.f64.s32 d1, s4
2466  |  vcmp.f64 d0, d1
2467  |  vmrs
2468  |  bne >1
2469  |  vmov CRET1, s4
2470  |  mov CRET2, #0
2471  |  bx lr
2472  |
2473  |.else
2474  |
2475  |  asr CARG4, CARG2, #31		// sign = 0 or -1.
2476  |  lsl CARG2, CARG2, #1
2477  |  orrs RB, CARG2, CARG1
2478  |  bxeq lr				// Return 0 for +-0.
2479  |  mov RB, #1024
2480  |  add RB, RB, #30
2481  |  sub RB, RB, CARG2, lsr #21
2482  |  cmp RB, #32
2483  |  bhs >1				// Fail if |x| < 0x1p0 || |x| >= 0x1p32.
2484  |  lsr CARG3, CARG1, #21
2485  |  orr CARG2, CARG3, CARG2, lsl #10	// Left-aligned mantissa.
2486  |   rsb CARG3, RB, #32
2487  |   lsl CARG3, CARG2, CARG3
2488  |  orr CARG2, CARG2, #0x80000000	// Merge leading 1.
2489  |   orrs CARG3, CARG3, CARG1, lsl #11
2490  |  lsr CARG1, CARG2, RB		// lo = right-aligned absolute value.
2491  |   bne >1				// Fail if fractional part != 0.
2492  |  adds CRET1, CARG1, CARG4
2493  |  bmi >1				// Fail if lo+sign >= 0x80000000.
2494  |  eor CRET1, CRET1, CARG4		// lo = sign?-lo:lo = (lo+sign)^sign.
2495  |  mov CRET2, #0
2496  |  bx lr
2497  |.endif
2498  |1:
2499  |  mov CRET1, #0x80000000
2500  |  mov CRET2, #0x80000000
2501  |  bx lr
2502  |
2503  |// int64_t lj_vm_num2i64(double x)
2504  |->vm_num2i64:
2505  |// fallthrough, same as lj_vm_num2u64.
2506  |
2507  |// uint64_t lj_vm_num2u64(double x)
2508  |->vm_num2u64:
2509  |.if HFABI
2510  |  vmov CARG1, CARG2, d0
2511  |.endif
2512  |  lsl RB, CARG2, #1
2513  |  lsr RB, RB, #21
2514  |  sub RB, RB, #1020
2515  |  sub RB, RB, #3
2516  |  cmp RB, #116
2517  |  bhs >3				// Exponent out of range.
2518  |  asr CARG4, CARG2, #31		// sign = 0 or -1.
2519  |  lsl CARG2, CARG2, #12
2520  |  lsr CARG2, CARG2, #12
2521  |   rsbs RB, RB, #52
2522  |  orr CARG2, CARG2, #0x00100000
2523  |   bmi >2				// Shift mantissa left or right?
2524  |  lsr CARG1, CARG1, RB		// 64 bit right shift.
2525  |  lsr CARG3, CARG2, RB
2526  |  rsb RB, RB, #32
2527  |  orr CARG1, CARG1, CARG2, lsl RB
2528  |  rsb RB, RB, #0
2529  |  orr CARG1, CARG1, CARG2, lsr RB
2530  |  adds CRET1, CARG1, CARG4		// m = sign?-m:m = (m+sign)^sign.
2531  |  adc CRET2, CARG3, CARG4
2532  |1:
2533  |  eor CRET1, CRET1, CARG4
2534  |  eor CRET2, CRET2, CARG4
2535  |  bx lr
2536  |2:
2537  |  rsb RB, RB, #0
2538  |  lsl CARG2, CARG2, RB		// 64 bit left shift.
2539  |  lsl CARG3, CARG1, RB
2540  |  sub RB, RB, #32
2541  |  orr CARG2, CARG2, CARG1, lsl RB
2542  |  rsb RB, RB, #0
2543  |  orr CARG2, CARG2, CARG1, lsr RB
2544  |  adds CRET1, CARG3, CARG4
2545  |  adc CRET2, CARG2, CARG4
2546  |  b <1
2547  |3:
2548  |  mov CRET1, #0
2549  |  mov CRET2, #0
2550  |  bx lr
2551  |
2552  |// int32_t lj_vm_tobit(double x)
2553  |.if FPU
2554  |->vm_tobit:
2555  |  vldr d1, >9
2556  |.if not HFABI
2557  |  vmov d0, CARG1, CARG2
2558  |.endif
2559  |  vadd.f64 d0, d0, d1
2560  |  vmov CARG1, s0
2561  |  bx lr
2562  |9:
2563  |  .long 0, 0x43380000		// (double)(2^52 + 2^51).
2564  |.endif
2565  |
2566  |//-----------------------------------------------------------------------
2567  |//-- Miscellaneous functions --------------------------------------------
2568  |//-----------------------------------------------------------------------
2569  |
2570  |.define NEXT_TAB,		TAB:CARG1
2571  |.define NEXT_RES,		CARG1
2572  |.define NEXT_IDX,		CARG2
2573  |.define NEXT_TMP0,		CARG3
2574  |.define NEXT_TMP1,		CARG4
2575  |.define NEXT_LIM,		r12
2576  |.define NEXT_RES_PTR,	sp
2577  |.define NEXT_RES_VAL,	[sp]
2578  |.define NEXT_RES_KEY_I,	[sp, #8]
2579  |.define NEXT_RES_KEY_IT,	[sp, #12]
2580  |
2581  |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2582  |// Next idx returned in CRET2.
2583  |->vm_next:
2584  |.if JIT
2585  |  ldr NEXT_TMP0, NEXT_TAB->array
2586  |   ldr NEXT_LIM, NEXT_TAB->asize
2587  |  add NEXT_TMP0, NEXT_TMP0, NEXT_IDX, lsl #3
2588  |1:  // Traverse array part.
2589  |   subs NEXT_TMP1, NEXT_IDX, NEXT_LIM
2590  |   bhs >5
2591  |  ldr NEXT_TMP1, [NEXT_TMP0, #4]
2592  |   str NEXT_IDX, NEXT_RES_KEY_I
2593  |   add NEXT_TMP0, NEXT_TMP0, #8
2594  |   add NEXT_IDX, NEXT_IDX, #1
2595  |  checktp NEXT_TMP1, LJ_TNIL
2596  |  beq <1				// Skip holes in array part.
2597  |  ldr NEXT_TMP0, [NEXT_TMP0, #-8]
2598  |   mov NEXT_RES, NEXT_RES_PTR
2599  |  strd NEXT_TMP0, NEXT_RES_VAL	// Stores NEXT_TMP1, too.
2600  |  mvn NEXT_TMP0, #~LJ_TISNUM
2601  |  str NEXT_TMP0, NEXT_RES_KEY_IT
2602  |  bx lr
2603  |
2604  |5:  // Traverse hash part.
2605  |  ldr NEXT_TMP0, NEXT_TAB->hmask
2606  |   ldr NODE:NEXT_RES, NEXT_TAB->node
2607  |   add NEXT_TMP1, NEXT_TMP1, NEXT_TMP1, lsl #1
2608  |  add NEXT_LIM, NEXT_LIM, NEXT_TMP0
2609  |   add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP1, lsl #3
2610  |6:
2611  |  cmp NEXT_IDX, NEXT_LIM
2612  |  bhi >9
2613  |  ldr NEXT_TMP1, NODE:NEXT_RES->val.it
2614  |  checktp NEXT_TMP1, LJ_TNIL
2615  |   add NEXT_IDX, NEXT_IDX, #1
2616  |  bxne lr
2617  |  // Skip holes in hash part.
2618  |  add NEXT_RES, NEXT_RES, #sizeof(Node)
2619  |  b <6
2620  |
2621  |9:  // End of iteration. Set the key to nil (not the value).
2622  |  mvn NEXT_TMP0, #0
2623  |   mov NEXT_RES, NEXT_RES_PTR
2624  |  str NEXT_TMP0, NEXT_RES_KEY_IT
2625  |  bx lr
2626  |.endif
2627  |
2628  |//-----------------------------------------------------------------------
2629  |//-- FFI helper functions -----------------------------------------------
2630  |//-----------------------------------------------------------------------
2631  |
2632  |// Handler for callback functions.
2633  |// Saveregs already performed. Callback slot number in [sp], g in r12.
2634  |->vm_ffi_callback:
2635  |.if FFI
2636  |.type CTSTATE, CTState, PC
2637  |  ldr CTSTATE, GL:r12->ctype_state
2638  |   add DISPATCH, r12, #GG_G2DISP
2639  |.if FPU
2640  |  str r4, SAVE_R4
2641  |  add r4, sp, CFRAME_SPACE+4+8*8
2642  |  vstmdb r4!, {d8-d15}
2643  |.endif
2644  |.if HFABI
2645  |  add r12, CTSTATE, #offsetof(CTState, cb.fpr[8])
2646  |.endif
2647  |  strd CARG34, CTSTATE->cb.gpr[2]
2648  |  strd CARG12, CTSTATE->cb.gpr[0]
2649  |.if HFABI
2650  |  vstmdb r12!, {d0-d7}
2651  |.endif
2652  |  ldr CARG4, [sp]
2653  |   add CARG3, sp, #CFRAME_SIZE
2654  |    mov CARG1, CTSTATE
2655  |  lsr CARG4, CARG4, #3
2656  |   str CARG3, CTSTATE->cb.stack
2657  |    mov CARG2, sp
2658  |  str CARG4, CTSTATE->cb.slot
2659  |  str CTSTATE, SAVE_PC		// Any value outside of bytecode is ok.
2660  |  bl extern lj_ccallback_enter	// (CTState *cts, void *cf)
2661  |  // Returns lua_State *.
2662  |  ldr BASE, L:CRET1->base
2663  |    mv_vmstate CARG2, INTERP
2664  |  ldr RC, L:CRET1->top
2665  |    mov MASKR8, #255
2666  |   ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2667  |    mov L, CRET1
2668  |  sub RC, RC, BASE
2669  |    lsl MASKR8, MASKR8, #3		// MASKR8 = 255*8.
2670  |    st_vmstate CARG2
2671  |  ins_callt
2672  |.endif
2673  |
2674  |->cont_ffi_callback:			// Return from FFI callback.
2675  |.if FFI
2676  |  ldr CTSTATE, [DISPATCH, #DISPATCH_GL(ctype_state)]
2677  |   str BASE, L->base
2678  |   str CARG4, L->top
2679  |  str L, CTSTATE->L
2680  |  mov CARG1, CTSTATE
2681  |  mov CARG2, RA
2682  |  bl extern lj_ccallback_leave	// (CTState *cts, TValue *o)
2683  |  ldrd CARG12, CTSTATE->cb.gpr[0]
2684  |.if HFABI
2685  |  vldr d0, CTSTATE->cb.fpr[0]
2686  |.endif
2687  |  b ->vm_leave_unw
2688  |.endif
2689  |
2690  |->vm_ffi_call:			// Call C function via FFI.
2691  |  // Caveat: needs special frame unwinding, see below.
2692  |.if FFI
2693  |  .type CCSTATE, CCallState, r4
2694  |  push {CCSTATE, r5, r11, lr}
2695  |  mov CCSTATE, CARG1
2696  |  ldr CARG1, CCSTATE:CARG1->spadj
2697  |   ldrb CARG2, CCSTATE->nsp
2698  |    add CARG3, CCSTATE, #offsetof(CCallState, stack)
2699  |.if HFABI
2700  |  add RB, CCSTATE, #offsetof(CCallState, fpr[0])
2701  |.endif
2702  |  mov r11, sp
2703  |  sub sp, sp, CARG1			// Readjust stack.
2704  |   subs CARG2, CARG2, #4
2705  |.if HFABI
2706  |  vldm RB, {d0-d7}
2707  |.endif
2708  |    ldr RB, CCSTATE->func
2709  |   bmi >2
2710  |1:  // Copy stack slots.
2711  |  ldr CARG4, [CARG3, CARG2]
2712  |  str CARG4, [sp, CARG2]
2713  |  subs CARG2, CARG2, #4
2714  |  bpl <1
2715  |2:
2716  |  ldrd CARG12, CCSTATE->gpr[0]
2717  |  ldrd CARG34, CCSTATE->gpr[2]
2718  |  blx RB
2719  |  mov sp, r11
2720  |.if HFABI
2721  |  add r12, CCSTATE, #offsetof(CCallState, fpr[4])
2722  |.endif
2723  |  strd CRET1, CCSTATE->gpr[0]
2724  |.if HFABI
2725  |  vstmdb r12!, {d0-d3}
2726  |.endif
2727  |  pop {CCSTATE, r5, r11, pc}
2728  |.endif
2729  |// Note: vm_ffi_call must be the last function in this object file!
2730  |
2731  |//-----------------------------------------------------------------------
2732}
2733
2734/* Generate the code for a single instruction. */
2735static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2736{
2737  int vk = 0;
2738  |=>defop:
2739
2740  switch (op) {
2741
2742  /* -- Comparison ops ---------------------------------------------------- */
2743
2744  /* Remember: all ops branch for a true comparison, fall through otherwise. */
2745
2746  case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2747    |  // RA = src1*8, RC = src2, JMP with RC = target
2748    |   lsl RC, RC, #3
2749    |  ldrd CARG12, [RA, BASE]!
2750    |    ldrh RB, [PC, #2]
2751    |   ldrd CARG34, [RC, BASE]!
2752    |    add PC, PC, #4
2753    |    add RB, PC, RB, lsl #2
2754    |  checktp CARG2, LJ_TISNUM
2755    |  bne >3
2756    |  checktp CARG4, LJ_TISNUM
2757    |  bne >4
2758    |  cmp CARG1, CARG3
2759    if (op == BC_ISLT) {
2760      |  sublt PC, RB, #0x20000
2761    } else if (op == BC_ISGE) {
2762      |  subge PC, RB, #0x20000
2763    } else if (op == BC_ISLE) {
2764      |  suble PC, RB, #0x20000
2765    } else {
2766      |  subgt PC, RB, #0x20000
2767    }
2768    |1:
2769    |  ins_next
2770    |
2771    |3: // CARG12 is not an integer.
2772    |.if FPU
2773    |   vldr d0, [RA]
2774    |  bhi ->vmeta_comp
2775    |  // d0 is a number.
2776    |  checktp CARG4, LJ_TISNUM
2777    |   vldr d1, [RC]
2778    |  blo >5
2779    |  bhi ->vmeta_comp
2780    |  // d0 is a number, CARG3 is an integer.
2781    |  vmov s4, CARG3
2782    |  vcvt.f64.s32 d1, s4
2783    |  b >5
2784    |4:  // CARG1 is an integer, CARG34 is not an integer.
2785    |   vldr d1, [RC]
2786    |  bhi ->vmeta_comp
2787    |  // CARG1 is an integer, d1 is a number.
2788    |  vmov s4, CARG1
2789    |  vcvt.f64.s32 d0, s4
2790    |5:  // d0 and d1 are numbers.
2791    |  vcmp.f64 d0, d1
2792    |  vmrs
2793    |  // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2794    if (op == BC_ISLT) {
2795      |  sublo PC, RB, #0x20000
2796    } else if (op == BC_ISGE) {
2797      |  subhs PC, RB, #0x20000
2798    } else if (op == BC_ISLE) {
2799      |  subls PC, RB, #0x20000
2800    } else {
2801      |  subhi PC, RB, #0x20000
2802    }
2803    |  b <1
2804    |.else
2805    |  bhi ->vmeta_comp
2806    |  // CARG12 is a number.
2807    |  checktp CARG4, LJ_TISNUM
2808    |  movlo RA, RB			// Save RB.
2809    |  blo >5
2810    |  bhi ->vmeta_comp
2811    |  // CARG12 is a number, CARG3 is an integer.
2812    |  mov CARG1, CARG3
2813    |  mov RC, RA
2814    |  mov RA, RB			// Save RB.
2815    |  bl extern __aeabi_i2d
2816    |  mov CARG3, CARG1
2817    |  mov CARG4, CARG2
2818    |  ldrd CARG12, [RC]		// Restore first operand.
2819    |  b >5
2820    |4:  // CARG1 is an integer, CARG34 is not an integer.
2821    |  bhi ->vmeta_comp
2822    |  // CARG1 is an integer, CARG34 is a number.
2823    |  mov RA, RB			// Save RB.
2824    |  bl extern __aeabi_i2d
2825    |  ldrd CARG34, [RC]		// Restore second operand.
2826    |5:  // CARG12 and CARG34 are numbers.
2827    |  bl extern __aeabi_cdcmple
2828    |  // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2829    if (op == BC_ISLT) {
2830      |  sublo PC, RA, #0x20000
2831    } else if (op == BC_ISGE) {
2832      |  subhs PC, RA, #0x20000
2833    } else if (op == BC_ISLE) {
2834      |  subls PC, RA, #0x20000
2835    } else {
2836      |  subhi PC, RA, #0x20000
2837    }
2838    |  b <1
2839    |.endif
2840    break;
2841
2842  case BC_ISEQV: case BC_ISNEV:
2843    vk = op == BC_ISEQV;
2844    |  // RA = src1*8, RC = src2, JMP with RC = target
2845    |   lsl RC, RC, #3
2846    |  ldrd CARG12, [RA, BASE]!
2847    |    ldrh RB, [PC, #2]
2848    |   ldrd CARG34, [RC, BASE]!
2849    |    add PC, PC, #4
2850    |    add RB, PC, RB, lsl #2
2851    |  checktp CARG2, LJ_TISNUM
2852    |  cmnls CARG4, #-LJ_TISNUM
2853    if (vk) {
2854      |  bls ->BC_ISEQN_Z
2855    } else {
2856      |  bls ->BC_ISNEN_Z
2857    }
2858    |  // Either or both types are not numbers.
2859    |.if FFI
2860    |  checktp CARG2, LJ_TCDATA
2861    |  checktpne CARG4, LJ_TCDATA
2862    |  beq ->vmeta_equal_cd
2863    |.endif
2864    |  cmp CARG2, CARG4			// Compare types.
2865    |  bne >2				// Not the same type?
2866    |  checktp CARG2, LJ_TISPRI
2867    |  bhs >1				// Same type and primitive type?
2868    |
2869    |  // Same types and not a primitive type. Compare GCobj or pvalue.
2870    |  cmp CARG1, CARG3
2871    if (vk) {
2872      |  bne >3				// Different GCobjs or pvalues?
2873      |1:  // Branch if same.
2874      |  sub PC, RB, #0x20000
2875      |2:  // Different.
2876      |  ins_next
2877      |3:
2878      |  checktp CARG2, LJ_TISTABUD
2879      |  bhi <2				// Different objects and not table/ud?
2880    } else {
2881      |  beq >1				// Same GCobjs or pvalues?
2882      |  checktp CARG2, LJ_TISTABUD
2883      |  bhi >2				// Different objects and not table/ud?
2884    }
2885    |  // Different tables or userdatas. Need to check __eq metamethod.
2886    |  // Field metatable must be at same offset for GCtab and GCudata!
2887    |  ldr TAB:RA, TAB:CARG1->metatable
2888    |  cmp TAB:RA, #0
2889    if (vk) {
2890      |  beq <2			// No metatable?
2891    } else {
2892      |  beq >2			// No metatable?
2893    }
2894    |  ldrb RA, TAB:RA->nomm
2895    |   mov CARG4, #1-vk		// ne = 0 or 1.
2896    |   mov CARG2, CARG1
2897    |  tst RA, #1<<MM_eq
2898    |  beq ->vmeta_equal		// 'no __eq' flag not set?
2899    if (vk) {
2900      |  b <2
2901    } else {
2902      |2:  // Branch if different.
2903      |  sub PC, RB, #0x20000
2904      |1:  // Same.
2905      |  ins_next
2906    }
2907    break;
2908
2909  case BC_ISEQS: case BC_ISNES:
2910    vk = op == BC_ISEQS;
2911    |  // RA = src*8, RC = str_const (~), JMP with RC = target
2912    |   mvn RC, RC
2913    |  ldrd CARG12, [BASE, RA]
2914    |    ldrh RB, [PC, #2]
2915    |   ldr STR:CARG3, [KBASE, RC, lsl #2]
2916    |    add PC, PC, #4
2917    |    add RB, PC, RB, lsl #2
2918    |  checktp CARG2, LJ_TSTR
2919    |.if FFI
2920    |  bne >7
2921    |  cmp CARG1, CARG3
2922    |.else
2923    |  cmpeq CARG1, CARG3
2924    |.endif
2925    if (vk) {
2926      |  subeq PC, RB, #0x20000
2927      |1:
2928    } else {
2929      |1:
2930      |  subne PC, RB, #0x20000
2931    }
2932    |  ins_next
2933    |
2934    |.if FFI
2935    |7:
2936    |  checktp CARG2, LJ_TCDATA
2937    |  bne <1
2938    |  b ->vmeta_equal_cd
2939    |.endif
2940    break;
2941
2942  case BC_ISEQN: case BC_ISNEN:
2943    vk = op == BC_ISEQN;
2944    |  // RA = src*8, RC = num_const (~), JMP with RC = target
2945    |   lsl RC, RC, #3
2946    |  ldrd CARG12, [RA, BASE]!
2947    |    ldrh RB, [PC, #2]
2948    |   ldrd CARG34, [RC, KBASE]!
2949    |    add PC, PC, #4
2950    |    add RB, PC, RB, lsl #2
2951    if (vk) {
2952      |->BC_ISEQN_Z:
2953    } else {
2954      |->BC_ISNEN_Z:
2955    }
2956    |  checktp CARG2, LJ_TISNUM
2957    |  bne >3
2958    |  checktp CARG4, LJ_TISNUM
2959    |  bne >4
2960    |  cmp CARG1, CARG3
2961    if (vk) {
2962      |  subeq PC, RB, #0x20000
2963      |1:
2964    } else {
2965      |1:
2966      |  subne PC, RB, #0x20000
2967    }
2968    |2:
2969    |  ins_next
2970    |
2971    |3:  // CARG12 is not an integer.
2972    |.if FFI
2973    |  bhi >7
2974    |.else
2975    if (!vk) {
2976      |  subhi PC, RB, #0x20000
2977    }
2978    |  bhi <2
2979    |.endif
2980    |.if FPU
2981    |  checktp CARG4, LJ_TISNUM
2982    |  vmov s4, CARG3
2983    |   vldr d0, [RA]
2984    |  vldrlo d1, [RC]
2985    |  vcvths.f64.s32 d1, s4
2986    |  b >5
2987    |4:  // CARG1 is an integer, d1 is a number.
2988    |  vmov s4, CARG1
2989    |   vldr d1, [RC]
2990    |  vcvt.f64.s32 d0, s4
2991    |5:  // d0 and d1 are numbers.
2992    |  vcmp.f64 d0, d1
2993    |  vmrs
2994    if (vk) {
2995      |  subeq PC, RB, #0x20000
2996    } else {
2997      |  subne PC, RB, #0x20000
2998    }
2999    |  b <2
3000    |.else
3001    |  // CARG12 is a number.
3002    |  checktp CARG4, LJ_TISNUM
3003    |  movlo RA, RB			// Save RB.
3004    |  blo >5
3005    |  // CARG12 is a number, CARG3 is an integer.
3006    |  mov CARG1, CARG3
3007    |  mov RC, RA
3008    |4:  // CARG1 is an integer, CARG34 is a number.
3009    |  mov RA, RB			// Save RB.
3010    |  bl extern __aeabi_i2d
3011    |  ldrd CARG34, [RC]		// Restore other operand.
3012    |5:  // CARG12 and CARG34 are numbers.
3013    |  bl extern __aeabi_cdcmpeq
3014    if (vk) {
3015      |  subeq PC, RA, #0x20000
3016    } else {
3017      |  subne PC, RA, #0x20000
3018    }
3019    |  b <2
3020    |.endif
3021    |
3022    |.if FFI
3023    |7:
3024    |  checktp CARG2, LJ_TCDATA
3025    |  bne <1
3026    |  b ->vmeta_equal_cd
3027    |.endif
3028    break;
3029
3030  case BC_ISEQP: case BC_ISNEP:
3031    vk = op == BC_ISEQP;
3032    |  // RA = src*8, RC = primitive_type (~), JMP with RC = target
3033    |  ldrd CARG12, [BASE, RA]
3034    |   ldrh RB, [PC, #2]
3035    |   add PC, PC, #4
3036    |  mvn RC, RC
3037    |   add RB, PC, RB, lsl #2
3038    |.if FFI
3039    |  checktp CARG2, LJ_TCDATA
3040    |  beq ->vmeta_equal_cd
3041    |.endif
3042    |  cmp CARG2, RC
3043    if (vk) {
3044      |  subeq PC, RB, #0x20000
3045    } else {
3046      |  subne PC, RB, #0x20000
3047    }
3048    |  ins_next
3049    break;
3050
3051  /* -- Unary test and copy ops ------------------------------------------- */
3052
3053  case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
3054    |  // RA = dst*8 or unused, RC = src, JMP with RC = target
3055    |  add RC, BASE, RC, lsl #3
3056    |   ldrh RB, [PC, #2]
3057    |  ldrd CARG12, [RC]
3058    |   add PC, PC, #4
3059    |   add RB, PC, RB, lsl #2
3060    |  checktp CARG2, LJ_TTRUE
3061    if (op == BC_ISTC || op == BC_IST) {
3062      |  subls PC, RB, #0x20000
3063      if (op == BC_ISTC) {
3064	|  strdls CARG12, [BASE, RA]
3065      }
3066    } else {
3067      |  subhi PC, RB, #0x20000
3068      if (op == BC_ISFC) {
3069	|  strdhi CARG12, [BASE, RA]
3070      }
3071    }
3072    |  ins_next
3073    break;
3074
3075  case BC_ISTYPE:
3076    |  // RA = src*8, RC = -type
3077    |  ldrd CARG12, [BASE, RA]
3078    |   ins_next1
3079    |  cmn CARG2, RC
3080    |   ins_next2
3081    |  bne ->vmeta_istype
3082    |   ins_next3
3083    break;
3084  case BC_ISNUM:
3085    |  // RA = src*8, RC = -(TISNUM-1)
3086    |  ldrd CARG12, [BASE, RA]
3087    |   ins_next1
3088    |  checktp CARG2, LJ_TISNUM
3089    |   ins_next2
3090    |  bhs ->vmeta_istype
3091    |   ins_next3
3092    break;
3093
3094  /* -- Unary ops --------------------------------------------------------- */
3095
3096  case BC_MOV:
3097    |  // RA = dst*8, RC = src
3098    |  lsl RC, RC, #3
3099    |   ins_next1
3100    |  ldrd CARG12, [BASE, RC]
3101    |   ins_next2
3102    |  strd CARG12, [BASE, RA]
3103    |   ins_next3
3104    break;
3105  case BC_NOT:
3106    |  // RA = dst*8, RC = src
3107    |  add RC, BASE, RC, lsl #3
3108    |   ins_next1
3109    |  ldr CARG1, [RC, #4]
3110    |   add RA, BASE, RA
3111    |   ins_next2
3112    |  checktp CARG1, LJ_TTRUE
3113    |  mvnls CARG2, #~LJ_TFALSE
3114    |  mvnhi CARG2, #~LJ_TTRUE
3115    |  str CARG2, [RA, #4]
3116    |   ins_next3
3117    break;
3118  case BC_UNM:
3119    |  // RA = dst*8, RC = src
3120    |  lsl RC, RC, #3
3121    |  ldrd CARG12, [BASE, RC]
3122    |   ins_next1
3123    |   ins_next2
3124    |  checktp CARG2, LJ_TISNUM
3125    |  bhi ->vmeta_unm
3126    |  eorne CARG2, CARG2, #0x80000000
3127    |  bne >5
3128    |  rsbs CARG1, CARG1, #0
3129    |  ldrdeq CARG12, >8
3130    |  ldrdvs CARG12, >9
3131    |5:
3132    |  strd CARG12, [BASE, RA]
3133    |   ins_next3
3134    |
3135    |.align 8
3136    |8:
3137    |  .long 0x00000000, 0x80000000	// -0.
3138    |9:
3139    |  .long 0x00000000, 0x41e00000	// 2^31.
3140    break;
3141  case BC_LEN:
3142    |  // RA = dst*8, RC = src
3143    |  lsl RC, RC, #3
3144    |  ldrd CARG12, [BASE, RC]
3145    |  checkstr CARG2, >2
3146    |  ldr CARG1, STR:CARG1->len
3147    |1:
3148    |  mvn CARG2, #~LJ_TISNUM
3149    |   ins_next1
3150    |   ins_next2
3151    |  strd CARG12, [BASE, RA]
3152    |   ins_next3
3153    |2:
3154    |  checktab CARG2, ->vmeta_len
3155#if LJ_52
3156    |  ldr TAB:CARG3, TAB:CARG1->metatable
3157    |  cmp TAB:CARG3, #0
3158    |  bne >9
3159    |3:
3160#endif
3161    |->BC_LEN_Z:
3162    |  .IOS mov RC, BASE
3163    |  bl extern lj_tab_len		// (GCtab *t)
3164    |  // Returns uint32_t (but less than 2^31).
3165    |  .IOS mov BASE, RC
3166    |  b <1
3167#if LJ_52
3168    |9:
3169    |  ldrb CARG4, TAB:CARG3->nomm
3170    |  tst CARG4, #1<<MM_len
3171    |  bne <3				// 'no __len' flag set: done.
3172    |  b ->vmeta_len
3173#endif
3174    break;
3175
3176  /* -- Binary ops -------------------------------------------------------- */
3177
3178    |.macro ins_arithcheck, cond, ncond, target
3179    ||if (vk == 1) {
3180    |   cmn CARG4, #-LJ_TISNUM
3181    |    cmn..cond CARG2, #-LJ_TISNUM
3182    ||} else {
3183    |   cmn CARG2, #-LJ_TISNUM
3184    |    cmn..cond CARG4, #-LJ_TISNUM
3185    ||}
3186    |  b..ncond target
3187    |.endmacro
3188    |.macro ins_arithcheck_int, target
3189    |  ins_arithcheck eq, ne, target
3190    |.endmacro
3191    |.macro ins_arithcheck_num, target
3192    |  ins_arithcheck lo, hs, target
3193    |.endmacro
3194    |
3195    |.macro ins_arithpre
3196    |  decode_RB8 RB, INS
3197    |   decode_RC8 RC, INS
3198    |  // RA = dst*8, RB = src1*8, RC = src2*8 | num_const*8
3199    ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
3200    ||switch (vk) {
3201    ||case 0:
3202    |   .if FPU
3203    |   ldrd CARG12, [RB, BASE]!
3204    |    ldrd CARG34, [RC, KBASE]!
3205    |   .else
3206    |   ldrd CARG12, [BASE, RB]
3207    |    ldrd CARG34, [KBASE, RC]
3208    |   .endif
3209    ||  break;
3210    ||case 1:
3211    |   .if FPU
3212    |   ldrd CARG34, [RB, BASE]!
3213    |    ldrd CARG12, [RC, KBASE]!
3214    |   .else
3215    |   ldrd CARG34, [BASE, RB]
3216    |    ldrd CARG12, [KBASE, RC]
3217    |   .endif
3218    ||  break;
3219    ||default:
3220    |   .if FPU
3221    |   ldrd CARG12, [RB, BASE]!
3222    |    ldrd CARG34, [RC, BASE]!
3223    |   .else
3224    |   ldrd CARG12, [BASE, RB]
3225    |    ldrd CARG34, [BASE, RC]
3226    |   .endif
3227    ||  break;
3228    ||}
3229    |.endmacro
3230    |
3231    |.macro ins_arithpre_fpu, reg1, reg2
3232    |.if FPU
3233    ||if (vk == 1) {
3234    |  vldr reg2, [RB]
3235    |  vldr reg1, [RC]
3236    ||} else {
3237    |  vldr reg1, [RB]
3238    |  vldr reg2, [RC]
3239    ||}
3240    |.endif
3241    |.endmacro
3242    |
3243    |.macro ins_arithpost_fpu, reg
3244    |   ins_next1
3245    |  add RA, BASE, RA
3246    |   ins_next2
3247    |  vstr reg, [RA]
3248    |   ins_next3
3249    |.endmacro
3250    |
3251    |.macro ins_arithfallback, ins
3252    ||switch (vk) {
3253    ||case 0:
3254    |   ins ->vmeta_arith_vn
3255    ||  break;
3256    ||case 1:
3257    |   ins ->vmeta_arith_nv
3258    ||  break;
3259    ||default:
3260    |   ins ->vmeta_arith_vv
3261    ||  break;
3262    ||}
3263    |.endmacro
3264    |
3265    |.macro ins_arithdn, intins, fpins, fpcall
3266    |  ins_arithpre
3267    |.if "intins" ~= "vm_modi" and not FPU
3268    |   ins_next1
3269    |.endif
3270    |  ins_arithcheck_int >5
3271    |.if "intins" == "smull"
3272    |  smull CARG1, RC, CARG3, CARG1
3273    |  cmp RC, CARG1, asr #31
3274    |  ins_arithfallback bne
3275    |.elif "intins" == "vm_modi"
3276    |  movs CARG2, CARG3
3277    |  ins_arithfallback beq
3278    |  bl ->vm_modi
3279    |  mvn CARG2, #~LJ_TISNUM
3280    |.else
3281    |  intins CARG1, CARG1, CARG3
3282    |  ins_arithfallback bvs
3283    |.endif
3284    |4:
3285    |.if "intins" == "vm_modi" or FPU
3286    |   ins_next1
3287    |.endif
3288    |   ins_next2
3289    |  strd CARG12, [BASE, RA]
3290    |   ins_next3
3291    |5:  // FP variant.
3292    |  ins_arithpre_fpu d6, d7
3293    |  ins_arithfallback ins_arithcheck_num
3294    |.if FPU
3295    |.if "intins" == "vm_modi"
3296    |  bl fpcall
3297    |.else
3298    |  fpins d6, d6, d7
3299    |.endif
3300    |  ins_arithpost_fpu d6
3301    |.else
3302    |  bl fpcall
3303    |.if "intins" ~= "vm_modi"
3304    |  ins_next1
3305    |.endif
3306    |  b <4
3307    |.endif
3308    |.endmacro
3309    |
3310    |.macro ins_arithfp, fpins, fpcall
3311    |  ins_arithpre
3312    |.if "fpins" ~= "extern" or HFABI
3313    |  ins_arithpre_fpu d0, d1
3314    |.endif
3315    |  ins_arithfallback ins_arithcheck_num
3316    |.if "fpins" == "extern"
3317    |  .IOS mov RC, BASE
3318    |  bl fpcall
3319    |  .IOS mov BASE, RC
3320    |.elif FPU
3321    |  fpins d0, d0, d1
3322    |.else
3323    |  bl fpcall
3324    |.endif
3325    |.if ("fpins" ~= "extern" or HFABI) and FPU
3326    |  ins_arithpost_fpu d0
3327    |.else
3328    |   ins_next1
3329    |   ins_next2
3330    |  strd CARG12, [BASE, RA]
3331    |   ins_next3
3332    |.endif
3333    |.endmacro
3334
3335  case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
3336    |  ins_arithdn adds, vadd.f64, extern __aeabi_dadd
3337    break;
3338  case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
3339    |  ins_arithdn subs, vsub.f64, extern __aeabi_dsub
3340    break;
3341  case BC_MULVN: case BC_MULNV: case BC_MULVV:
3342    |  ins_arithdn smull, vmul.f64, extern __aeabi_dmul
3343    break;
3344  case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
3345    |  ins_arithfp vdiv.f64, extern __aeabi_ddiv
3346    break;
3347  case BC_MODVN: case BC_MODNV: case BC_MODVV:
3348    |  ins_arithdn vm_modi, vm_mod, ->vm_mod
3349    break;
3350  case BC_POW:
3351    |  // NYI: (partial) integer arithmetic.
3352    |  ins_arithfp extern, extern pow
3353    break;
3354
3355  case BC_CAT:
3356    |  decode_RB8 RC, INS
3357    |   decode_RC8 RB, INS
3358    |  // RA = dst*8, RC = src_start*8, RB = src_end*8  (note: RB/RC swapped!)
3359    |  sub CARG3, RB, RC
3360    |   str BASE, L->base
3361    |  add CARG2, BASE, RB
3362    |->BC_CAT_Z:
3363    |  // RA = dst*8, RC = src_start*8, CARG2 = top-1
3364    |  mov CARG1, L
3365    |   str PC, SAVE_PC
3366    |  lsr CARG3, CARG3, #3
3367    |  bl extern lj_meta_cat		// (lua_State *L, TValue *top, int left)
3368    |  // Returns NULL (finished) or TValue * (metamethod).
3369    |  ldr BASE, L->base
3370    |  cmp CRET1, #0
3371    |  bne ->vmeta_binop
3372    |  ldrd CARG34, [BASE, RC]
3373    |   ins_next1
3374    |   ins_next2
3375    |  strd CARG34, [BASE, RA]		// Copy result to RA.
3376    |   ins_next3
3377    break;
3378
3379  /* -- Constant ops ------------------------------------------------------ */
3380
3381  case BC_KSTR:
3382    |  // RA = dst*8, RC = str_const (~)
3383    |  mvn RC, RC
3384    |   ins_next1
3385    |  ldr CARG1, [KBASE, RC, lsl #2]
3386    |  mvn CARG2, #~LJ_TSTR
3387    |   ins_next2
3388    |  strd CARG12, [BASE, RA]
3389    |   ins_next3
3390    break;
3391  case BC_KCDATA:
3392    |.if FFI
3393    |  // RA = dst*8, RC = cdata_const (~)
3394    |  mvn RC, RC
3395    |   ins_next1
3396    |  ldr CARG1, [KBASE, RC, lsl #2]
3397    |  mvn CARG2, #~LJ_TCDATA
3398    |   ins_next2
3399    |  strd CARG12, [BASE, RA]
3400    |   ins_next3
3401    |.endif
3402    break;
3403  case BC_KSHORT:
3404    |  // RA = dst*8, (RC = int16_literal)
3405    |  mov CARG1, INS, asr #16			// Refetch sign-extended reg.
3406    |  mvn CARG2, #~LJ_TISNUM
3407    |   ins_next1
3408    |   ins_next2
3409    |  strd CARG12, [BASE, RA]
3410    |   ins_next3
3411    break;
3412  case BC_KNUM:
3413    |  // RA = dst*8, RC = num_const
3414    |  lsl RC, RC, #3
3415    |   ins_next1
3416    |  ldrd CARG12, [KBASE, RC]
3417    |   ins_next2
3418    |  strd CARG12, [BASE, RA]
3419    |   ins_next3
3420    break;
3421  case BC_KPRI:
3422    |  // RA = dst*8, RC = primitive_type (~)
3423    |  add RA, BASE, RA
3424    |  mvn RC, RC
3425    |   ins_next1
3426    |   ins_next2
3427    |  str RC, [RA, #4]
3428    |   ins_next3
3429    break;
3430  case BC_KNIL:
3431    |  // RA = base*8, RC = end
3432    |  add RA, BASE, RA
3433    |   add RC, BASE, RC, lsl #3
3434    |  mvn CARG1, #~LJ_TNIL
3435    |  str CARG1, [RA, #4]
3436    |   add RA, RA, #8
3437    |1:
3438    |  str CARG1, [RA, #4]
3439    |  cmp RA, RC
3440    |   add RA, RA, #8
3441    |  blt <1
3442    |  ins_next_
3443    break;
3444
3445  /* -- Upvalue and function ops ------------------------------------------ */
3446
3447  case BC_UGET:
3448    |  // RA = dst*8, RC = uvnum
3449    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3450    |   lsl RC, RC, #2
3451    |   add RC, RC, #offsetof(GCfuncL, uvptr)
3452    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RC]
3453    |  ldr CARG2, UPVAL:CARG2->v
3454    |  ldrd CARG34, [CARG2]
3455    |   ins_next1
3456    |   ins_next2
3457    |  strd CARG34, [BASE, RA]
3458    |   ins_next3
3459    break;
3460  case BC_USETV:
3461    |  // RA = uvnum*8, RC = src
3462    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3463    |   lsr RA, RA, #1
3464    |   add RA, RA, #offsetof(GCfuncL, uvptr)
3465    |    lsl RC, RC, #3
3466    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA]
3467    |    ldrd CARG34, [BASE, RC]
3468    |  ldrb RB, UPVAL:CARG2->marked
3469    |  ldrb RC, UPVAL:CARG2->closed
3470    |    ldr CARG2, UPVAL:CARG2->v
3471    |  tst RB, #LJ_GC_BLACK		// isblack(uv)
3472    |   add RB, CARG4, #-LJ_TISGCV
3473    |  cmpne RC, #0
3474    |   strd CARG34, [CARG2]
3475    |  bne >2				// Upvalue is closed and black?
3476    |1:
3477    |   ins_next
3478    |
3479    |2:  // Check if new value is collectable.
3480    |  cmn RB, #-(LJ_TNUMX - LJ_TISGCV)
3481    |   ldrbhi RC, GCOBJ:CARG3->gch.marked
3482    |  bls <1				// tvisgcv(v)
3483    |    sub CARG1, DISPATCH, #-GG_DISP2G
3484    |   tst RC, #LJ_GC_WHITES
3485    |  // Crossed a write barrier. Move the barrier forward.
3486    |.if IOS
3487    |  beq <1
3488    |  mov RC, BASE
3489    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
3490    |  mov BASE, RC
3491    |.else
3492    |  blne extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
3493    |.endif
3494    |  b <1
3495    break;
3496  case BC_USETS:
3497    |  // RA = uvnum*8, RC = str_const (~)
3498    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3499    |   lsr RA, RA, #1
3500    |   add RA, RA, #offsetof(GCfuncL, uvptr)
3501    |    mvn RC, RC
3502    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA]
3503    |    ldr STR:CARG3, [KBASE, RC, lsl #2]
3504    |  ldrb RB, UPVAL:CARG2->marked
3505    |     ldrb RC, UPVAL:CARG2->closed
3506    |   ldr CARG2, UPVAL:CARG2->v
3507    |    mvn CARG4, #~LJ_TSTR
3508    |  tst RB, #LJ_GC_BLACK		// isblack(uv)
3509    |    ldrb RB, STR:CARG3->marked
3510    |   strd CARG34, [CARG2]
3511    |  bne >2
3512    |1:
3513    |   ins_next
3514    |
3515    |2:  // Check if string is white and ensure upvalue is closed.
3516    |  tst RB, #LJ_GC_WHITES		// iswhite(str)
3517    |  cmpne RC, #0
3518    |   sub CARG1, DISPATCH, #-GG_DISP2G
3519    |  // Crossed a write barrier. Move the barrier forward.
3520    |.if IOS
3521    |  beq <1
3522    |  mov RC, BASE
3523    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
3524    |  mov BASE, RC
3525    |.else
3526    |  blne extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
3527    |.endif
3528    |  b <1
3529    break;
3530  case BC_USETN:
3531    |  // RA = uvnum*8, RC = num_const
3532    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3533    |   lsr RA, RA, #1
3534    |   add RA, RA, #offsetof(GCfuncL, uvptr)
3535    |    lsl RC, RC, #3
3536    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA]
3537    |    ldrd CARG34, [KBASE, RC]
3538    |  ldr CARG2, UPVAL:CARG2->v
3539    |   ins_next1
3540    |   ins_next2
3541    |  strd CARG34, [CARG2]
3542    |   ins_next3
3543    break;
3544  case BC_USETP:
3545    |  // RA = uvnum*8, RC = primitive_type (~)
3546    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3547    |   lsr RA, RA, #1
3548    |   add RA, RA, #offsetof(GCfuncL, uvptr)
3549    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA]
3550    |   mvn RC, RC
3551    |  ldr CARG2, UPVAL:CARG2->v
3552    |   ins_next1
3553    |   ins_next2
3554    |  str RC, [CARG2, #4]
3555    |   ins_next3
3556    break;
3557
3558  case BC_UCLO:
3559    |  // RA = level*8, RC = target
3560    |  ldr CARG3, L->openupval
3561    |   add RC, PC, RC, lsl #2
3562    |   str BASE, L->base
3563    |  cmp CARG3, #0
3564    |   sub PC, RC, #0x20000
3565    |  beq >1
3566    |   mov CARG1, L
3567    |   add CARG2, BASE, RA
3568    |  bl extern lj_func_closeuv	// (lua_State *L, TValue *level)
3569    |  ldr BASE, L->base
3570    |1:
3571    |  ins_next
3572    break;
3573
3574  case BC_FNEW:
3575    |  // RA = dst*8, RC = proto_const (~) (holding function prototype)
3576    |  mvn RC, RC
3577    |   str BASE, L->base
3578    |  ldr CARG2, [KBASE, RC, lsl #2]
3579    |   str PC, SAVE_PC
3580    |  ldr CARG3, [BASE, FRAME_FUNC]
3581    |   mov CARG1, L
3582    |  // (lua_State *L, GCproto *pt, GCfuncL *parent)
3583    |  bl extern lj_func_newL_gc
3584    |  // Returns GCfuncL *.
3585    |  ldr BASE, L->base
3586    |  mvn CARG2, #~LJ_TFUNC
3587    |   ins_next1
3588    |   ins_next2
3589    |  strd CARG12, [BASE, RA]
3590    |   ins_next3
3591    break;
3592
3593  /* -- Table ops --------------------------------------------------------- */
3594
3595  case BC_TNEW:
3596  case BC_TDUP:
3597    |  // RA = dst*8, RC = (hbits|asize) | tab_const (~)
3598    if (op == BC_TDUP) {
3599      |  mvn RC, RC
3600    }
3601    |  ldr CARG3, [DISPATCH, #DISPATCH_GL(gc.total)]
3602    |   ldr CARG4, [DISPATCH, #DISPATCH_GL(gc.threshold)]
3603    |    str BASE, L->base
3604    |    str PC, SAVE_PC
3605    |  cmp CARG3, CARG4
3606    |   mov CARG1, L
3607    |  bhs >5
3608    |1:
3609    if (op == BC_TNEW) {
3610      |  lsl CARG2, RC, #21
3611      |   lsr CARG3, RC, #11
3612      |  asr RC, CARG2, #21
3613      |  lsr CARG2, CARG2, #21
3614      |  cmn RC, #1
3615      |  addeq CARG2, CARG2, #2
3616      |  bl extern lj_tab_new  // (lua_State *L, int32_t asize, uint32_t hbits)
3617      |  // Returns GCtab *.
3618    } else {
3619      |  ldr CARG2, [KBASE, RC, lsl #2]
3620      |  bl extern lj_tab_dup  // (lua_State *L, Table *kt)
3621      |  // Returns GCtab *.
3622    }
3623    |  ldr BASE, L->base
3624    |  mvn CARG2, #~LJ_TTAB
3625    |   ins_next1
3626    |   ins_next2
3627    |  strd CARG12, [BASE, RA]
3628    |   ins_next3
3629    |5:
3630    |  bl extern lj_gc_step_fixtop  // (lua_State *L)
3631    |  mov CARG1, L
3632    |  b <1
3633    break;
3634
3635  case BC_GGET:
3636    |  // RA = dst*8, RC = str_const (~)
3637  case BC_GSET:
3638    |  // RA = dst*8, RC = str_const (~)
3639    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3640    |   mvn RC, RC
3641    |  ldr TAB:CARG1, LFUNC:CARG2->env
3642    |   ldr STR:RC, [KBASE, RC, lsl #2]
3643    if (op == BC_GGET) {
3644      |  b ->BC_TGETS_Z
3645    } else {
3646      |  b ->BC_TSETS_Z
3647    }
3648    break;
3649
3650  case BC_TGETV:
3651    |  decode_RB8 RB, INS
3652    |   decode_RC8 RC, INS
3653    |  // RA = dst*8, RB = table*8, RC = key*8
3654    |  ldrd TAB:CARG12, [BASE, RB]
3655    |   ldrd CARG34, [BASE, RC]
3656    |  checktab CARG2, ->vmeta_tgetv  // STALL: load CARG12.
3657    |   checktp CARG4, LJ_TISNUM	// Integer key?
3658    |  ldreq CARG4, TAB:CARG1->array
3659    |    ldreq CARG2, TAB:CARG1->asize
3660    |   bne >9
3661    |
3662    |  add CARG4, CARG4, CARG3, lsl #3
3663    |    cmp CARG3, CARG2		// In array part?
3664    |  ldrdlo CARG34, [CARG4]
3665    |    bhs ->vmeta_tgetv
3666    |   ins_next1  // Overwrites RB!
3667    |  checktp CARG4, LJ_TNIL
3668    |  beq >5
3669    |1:
3670    |   ins_next2
3671    |  strd CARG34, [BASE, RA]
3672    |   ins_next3
3673    |
3674    |5:  // Check for __index if table value is nil.
3675    |  ldr TAB:CARG2, TAB:CARG1->metatable
3676    |  cmp TAB:CARG2, #0
3677    |  beq <1				// No metatable: done.
3678    |  ldrb CARG2, TAB:CARG2->nomm
3679    |  tst CARG2, #1<<MM_index
3680    |  bne <1				// 'no __index' flag set: done.
3681    |  decode_RB8 RB, INS		// Restore RB.
3682    |  b ->vmeta_tgetv
3683    |
3684    |9:
3685    |  checktp CARG4, LJ_TSTR		// String key?
3686    |   moveq STR:RC, CARG3
3687    |  beq ->BC_TGETS_Z
3688    |  b ->vmeta_tgetv
3689    break;
3690  case BC_TGETS:
3691    |  decode_RB8 RB, INS
3692    |   and RC, RC, #255
3693    |  // RA = dst*8, RB = table*8, RC = str_const (~)
3694    |  ldrd CARG12, [BASE, RB]
3695    |   mvn RC, RC
3696    |   ldr STR:RC, [KBASE, RC, lsl #2]  // STALL: early RC.
3697    |  checktab CARG2, ->vmeta_tgets1
3698    |->BC_TGETS_Z:
3699    |  // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8
3700    |  ldr CARG3, TAB:CARG1->hmask
3701    |   ldr CARG4, STR:RC->sid
3702    |    ldr NODE:INS, TAB:CARG1->node
3703    |     mov TAB:RB, TAB:CARG1
3704    |  and CARG3, CARG3, CARG4			// idx = str->sid & tab->hmask
3705    |  add CARG3, CARG3, CARG3, lsl #1
3706    |    add NODE:INS, NODE:INS, CARG3, lsl #3	// node = tab->node + idx*3*8
3707    |1:
3708    |  ldrd CARG12, NODE:INS->key  // STALL: early NODE:INS.
3709    |   ldrd CARG34, NODE:INS->val
3710    |    ldr NODE:INS, NODE:INS->next
3711    |  checktp CARG2, LJ_TSTR
3712    |  cmpeq CARG1, STR:RC
3713    |  bne >4
3714    |   checktp CARG4, LJ_TNIL
3715    |   beq >5
3716    |3:
3717    |   ins_next1
3718    |   ins_next2
3719    |  strd CARG34, [BASE, RA]
3720    |   ins_next3
3721    |
3722    |4:  // Follow hash chain.
3723    |  cmp NODE:INS, #0
3724    |  bne <1
3725    |  // End of hash chain: key not found, nil result.
3726    |
3727    |5:  // Check for __index if table value is nil.
3728    |  ldr TAB:CARG1, TAB:RB->metatable
3729    |   mov CARG3, #0  // Optional clear of undef. value (during load stall).
3730    |   mvn CARG4, #~LJ_TNIL
3731    |  cmp TAB:CARG1, #0
3732    |  beq <3				// No metatable: done.
3733    |  ldrb CARG2, TAB:CARG1->nomm
3734    |  tst CARG2, #1<<MM_index
3735    |  bne <3				// 'no __index' flag set: done.
3736    |  b ->vmeta_tgets
3737    break;
3738  case BC_TGETB:
3739    |  decode_RB8 RB, INS
3740    |   and RC, RC, #255
3741    |  // RA = dst*8, RB = table*8, RC = index
3742    |  ldrd CARG12, [BASE, RB]
3743    |  checktab CARG2, ->vmeta_tgetb  // STALL: load CARG12.
3744    |   ldr CARG3, TAB:CARG1->asize
3745    |  ldr CARG4, TAB:CARG1->array
3746    |  lsl CARG2, RC, #3
3747    |   cmp RC, CARG3
3748    |  ldrdlo CARG34, [CARG4, CARG2]
3749    |   bhs ->vmeta_tgetb
3750    |   ins_next1  // Overwrites RB!
3751    |  checktp CARG4, LJ_TNIL
3752    |  beq >5
3753    |1:
3754    |   ins_next2
3755    |  strd CARG34, [BASE, RA]
3756    |   ins_next3
3757    |
3758    |5:  // Check for __index if table value is nil.
3759    |  ldr TAB:CARG2, TAB:CARG1->metatable
3760    |  cmp TAB:CARG2, #0
3761    |  beq <1				// No metatable: done.
3762    |  ldrb CARG2, TAB:CARG2->nomm
3763    |  tst CARG2, #1<<MM_index
3764    |  bne <1				// 'no __index' flag set: done.
3765    |  b ->vmeta_tgetb
3766    break;
3767  case BC_TGETR:
3768    |  decode_RB8 RB, INS
3769    |   decode_RC8 RC, INS
3770    |  // RA = dst*8, RB = table*8, RC = key*8
3771    |  ldr TAB:CARG1, [BASE, RB]
3772    |   ldr CARG2, [BASE, RC]
3773    |  ldr CARG4, TAB:CARG1->array
3774    |    ldr CARG3, TAB:CARG1->asize
3775    |  add CARG4, CARG4, CARG2, lsl #3
3776    |    cmp CARG2, CARG3		// In array part?
3777    |    bhs ->vmeta_tgetr
3778    |  ldrd CARG12, [CARG4]
3779    |->BC_TGETR_Z:
3780    |   ins_next1
3781    |   ins_next2
3782    |  strd CARG12, [BASE, RA]
3783    |   ins_next3
3784    break;
3785
3786  case BC_TSETV:
3787    |  decode_RB8 RB, INS
3788    |   decode_RC8 RC, INS
3789    |  // RA = src*8, RB = table*8, RC = key*8
3790    |  ldrd TAB:CARG12, [BASE, RB]
3791    |   ldrd CARG34, [BASE, RC]
3792    |  checktab CARG2, ->vmeta_tsetv  // STALL: load CARG12.
3793    |   checktp CARG4, LJ_TISNUM	// Integer key?
3794    |  ldreq CARG2, TAB:CARG1->array
3795    |    ldreq CARG4, TAB:CARG1->asize
3796    |   bne >9
3797    |
3798    |  add CARG2, CARG2, CARG3, lsl #3
3799    |    cmp CARG3, CARG4		// In array part?
3800    |  ldrlo INS, [CARG2, #4]
3801    |    bhs ->vmeta_tsetv
3802    |   ins_next1  // Overwrites RB!
3803    |  checktp INS, LJ_TNIL
3804    |  ldrb INS, TAB:CARG1->marked
3805    |   ldrd CARG34, [BASE, RA]
3806    |  beq >5
3807    |1:
3808    |  tst INS, #LJ_GC_BLACK		// isblack(table)
3809    |   strd CARG34, [CARG2]
3810    |  bne >7
3811    |2:
3812    |   ins_next2
3813    |   ins_next3
3814    |
3815    |5:  // Check for __newindex if previous value is nil.
3816    |  ldr TAB:RA, TAB:CARG1->metatable
3817    |  cmp TAB:RA, #0
3818    |  beq <1				// No metatable: done.
3819    |  ldrb RA, TAB:RA->nomm
3820    |  tst RA, #1<<MM_newindex
3821    |  bne <1				// 'no __newindex' flag set: done.
3822    |  ldr INS, [PC, #-4]		// Restore RA and RB.
3823    |  decode_RB8 RB, INS
3824    |  decode_RA8 RA, INS
3825    |  b ->vmeta_tsetv
3826    |
3827    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3828    |  barrierback TAB:CARG1, INS, CARG3
3829    |  b <2
3830    |
3831    |9:
3832    |  checktp CARG4, LJ_TSTR		// String key?
3833    |   moveq STR:RC, CARG3
3834    |  beq ->BC_TSETS_Z
3835    |  b ->vmeta_tsetv
3836    break;
3837  case BC_TSETS:
3838    |  decode_RB8 RB, INS
3839    |   and RC, RC, #255
3840    |  // RA = src*8, RB = table*8, RC = str_const (~)
3841    |  ldrd CARG12, [BASE, RB]
3842    |   mvn RC, RC
3843    |   ldr STR:RC, [KBASE, RC, lsl #2]  // STALL: early RC.
3844    |  checktab CARG2, ->vmeta_tsets1
3845    |->BC_TSETS_Z:
3846    |  // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8
3847    |  ldr CARG3, TAB:CARG1->hmask
3848    |   ldr CARG4, STR:RC->sid
3849    |    ldr NODE:INS, TAB:CARG1->node
3850    |     mov TAB:RB, TAB:CARG1
3851    |  and CARG3, CARG3, CARG4			// idx = str->sid & tab->hmask
3852    |  add CARG3, CARG3, CARG3, lsl #1
3853    |   mov CARG4, #0
3854    |    add NODE:INS, NODE:INS, CARG3, lsl #3	// node = tab->node + idx*3*8
3855    |   strb CARG4, TAB:RB->nomm		// Clear metamethod cache.
3856    |1:
3857    |  ldrd CARG12, NODE:INS->key
3858    |   ldr CARG4, NODE:INS->val.it
3859    |    ldr NODE:CARG3, NODE:INS->next
3860    |  checktp CARG2, LJ_TSTR
3861    |  cmpeq CARG1, STR:RC
3862    |  bne >5
3863    |  ldrb CARG2, TAB:RB->marked
3864    |   checktp CARG4, LJ_TNIL		// Key found, but nil value?
3865    |    ldrd CARG34, [BASE, RA]
3866    |   beq >4
3867    |2:
3868    |  tst CARG2, #LJ_GC_BLACK		// isblack(table)
3869    |    strd CARG34, NODE:INS->val
3870    |  bne >7
3871    |3:
3872    |   ins_next
3873    |
3874    |4:  // Check for __newindex if previous value is nil.
3875    |  ldr TAB:CARG1, TAB:RB->metatable
3876    |  cmp TAB:CARG1, #0
3877    |  beq <2				// No metatable: done.
3878    |  ldrb CARG1, TAB:CARG1->nomm
3879    |  tst CARG1, #1<<MM_newindex
3880    |  bne <2				// 'no __newindex' flag set: done.
3881    |  b ->vmeta_tsets
3882    |
3883    |5:  // Follow hash chain.
3884    |  movs NODE:INS, NODE:CARG3
3885    |  bne <1
3886    |  // End of hash chain: key not found, add a new one.
3887    |
3888    |  // But check for __newindex first.
3889    |  ldr TAB:CARG1, TAB:RB->metatable
3890    |   mov CARG3, TMPDp
3891    |   str PC, SAVE_PC
3892    |  cmp TAB:CARG1, #0		// No metatable: continue.
3893    |   str BASE, L->base
3894    |  ldrbne CARG2, TAB:CARG1->nomm
3895    |   mov CARG1, L
3896    |  beq >6
3897    |  tst CARG2, #1<<MM_newindex
3898    |  beq ->vmeta_tsets		// 'no __newindex' flag NOT set: check.
3899    |6:
3900    |  mvn CARG4, #~LJ_TSTR
3901    |   str STR:RC, TMPDlo
3902    |   mov CARG2, TAB:RB
3903    |  str CARG4, TMPDhi
3904    |  bl extern lj_tab_newkey		// (lua_State *L, GCtab *t, TValue *k)
3905    |  // Returns TValue *.
3906    |  ldr BASE, L->base
3907    |  ldrd CARG34, [BASE, RA]
3908    |  strd CARG34, [CRET1]
3909    |  b <3				// No 2nd write barrier needed.
3910    |
3911    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3912    |  barrierback TAB:RB, CARG2, CARG3
3913    |  b <3
3914    break;
3915  case BC_TSETB:
3916    |  decode_RB8 RB, INS
3917    |   and RC, RC, #255
3918    |  // RA = src*8, RB = table*8, RC = index
3919    |  ldrd CARG12, [BASE, RB]
3920    |  checktab CARG2, ->vmeta_tsetb  // STALL: load CARG12.
3921    |   ldr CARG3, TAB:CARG1->asize
3922    |  ldr RB, TAB:CARG1->array
3923    |  lsl CARG2, RC, #3
3924    |   cmp RC, CARG3
3925    |  ldrdlo CARG34, [CARG2, RB]!
3926    |   bhs ->vmeta_tsetb
3927    |   ins_next1  // Overwrites RB!
3928    |  checktp CARG4, LJ_TNIL
3929    |  ldrb INS, TAB:CARG1->marked
3930    |   ldrd CARG34, [BASE, RA]
3931    |  beq >5
3932    |1:
3933    |  tst INS, #LJ_GC_BLACK		// isblack(table)
3934    |    strd CARG34, [CARG2]
3935    |  bne >7
3936    |2:
3937    |   ins_next2
3938    |   ins_next3
3939    |
3940    |5:  // Check for __newindex if previous value is nil.
3941    |  ldr TAB:RA, TAB:CARG1->metatable
3942    |  cmp TAB:RA, #0
3943    |  beq <1				// No metatable: done.
3944    |  ldrb RA, TAB:RA->nomm
3945    |  tst RA, #1<<MM_newindex
3946    |  bne <1				// 'no __newindex' flag set: done.
3947    |  ldr INS, [PC, #-4]		// Restore INS.
3948    |  decode_RA8 RA, INS
3949    |  b ->vmeta_tsetb
3950    |
3951    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3952    |  barrierback TAB:CARG1, INS, CARG3
3953    |  b <2
3954    break;
3955  case BC_TSETR:
3956    |  decode_RB8 RB, INS
3957    |   decode_RC8 RC, INS
3958    |  // RA = src*8, RB = table*8, RC = key*8
3959    |  ldr TAB:CARG2, [BASE, RB]
3960    |   ldr CARG3, [BASE, RC]
3961    |     ldrb INS, TAB:CARG2->marked
3962    |  ldr CARG1, TAB:CARG2->array
3963    |    ldr CARG4, TAB:CARG2->asize
3964    |     tst INS, #LJ_GC_BLACK		// isblack(table)
3965    |  add CARG1, CARG1, CARG3, lsl #3
3966    |     bne >7
3967    |2:
3968    |    cmp CARG3, CARG4		// In array part?
3969    |    bhs ->vmeta_tsetr
3970    |->BC_TSETR_Z:
3971    |  ldrd CARG34, [BASE, RA]
3972    |   ins_next1
3973    |   ins_next2
3974    |  strd CARG34, [CARG1]
3975    |   ins_next3
3976    |
3977    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3978    |  barrierback TAB:CARG2, INS, RB
3979    |  b <2
3980    break;
3981
3982  case BC_TSETM:
3983    |  // RA = base*8 (table at base-1), RC = num_const (start index)
3984    |  add RA, BASE, RA
3985    |1:
3986    |   ldr RB, SAVE_MULTRES
3987    |  ldr TAB:CARG2, [RA, #-8]		// Guaranteed to be a table.
3988    |  ldr CARG1, [KBASE, RC, lsl #3]	// Integer constant is in lo-word.
3989    |   subs RB, RB, #8
3990    |  ldr CARG4, TAB:CARG2->asize
3991    |   beq >4				// Nothing to copy?
3992    |  add CARG3, CARG1, RB, lsr #3
3993    |  cmp CARG3, CARG4
3994    |   ldr CARG4, TAB:CARG2->array
3995    |    add RB, RA, RB
3996    |  bhi >5
3997    |   add INS, CARG4, CARG1, lsl #3
3998    |    ldrb CARG1, TAB:CARG2->marked
3999    |3:  // Copy result slots to table.
4000    |   ldrd CARG34, [RA], #8
4001    |   strd CARG34, [INS], #8
4002    |  cmp RA, RB
4003    |  blo <3
4004    |    tst CARG1, #LJ_GC_BLACK	// isblack(table)
4005    |    bne >7
4006    |4:
4007    |  ins_next
4008    |
4009    |5:  // Need to resize array part.
4010    |   str BASE, L->base
4011    |  mov CARG1, L
4012    |   str PC, SAVE_PC
4013    |  bl extern lj_tab_reasize		// (lua_State *L, GCtab *t, int nasize)
4014    |  // Must not reallocate the stack.
4015    |  .IOS ldr BASE, L->base
4016    |  b <1
4017    |
4018    |7:  // Possible table write barrier for any value. Skip valiswhite check.
4019    |  barrierback TAB:CARG2, CARG1, CARG3
4020    |  b <4
4021    break;
4022
4023  /* -- Calls and vararg handling ----------------------------------------- */
4024
4025  case BC_CALLM:
4026    |  // RA = base*8, (RB = nresults+1,) RC = extra_nargs
4027    |  ldr CARG1, SAVE_MULTRES
4028    |  decode_RC8 NARGS8:RC, INS
4029    |  add NARGS8:RC, NARGS8:RC, CARG1
4030    |  b ->BC_CALL_Z
4031    break;
4032  case BC_CALL:
4033    |  decode_RC8 NARGS8:RC, INS
4034    |  // RA = base*8, (RB = nresults+1,) RC = (nargs+1)*8
4035    |->BC_CALL_Z:
4036    |  mov RB, BASE			// Save old BASE for vmeta_call.
4037    |  ldrd CARG34, [BASE, RA]!
4038    |   sub NARGS8:RC, NARGS8:RC, #8
4039    |   add BASE, BASE, #8
4040    |  checkfunc CARG4, ->vmeta_call
4041    |  ins_call
4042    break;
4043
4044  case BC_CALLMT:
4045    |  // RA = base*8, (RB = 0,) RC = extra_nargs
4046    |  ldr CARG1, SAVE_MULTRES
4047    |  add NARGS8:RC, CARG1, RC, lsl #3
4048    |  b ->BC_CALLT1_Z
4049    break;
4050  case BC_CALLT:
4051    |  lsl NARGS8:RC, RC, #3
4052    |  // RA = base*8, (RB = 0,) RC = (nargs+1)*8
4053    |->BC_CALLT1_Z:
4054    |  ldrd LFUNC:CARG34, [RA, BASE]!
4055    |   sub NARGS8:RC, NARGS8:RC, #8
4056    |   add RA, RA, #8
4057    |  checkfunc CARG4, ->vmeta_callt
4058    |  ldr PC, [BASE, FRAME_PC]
4059    |->BC_CALLT2_Z:
4060    |   mov RB, #0
4061    |   ldrb CARG4, LFUNC:CARG3->ffid
4062    |  tst PC, #FRAME_TYPE
4063    |  bne >7
4064    |1:
4065    |  str LFUNC:CARG3, [BASE, FRAME_FUNC]  // Copy function down, but keep PC.
4066    |  cmp NARGS8:RC, #0
4067    |  beq >3
4068    |2:
4069    |  ldrd CARG12, [RA, RB]
4070    |   add INS, RB, #8
4071    |   cmp INS, NARGS8:RC
4072    |  strd CARG12, [BASE, RB]
4073    |    mov RB, INS
4074    |   bne <2
4075    |3:
4076    |  cmp CARG4, #1			// (> FF_C) Calling a fast function?
4077    |  bhi >5
4078    |4:
4079    |  ins_callt
4080    |
4081    |5:  // Tailcall to a fast function with a Lua frame below.
4082    |  ldr INS, [PC, #-4]
4083    |  decode_RA8 RA, INS
4084    |  sub CARG1, BASE, RA
4085    |  ldr LFUNC:CARG1, [CARG1, #-16]
4086    |  ldr CARG1, LFUNC:CARG1->field_pc
4087    |  ldr KBASE, [CARG1, #PC2PROTO(k)]
4088    |  b <4
4089    |
4090    |7:  // Tailcall from a vararg function.
4091    |  eor PC, PC, #FRAME_VARG
4092    |  tst PC, #FRAME_TYPEP		// Vararg frame below?
4093    |  movne CARG4, #0			// Clear ffid if no Lua function below.
4094    |  bne <1
4095    |  sub BASE, BASE, PC
4096    |  ldr PC, [BASE, FRAME_PC]
4097    |  tst PC, #FRAME_TYPE
4098    |  movne CARG4, #0			// Clear ffid if no Lua function below.
4099    |  b <1
4100    break;
4101
4102  case BC_ITERC:
4103    |  // RA = base*8, (RB = nresults+1, RC = nargs+1 (2+1))
4104    |  add RA, BASE, RA
4105    |   mov RB, BASE			// Save old BASE for vmeta_call.
4106    |  ldrd CARG34, [RA, #-16]
4107    |   ldrd CARG12, [RA, #-8]
4108    |    add BASE, RA, #8
4109    |  strd CARG34, [RA, #8]		// Copy state.
4110    |   strd CARG12, [RA, #16]		// Copy control var.
4111    |  // STALL: locked CARG34.
4112    |  ldrd LFUNC:CARG34, [RA, #-24]
4113    |    mov NARGS8:RC, #16		// Iterators get 2 arguments.
4114    |  // STALL: load CARG34.
4115    |  strd LFUNC:CARG34, [RA]		// Copy callable.
4116    |  checkfunc CARG4, ->vmeta_call
4117    |  ins_call
4118    break;
4119
4120  case BC_ITERN:
4121    |.if JIT
4122    |  hotloop
4123    |.endif
4124    |->vm_IITERN:
4125    |  // RA = base*8, (RB = nresults+1, RC = nargs+1 (2+1))
4126    |  add RA, BASE, RA
4127    |  ldr TAB:RB, [RA, #-16]
4128    |  ldr CARG1, [RA, #-8]		// Get index from control var.
4129    |  ldr INS, TAB:RB->asize
4130    |   ldr CARG2, TAB:RB->array
4131    |    add PC, PC, #4
4132    |1:  // Traverse array part.
4133    |  subs RC, CARG1, INS
4134    |   add CARG3, CARG2, CARG1, lsl #3
4135    |  bhs >5				// Index points after array part?
4136    |   ldrd CARG34, [CARG3]
4137    |   checktp CARG4, LJ_TNIL
4138    |   addeq CARG1, CARG1, #1		// Skip holes in array part.
4139    |   beq <1
4140    |  ldrh RC, [PC, #-2]
4141    |   mvn CARG2, #~LJ_TISNUM
4142    |    strd CARG34, [RA, #8]
4143    |  add RC, PC, RC, lsl #2
4144    |    add RB, CARG1, #1
4145    |   strd CARG12, [RA]
4146    |  sub PC, RC, #0x20000
4147    |    str RB, [RA, #-8]		// Update control var.
4148    |3:
4149    |  ins_next
4150    |
4151    |5:  // Traverse hash part.
4152    |  ldr CARG4, TAB:RB->hmask
4153    |   ldr NODE:RB, TAB:RB->node
4154    |6:
4155    |   add CARG1, RC, RC, lsl #1
4156    |  cmp RC, CARG4			// End of iteration? Branch to ITERL+1.
4157    |   add NODE:CARG3, NODE:RB, CARG1, lsl #3  // node = tab->node + idx*3*8
4158    |  bhi <3
4159    |   ldrd CARG12, NODE:CARG3->val
4160    |   checktp CARG2, LJ_TNIL
4161    |   add RC, RC, #1
4162    |   beq <6				// Skip holes in hash part.
4163    |  ldrh RB, [PC, #-2]
4164    |   add RC, RC, INS
4165    |    ldrd CARG34, NODE:CARG3->key
4166    |   str RC, [RA, #-8]		// Update control var.
4167    |   strd CARG12, [RA, #8]
4168    |  add RC, PC, RB, lsl #2
4169    |  sub PC, RC, #0x20000
4170    |    strd CARG34, [RA]
4171    |  b <3
4172    break;
4173
4174  case BC_ISNEXT:
4175    |  // RA = base*8, RC = target (points to ITERN)
4176    |  add RA, BASE, RA
4177    |     add RC, PC, RC, lsl #2
4178    |  ldrd CFUNC:CARG12, [RA, #-24]
4179    |   ldr CARG3, [RA, #-12]
4180    |    ldr CARG4, [RA, #-4]
4181    |  checktp CARG2, LJ_TFUNC
4182    |  ldrbeq CARG1, CFUNC:CARG1->ffid
4183    |   checktpeq CARG3, LJ_TTAB
4184    |    checktpeq CARG4, LJ_TNIL
4185    |  cmpeq CARG1, #FF_next_N
4186    |     subeq PC, RC, #0x20000
4187    |  bne >5
4188    |   ins_next1
4189    |   ins_next2
4190    |  mov CARG1, #0
4191    |  mvn CARG2, #~LJ_KEYINDEX
4192    |  strd CARG1, [RA, #-8]		// Initialize control var.
4193    |1:
4194    |   ins_next3
4195    |5:  // Despecialize bytecode if any of the checks fail.
4196    |  mov CARG1, #BC_JMP
4197    |   mov OP, #BC_ITERC
4198    |  strb CARG1, [PC, #-4]
4199    |   sub PC, RC, #0x20000
4200    |.if JIT
4201    |   ldrb CARG1, [PC]
4202    |   cmp CARG1, #BC_ITERN
4203    |   bne >6
4204    |.endif
4205    |   strb OP, [PC]			// Subsumes ins_next1.
4206    |   ins_next2
4207    |  b <1
4208    |.if JIT
4209    |6:  // Unpatch JLOOP.
4210    |  ldr CARG1, [DISPATCH, #DISPATCH_J(trace)]
4211    |  ldrh CARG2, [PC, #2]
4212    |  ldr TRACE:CARG1, [CARG1, CARG2, lsl #2]
4213    |  // Subsumes ins_next1 and ins_next2.
4214    |  ldr INS, TRACE:CARG1->startins
4215    |  bic INS, INS, #0xff
4216    |  orr INS, INS, OP
4217    |  str INS, [PC], #4
4218    |  b <1
4219    |.endif
4220    break;
4221
4222  case BC_VARG:
4223    |  decode_RB8 RB, INS
4224    |   decode_RC8 RC, INS
4225    |  // RA = base*8, RB = (nresults+1)*8, RC = numparams*8
4226    |  ldr CARG1, [BASE, FRAME_PC]
4227    |  add RC, BASE, RC
4228    |   add RA, BASE, RA
4229    |  add RC, RC, #FRAME_VARG
4230    |   add CARG4, RA, RB
4231    |  sub CARG3, BASE, #8		// CARG3 = vtop
4232    |  sub RC, RC, CARG1		// RC = vbase
4233    |  // Note: RC may now be even _above_ BASE if nargs was < numparams.
4234    |  cmp RB, #0
4235    |   sub CARG1, CARG3, RC
4236    |  beq >5				// Copy all varargs?
4237    |   sub CARG4, CARG4, #16
4238    |1:  // Copy vararg slots to destination slots.
4239    |  cmp RC, CARG3
4240    |  ldrdlo CARG12, [RC], #8
4241    |  mvnhs CARG2, #~LJ_TNIL
4242    |   cmp RA, CARG4
4243    |  strd CARG12, [RA], #8
4244    |   blo <1
4245    |2:
4246    |  ins_next
4247    |
4248    |5:  // Copy all varargs.
4249    |  ldr CARG4, L->maxstack
4250    |   cmp CARG1, #0
4251    |   movle RB, #8			// MULTRES = (0+1)*8
4252    |   addgt RB, CARG1, #8
4253    |  add CARG2, RA, CARG1
4254    |   str RB, SAVE_MULTRES
4255    |   ble <2
4256    |  cmp CARG2, CARG4
4257    |  bhi >7
4258    |6:
4259    |   ldrd CARG12, [RC], #8
4260    |   strd CARG12, [RA], #8
4261    |  cmp RC, CARG3
4262    |  blo <6
4263    |  b <2
4264    |
4265    |7:  // Grow stack for varargs.
4266    |  lsr CARG2, CARG1, #3
4267    |   str RA, L->top
4268    |  mov CARG1, L
4269    |   str BASE, L->base
4270    |  sub RC, RC, BASE			// Need delta, because BASE may change.
4271    |   str PC, SAVE_PC
4272    |  sub RA, RA, BASE
4273    |  bl extern lj_state_growstack	// (lua_State *L, int n)
4274    |  ldr BASE, L->base
4275    |  add RA, BASE, RA
4276    |  add RC, BASE, RC
4277    |  sub CARG3, BASE, #8
4278    |  b <6
4279    break;
4280
4281  /* -- Returns ----------------------------------------------------------- */
4282
4283  case BC_RETM:
4284    |  // RA = results*8, RC = extra results
4285    |  ldr CARG1, SAVE_MULTRES
4286    |   ldr PC, [BASE, FRAME_PC]
4287    |    add RA, BASE, RA
4288    |  add RC, CARG1, RC, lsl #3
4289    |  b ->BC_RETM_Z
4290    break;
4291
4292  case BC_RET:
4293    |  // RA = results*8, RC = nresults+1
4294    |  ldr PC, [BASE, FRAME_PC]
4295    |   lsl RC, RC, #3
4296    |    add RA, BASE, RA
4297    |->BC_RETM_Z:
4298    |   str RC, SAVE_MULTRES
4299    |1:
4300    |  ands CARG1, PC, #FRAME_TYPE
4301    |   eor CARG2, PC, #FRAME_VARG
4302    |  bne ->BC_RETV2_Z
4303    |
4304    |->BC_RET_Z:
4305    |  // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
4306    |  ldr INS, [PC, #-4]
4307    |  subs CARG4, RC, #8
4308    |   sub CARG3, BASE, #8
4309    |  beq >3
4310    |2:
4311    |  ldrd CARG12, [RA], #8
4312    |   add BASE, BASE, #8
4313    |   subs CARG4, CARG4, #8
4314    |  strd CARG12, [BASE, #-16]
4315    |   bne <2
4316    |3:
4317    |  decode_RA8 RA, INS
4318    |  sub CARG4, CARG3, RA
4319    |   decode_RB8 RB, INS
4320    |  ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
4321    |5:
4322    |  cmp RB, RC			// More results expected?
4323    |  bhi >6
4324    |  mov BASE, CARG4
4325    |  ldr CARG2, LFUNC:CARG1->field_pc
4326    |   ins_next1
4327    |   ins_next2
4328    |  ldr KBASE, [CARG2, #PC2PROTO(k)]
4329    |   ins_next3
4330    |
4331    |6:  // Fill up results with nil.
4332    |  mvn CARG2, #~LJ_TNIL
4333    |  add BASE, BASE, #8
4334    |   add RC, RC, #8
4335    |  str CARG2, [BASE, #-12]
4336    |  b <5
4337    |
4338    |->BC_RETV1_Z:  // Non-standard return case.
4339    |  add RA, BASE, RA
4340    |->BC_RETV2_Z:
4341    |  tst CARG2, #FRAME_TYPEP
4342    |  bne ->vm_return
4343    |  // Return from vararg function: relocate BASE down.
4344    |  sub BASE, BASE, CARG2
4345    |  ldr PC, [BASE, FRAME_PC]
4346    |  b <1
4347    break;
4348
4349  case BC_RET0: case BC_RET1:
4350    |  // RA = results*8, RC = nresults+1
4351    |  ldr PC, [BASE, FRAME_PC]
4352    |   lsl RC, RC, #3
4353    |   str RC, SAVE_MULTRES
4354    |  ands CARG1, PC, #FRAME_TYPE
4355    |   eor CARG2, PC, #FRAME_VARG
4356    |   ldreq INS, [PC, #-4]
4357    |  bne ->BC_RETV1_Z
4358    if (op == BC_RET1) {
4359      |  ldrd CARG12, [BASE, RA]
4360    }
4361    |  sub CARG4, BASE, #8
4362    |   decode_RA8 RA, INS
4363    if (op == BC_RET1) {
4364      |  strd CARG12, [CARG4]
4365    }
4366    |  sub BASE, CARG4, RA
4367    |   decode_RB8 RB, INS
4368    |  ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
4369    |5:
4370    |  cmp RB, RC
4371    |  bhi >6
4372    |  ldr CARG2, LFUNC:CARG1->field_pc
4373    |   ins_next1
4374    |   ins_next2
4375    |  ldr KBASE, [CARG2, #PC2PROTO(k)]
4376    |   ins_next3
4377    |
4378    |6:  // Fill up results with nil.
4379    |  sub CARG2, CARG4, #4
4380    |  mvn CARG3, #~LJ_TNIL
4381    |  str CARG3, [CARG2, RC]
4382    |  add RC, RC, #8
4383    |  b <5
4384    break;
4385
4386  /* -- Loops and branches ------------------------------------------------ */
4387
4388  |.define FOR_IDX,  [RA];      .define FOR_TIDX,  [RA, #4]
4389  |.define FOR_STOP, [RA, #8];  .define FOR_TSTOP, [RA, #12]
4390  |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
4391  |.define FOR_EXT,  [RA, #24]; .define FOR_TEXT,  [RA, #28]
4392
4393  case BC_FORL:
4394    |.if JIT
4395    |  hotloop
4396    |.endif
4397    |  // Fall through. Assumes BC_IFORL follows.
4398    break;
4399
4400  case BC_JFORI:
4401  case BC_JFORL:
4402#if !LJ_HASJIT
4403    break;
4404#endif
4405  case BC_FORI:
4406  case BC_IFORL:
4407    |  // RA = base*8, RC = target (after end of loop or start of loop)
4408    vk = (op == BC_IFORL || op == BC_JFORL);
4409    |  ldrd CARG12, [RA, BASE]!
4410    if (op != BC_JFORL) {
4411      |   add RC, PC, RC, lsl #2
4412    }
4413    if (!vk) {
4414      |  ldrd CARG34, FOR_STOP
4415      |   checktp CARG2, LJ_TISNUM
4416      |  ldr RB, FOR_TSTEP
4417      |   bne >5
4418      |  checktp CARG4, LJ_TISNUM
4419      |   ldr CARG4, FOR_STEP
4420      |  checktpeq RB, LJ_TISNUM
4421      |  bne ->vmeta_for
4422      |  cmp CARG4, #0
4423      |  blt >4
4424      |  cmp CARG1, CARG3
4425    } else {
4426      |  ldrd CARG34, FOR_STEP
4427      |   checktp CARG2, LJ_TISNUM
4428      |   bne >5
4429      |  adds CARG1, CARG1, CARG3
4430      |   ldr CARG4, FOR_STOP
4431      if (op == BC_IFORL) {
4432	|  addvs RC, PC, #0x20000		// Overflow: prevent branch.
4433      } else {
4434	|  bvs >2				// Overflow: do not enter mcode.
4435      }
4436      |  cmp CARG3, #0
4437      |  blt >4
4438      |  cmp CARG1, CARG4
4439    }
4440    |1:
4441    if (op == BC_FORI) {
4442      |  subgt PC, RC, #0x20000
4443    } else if (op == BC_JFORI) {
4444      |  sub PC, RC, #0x20000
4445      |  ldrhle RC, [PC, #-2]
4446    } else if (op == BC_IFORL) {
4447      |  suble PC, RC, #0x20000
4448    }
4449    if (vk) {
4450      |  strd CARG12, FOR_IDX
4451    }
4452    |2:
4453    |   ins_next1
4454    |   ins_next2
4455    |  strd CARG12, FOR_EXT
4456    if (op == BC_JFORI || op == BC_JFORL) {
4457      |  ble =>BC_JLOOP
4458    }
4459    |3:
4460    |   ins_next3
4461    |
4462    |4:  // Invert check for negative step.
4463    if (!vk) {
4464      |  cmp CARG3, CARG1
4465    } else {
4466      |  cmp CARG4, CARG1
4467    }
4468    |  b <1
4469    |
4470    |5:  // FP loop.
4471    if (!vk) {
4472      |  cmnlo CARG4, #-LJ_TISNUM
4473      |  cmnlo RB, #-LJ_TISNUM
4474      |  bhs ->vmeta_for
4475      |.if FPU
4476      |  vldr d0, FOR_IDX
4477      |  vldr d1, FOR_STOP
4478      |  cmp RB, #0
4479      |  vstr d0, FOR_EXT
4480      |.else
4481      |  cmp RB, #0
4482      |   strd CARG12, FOR_EXT
4483      |  blt >8
4484      |.endif
4485    } else {
4486      |.if FPU
4487      |  vldr d0, FOR_IDX
4488      |  vldr d2, FOR_STEP
4489      |  vldr d1, FOR_STOP
4490      |  cmp CARG4, #0
4491      |  vadd.f64 d0, d0, d2
4492      |.else
4493      |  cmp CARG4, #0
4494      |  blt >8
4495      |  bl extern __aeabi_dadd
4496      |   strd CARG12, FOR_IDX
4497      |  ldrd CARG34, FOR_STOP
4498      |   strd CARG12, FOR_EXT
4499      |.endif
4500    }
4501    |6:
4502    |.if FPU
4503    |  vcmpge.f64 d0, d1
4504    |  vcmplt.f64 d1, d0
4505    |  vmrs
4506    |.else
4507    |  bl extern __aeabi_cdcmple
4508    |.endif
4509    if (vk) {
4510      |.if FPU
4511      |  vstr d0, FOR_IDX
4512      |  vstr d0, FOR_EXT
4513      |.endif
4514    }
4515    if (op == BC_FORI) {
4516      |  subhi PC, RC, #0x20000
4517    } else if (op == BC_JFORI) {
4518      |  sub PC, RC, #0x20000
4519      |  ldrhls RC, [PC, #-2]
4520      |  bls =>BC_JLOOP
4521    } else if (op == BC_IFORL) {
4522      |  subls PC, RC, #0x20000
4523    } else {
4524      |  bls =>BC_JLOOP
4525    }
4526    |  ins_next1
4527    |  ins_next2
4528    |  b <3
4529    |
4530    |.if not FPU
4531    |8:  // Invert check for negative step.
4532    if (vk) {
4533      |  bl extern __aeabi_dadd
4534      |  strd CARG12, FOR_IDX
4535      |  strd CARG12, FOR_EXT
4536    }
4537    |  mov CARG3, CARG1
4538    |  mov CARG4, CARG2
4539    |  ldrd CARG12, FOR_STOP
4540    |  b <6
4541    |.endif
4542    break;
4543
4544  case BC_ITERL:
4545    |.if JIT
4546    |  hotloop
4547    |.endif
4548    |  // Fall through. Assumes BC_IITERL follows.
4549    break;
4550
4551  case BC_JITERL:
4552#if !LJ_HASJIT
4553    break;
4554#endif
4555  case BC_IITERL:
4556    |  // RA = base*8, RC = target
4557    |  ldrd CARG12, [RA, BASE]!
4558    if (op == BC_JITERL) {
4559      |  cmn CARG2, #-LJ_TNIL		// Stop if iterator returned nil.
4560      |  strdne CARG12, [RA, #-8]
4561      |  bne =>BC_JLOOP
4562    } else {
4563      |   add RC, PC, RC, lsl #2
4564      |  // STALL: load CARG12.
4565      |  cmn CARG2, #-LJ_TNIL		// Stop if iterator returned nil.
4566      |  subne PC, RC, #0x20000		// Otherwise save control var + branch.
4567      |  strdne CARG12, [RA, #-8]
4568    }
4569    |  ins_next
4570    break;
4571
4572  case BC_LOOP:
4573    |  // RA = base*8, RC = target (loop extent)
4574    |  // Note: RA/RC is only used by trace recorder to determine scope/extent
4575    |  // This opcode does NOT jump, it's only purpose is to detect a hot loop.
4576    |.if JIT
4577    |  hotloop
4578    |.endif
4579    |  // Fall through. Assumes BC_ILOOP follows.
4580    break;
4581
4582  case BC_ILOOP:
4583    |  // RA = base*8, RC = target (loop extent)
4584    |  ins_next
4585    break;
4586
4587  case BC_JLOOP:
4588    |.if JIT
4589    |  // RA = base (ignored), RC = traceno
4590    |  ldr CARG1, [DISPATCH, #DISPATCH_J(trace)]
4591    |   mov CARG2, #0  // Traces on ARM don't store the trace number, so use 0.
4592    |  ldr TRACE:RC, [CARG1, RC, lsl #2]
4593    |   st_vmstate CARG2
4594    |  ldr RA, TRACE:RC->mcode
4595    |   str BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
4596    |   str L, [DISPATCH, #DISPATCH_GL(tmpbuf.L)]
4597    |  bx RA
4598    |.endif
4599    break;
4600
4601  case BC_JMP:
4602    |  // RA = base*8 (only used by trace recorder), RC = target
4603    |  add RC, PC, RC, lsl #2
4604    |  sub PC, RC, #0x20000
4605    |  ins_next
4606    break;
4607
4608  /* -- Function headers -------------------------------------------------- */
4609
4610  case BC_FUNCF:
4611    |.if JIT
4612    |  hotcall
4613    |.endif
4614  case BC_FUNCV:  /* NYI: compiled vararg functions. */
4615    |  // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
4616    break;
4617
4618  case BC_JFUNCF:
4619#if !LJ_HASJIT
4620    break;
4621#endif
4622  case BC_IFUNCF:
4623    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
4624    |  ldr CARG1, L->maxstack
4625    |   ldrb CARG2, [PC, #-4+PC2PROTO(numparams)]
4626    |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
4627    |  cmp RA, CARG1
4628    |  bhi ->vm_growstack_l
4629    if (op != BC_JFUNCF) {
4630      |  ins_next1
4631      |  ins_next2
4632    }
4633    |2:
4634    |  cmp NARGS8:RC, CARG2, lsl #3	// Check for missing parameters.
4635    |   mvn CARG4, #~LJ_TNIL
4636    |  blo >3
4637    if (op == BC_JFUNCF) {
4638      |  decode_RD RC, INS
4639      |  b =>BC_JLOOP
4640    } else {
4641      |  ins_next3
4642    }
4643    |
4644    |3:  // Clear missing parameters.
4645    |  strd CARG34, [BASE, NARGS8:RC]
4646    |  add NARGS8:RC, NARGS8:RC, #8
4647    |  b <2
4648    break;
4649
4650  case BC_JFUNCV:
4651#if !LJ_HASJIT
4652    break;
4653#endif
4654    |  NYI  // NYI: compiled vararg functions
4655    break;  /* NYI: compiled vararg functions. */
4656
4657  case BC_IFUNCV:
4658    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
4659    |  ldr CARG1, L->maxstack
4660    |   add CARG4, BASE, RC
4661    |  add RA, RA, RC
4662    |   str LFUNC:CARG3, [CARG4]	// Store copy of LFUNC.
4663    |   add CARG2, RC, #8+FRAME_VARG
4664    |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
4665    |  cmp RA, CARG1
4666    |   str CARG2, [CARG4, #4]		// Store delta + FRAME_VARG.
4667    |  bhs ->vm_growstack_l
4668    |  ldrb RB, [PC, #-4+PC2PROTO(numparams)]
4669    |   mov RA, BASE
4670    |   mov RC, CARG4
4671    |  cmp RB, #0
4672    |   add BASE, CARG4, #8
4673    |  beq >3
4674    |  mvn CARG3, #~LJ_TNIL
4675    |1:
4676    |  cmp RA, RC			// Less args than parameters?
4677    |   ldrdlo CARG12, [RA], #8
4678    |   movhs CARG2, CARG3
4679    |    strlo CARG3, [RA, #-4]		// Clear old fixarg slot (help the GC).
4680    |2:
4681    |  subs RB, RB, #1
4682    |   strd CARG12, [CARG4, #8]!
4683    |  bne <1
4684    |3:
4685    |  ins_next
4686    break;
4687
4688  case BC_FUNCC:
4689  case BC_FUNCCW:
4690    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
4691    if (op == BC_FUNCC) {
4692      |  ldr CARG4, CFUNC:CARG3->f
4693    } else {
4694      |  ldr CARG4, [DISPATCH, #DISPATCH_GL(wrapf)]
4695    }
4696    |   add CARG2, RA, NARGS8:RC
4697    |   ldr CARG1, L->maxstack
4698    |  add RC, BASE, NARGS8:RC
4699    |    str BASE, L->base
4700    |   cmp CARG2, CARG1
4701    |  str RC, L->top
4702    if (op == BC_FUNCCW) {
4703      |  ldr CARG2, CFUNC:CARG3->f
4704    }
4705    |    mv_vmstate CARG3, C
4706    |  mov CARG1, L
4707    |   bhi ->vm_growstack_c		// Need to grow stack.
4708    |    st_vmstate CARG3
4709    |  blx CARG4			// (lua_State *L [, lua_CFunction f])
4710    |  // Returns nresults.
4711    |  ldr BASE, L->base
4712    |    mv_vmstate CARG3, INTERP
4713    |   ldr CRET2, L->top
4714    |    str L, [DISPATCH, #DISPATCH_GL(cur_L)]
4715    |   lsl RC, CRET1, #3
4716    |    st_vmstate CARG3
4717    |  ldr PC, [BASE, FRAME_PC]
4718    |   sub RA, CRET2, RC		// RA = L->top - nresults*8
4719    |  b ->vm_returnc
4720    break;
4721
4722  /* ---------------------------------------------------------------------- */
4723
4724  default:
4725    fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
4726    exit(2);
4727    break;
4728  }
4729}
4730
4731static int build_backend(BuildCtx *ctx)
4732{
4733  int op;
4734
4735  dasm_growpc(Dst, BC__MAX);
4736
4737  build_subroutines(ctx);
4738
4739  |.code_op
4740  for (op = 0; op < BC__MAX; op++)
4741    build_ins(ctx, (BCOp)op, op);
4742
4743  return BC__MAX;
4744}
4745
4746/* Emit pseudo frame-info for all assembler functions. */
4747static void emit_asm_debug(BuildCtx *ctx)
4748{
4749  int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
4750  int i;
4751  switch (ctx->mode) {
4752  case BUILD_elfasm:
4753    fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
4754    fprintf(ctx->fp,
4755	".Lframe0:\n"
4756	"\t.long .LECIE0-.LSCIE0\n"
4757	".LSCIE0:\n"
4758	"\t.long 0xffffffff\n"
4759	"\t.byte 0x1\n"
4760	"\t.string \"\"\n"
4761	"\t.uleb128 0x1\n"
4762	"\t.sleb128 -4\n"
4763	"\t.byte 0xe\n"				/* Return address is in lr. */
4764	"\t.byte 0xc\n\t.uleb128 0xd\n\t.uleb128 0\n"	/* def_cfa sp */
4765	"\t.align 2\n"
4766	".LECIE0:\n\n");
4767    fprintf(ctx->fp,
4768	".LSFDE0:\n"
4769	"\t.long .LEFDE0-.LASFDE0\n"
4770	".LASFDE0:\n"
4771	"\t.long .Lframe0\n"
4772	"\t.long .Lbegin\n"
4773	"\t.long %d\n"
4774	"\t.byte 0xe\n\t.uleb128 %d\n"		/* def_cfa_offset */
4775	"\t.byte 0x8e\n\t.uleb128 1\n",		/* offset lr */
4776	fcofs, CFRAME_SIZE);
4777    for (i = 11; i >= (LJ_ARCH_HASFPU ? 5 : 4); i--)  /* offset r4-r11 */
4778      fprintf(ctx->fp, "\t.byte %d\n\t.uleb128 %d\n", 0x80+i, 2+(11-i));
4779#if LJ_ARCH_HASFPU
4780    for (i = 15; i >= 8; i--)  /* offset d8-d15 */
4781      fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 %d, %d\n",
4782	64+2*i, 10+2*(15-i));
4783    fprintf(ctx->fp, "\t.byte 0x84\n\t.uleb128 %d\n", 25);  /* offset r4 */
4784#endif
4785    fprintf(ctx->fp,
4786	"\t.align 2\n"
4787	".LEFDE0:\n\n");
4788#if LJ_HASFFI
4789    fprintf(ctx->fp,
4790	".LSFDE1:\n"
4791	"\t.long .LEFDE1-.LASFDE1\n"
4792	".LASFDE1:\n"
4793	"\t.long .Lframe0\n"
4794	"\t.long lj_vm_ffi_call\n"
4795	"\t.long %d\n"
4796	"\t.byte 0xe\n\t.uleb128 16\n"		/* def_cfa_offset */
4797	"\t.byte 0x8e\n\t.uleb128 1\n"		/* offset lr */
4798	"\t.byte 0x8b\n\t.uleb128 2\n"		/* offset r11 */
4799	"\t.byte 0x85\n\t.uleb128 3\n"		/* offset r5 */
4800	"\t.byte 0x84\n\t.uleb128 4\n"		/* offset r4 */
4801	"\t.byte 0xd\n\t.uleb128 0xb\n"		/* def_cfa_register r11 */
4802	"\t.align 2\n"
4803	".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4804#endif
4805    break;
4806  default:
4807    break;
4808  }
4809}
4810