luna - luajit/src/vm

   1|// Low-level VM code for ARM64 CPUs.
   2|// Bytecode interpreter, fast functions and helper functions.
   3|// Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4|
   5|.arch arm64
   6|.section code_op, code_sub
   7|
   8|.actionlist build_actionlist
   9|.globals GLOB_
  10|.globalnames globnames
  11|.externnames extnames
  12|
  13|// Note: The ragged indentation of the instructions is intentional.
  14|//       The starting columns indicate data dependencies.
  15|
  16|//-----------------------------------------------------------------------
  17|
  18|// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
  19|//
  20|// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
  21|// x18 is reserved on most platforms. Don't use it, save it or restore it.
  22|// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
  23|// depending on the instruction.
  24|// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
  25|//
  26|// x0-x7/v0-v7 hold parameters and results.
  27|
  28|// Fixed register assignments for the interpreter.
  29|
  30|// The following must be C callee-save.
  31|.define BASE,		x19	// Base of current Lua stack frame.
  32|.define KBASE,		x20	// Constants of current Lua function.
  33|.define PC,		x21	// Next PC.
  34|.define GLREG,		x22	// Global state.
  35|.define LREG,		x23	// Register holding lua_State (also in SAVE_L).
  36|.define TISNUM,	x24	// Constant LJ_TISNUM << 47.
  37|.define TISNUMhi,	x25	// Constant LJ_TISNUM << 15.
  38|.define TISNIL,	x26	// Constant -1LL.
  39|.define fp,		x29	// Yes, we have to maintain a frame pointer.
  40|
  41|.define ST_INTERP,	w26	// Constant -1.
  42|
  43|// The following temporaries are not saved across C calls, except for RA/RC.
  44|.define RA,		x27
  45|.define RC,		x28
  46|.define RB,		x17
  47|.define RAw,		w27
  48|.define RCw,		w28
  49|.define RBw,		w17
  50|.define INS,		x16
  51|.define INSw,		w16
  52|.define ITYPE,		x15
  53|.define TMP0,		x8
  54|.define TMP1,		x9
  55|.define TMP2,		x10
  56|.define TMP3,		x11
  57|.define TMP0w,		w8
  58|.define TMP1w,		w9
  59|.define TMP2w,		w10
  60|.define TMP3w,		w11
  61|
  62|// Calling conventions. Also used as temporaries.
  63|.define CARG1,		x0
  64|.define CARG2,		x1
  65|.define CARG3,		x2
  66|.define CARG4,		x3
  67|.define CARG5,		x4
  68|.define CARG1w,	w0
  69|.define CARG2w,	w1
  70|.define CARG3w,	w2
  71|.define CARG4w,	w3
  72|.define CARG5w,	w4
  73|
  74|.define FARG1,		d0
  75|.define FARG2,		d1
  76|
  77|.define CRET1,		x0
  78|.define CRET1w,	w0
  79|
  80|//-----------------------------------------------------------------------
  81|
  82|// ARM64e pointer authentication codes (PAC).
  83|.if PAUTH
  84|.macro sp_auth; pacibsp; .endmacro
  85|.macro br_auth, reg; braaz reg; .endmacro
  86|.macro blr_auth, reg; blraaz reg; .endmacro
  87|.macro ret_auth; retab; .endmacro
  88|.else
  89|.macro sp_auth; .endmacro
  90|.macro br_auth, reg; br reg; .endmacro
  91|.macro blr_auth, reg; blr reg; .endmacro
  92|.macro ret_auth; ret; .endmacro
  93|.endif
  94|
  95|// ARM64 branch target identification (BTI).
  96|.if BRANCH_TRACK
  97|.macro bti_jump; bti j; .endmacro
  98|.macro bti_call; bti c; .endmacro
  99|.macro bti_tailcall; bti jc; .endmacro
 100|.else
 101|.macro bti_jump; .endmacro
 102|.macro bti_call; .endmacro
 103|.macro bti_tailcall; .endmacro
 104|.endif
 105|
 106|//-----------------------------------------------------------------------
 107|
 108|// Stack layout while in interpreter. Must match with lj_frame.h.
 109|
 110|.define CFRAME_SPACE,	208
 111|//----- 16 byte aligned, <-- sp entering interpreter
 112|.define SAVE_FP_LR_,	192
 113|.define SAVE_GPR_,	112		// 112+10*8: 64 bit GPR saves
 114|.define SAVE_FPR_,	48		// 48+8*8: 64 bit FPR saves
 115|// Unused		[sp, #44]	// 32 bit values
 116|.define SAVE_NRES,	[sp, #40]
 117|.define SAVE_ERRF,	[sp, #36]
 118|.define SAVE_MULTRES,	[sp, #32]
 119|.define TMPD,		[sp, #24]	// 64 bit values
 120|.define SAVE_L,	[sp, #16]
 121|.define SAVE_PC,	[sp, #8]
 122|.define SAVE_CFRAME,	[sp, #0]
 123|//----- 16 byte aligned, <-- sp while in interpreter.
 124|
 125|.define TMPDofs,	#24
 126|
 127|.if WIN
 128|// Windows unwind data is suited to r1 stored first.
 129|.macro stp_unwind, r1, r2, where
 130|  stp r1, r2, where
 131|.endmacro
 132|.macro ldp_unwind, r1, r2, where
 133|  ldp r1, r2, where
 134|.endmacro
 135|.macro ldp_unwind, r1, r2, where, post_index
 136|  ldp r1, r2, where, post_index
 137|.endmacro
 138|.else
 139|// Otherwise store r2 first for compact unwind info (OSX).
 140|.macro stp_unwind, r1, r2, where
 141|  stp r2, r1, where
 142|.endmacro
 143|.macro ldp_unwind, r1, r2, where
 144|  ldp r2, r1, where
 145|.endmacro
 146|.macro ldp_unwind, r1, r2, where, post_index
 147|  ldp r2, r1, where, post_index
 148|.endmacro
 149|.endif
 150|
 151|.macro save_, gpr1, gpr2, fpr1, fpr2
 152|  stp_unwind d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(14-fpr1)*8]
 153|  stp_unwind x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(27-gpr1)*8]
 154|.endmacro
 155|.macro rest_, gpr1, gpr2, fpr1, fpr2
 156|  ldp_unwind d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(14-fpr1)*8]
 157|  ldp_unwind x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(27-gpr1)*8]
 158|.endmacro
 159|
 160|.macro saveregs
 161|  sp_auth
 162|  sub sp, sp, # CFRAME_SPACE
 163|  stp fp, lr, [sp, # SAVE_FP_LR_]
 164|  add fp, sp, # SAVE_FP_LR_
 165|  stp_unwind x19, x20, [sp, # SAVE_GPR_+(27-19)*8]
 166|  save_ 21, 22, 8, 9
 167|  save_ 23, 24, 10, 11
 168|  save_ 25, 26, 12, 13
 169|  save_ 27, 28, 14, 15
 170|.endmacro
 171|.macro restoreregs
 172|  ldp_unwind x19, x20, [sp, # SAVE_GPR_+(27-19)*8]
 173|  rest_ 21, 22, 8, 9
 174|  rest_ 23, 24, 10, 11
 175|  rest_ 25, 26, 12, 13
 176|  rest_ 27, 28, 14, 15
 177|  ldp fp, lr, [sp, # SAVE_FP_LR_]
 178|  add sp, sp, # CFRAME_SPACE
 179|.endmacro
 180|
 181|// Type definitions. Some of these are only used for documentation.
 182|.type L,		lua_State,	LREG
 183|.type GL,		global_State,	GLREG
 184|.type TVALUE,		TValue
 185|.type GCOBJ,		GCobj
 186|.type STR,		GCstr
 187|.type TAB,		GCtab
 188|.type LFUNC,		GCfuncL
 189|.type CFUNC,		GCfuncC
 190|.type PROTO,		GCproto
 191|.type UPVAL,		GCupval
 192|.type NODE,		Node
 193|.type NARGS8,		int
 194|.type TRACE,		GCtrace
 195|.type SBUF,		SBuf
 196|
 197|//-----------------------------------------------------------------------
 198|
 199|// Trap for not-yet-implemented parts.
 200|.macro NYI; brk; .endmacro
 201|
 202|//-----------------------------------------------------------------------
 203|
 204|// Access to frame relative to BASE.
 205|.define FRAME_FUNC,	#-16
 206|.define FRAME_PC,	#-8
 207|
 208|// Endian-specific defines.
 209|.if ENDIAN_LE
 210|.define LO,		0
 211|.define OFS_RD,	2
 212|.define OFS_RB,	3
 213|.define OFS_RA,	1
 214|.define OFS_OP,	0
 215|.else
 216|.define LO,		4
 217|.define OFS_RD,	0
 218|.define OFS_RB,	0
 219|.define OFS_RA,	2
 220|.define OFS_OP,	3
 221|.endif
 222|
 223|.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
 224|.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
 225|.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
 226|.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
 227|.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
 228|
 229|// Instruction decode+dispatch.
 230|.macro ins_NEXT
 231|  ldr INSw, [PC], #4
 232|  add TMP1, GL, INS, uxtb #3
 233|   decode_RA RA, INS
 234|  ldr TMP0, [TMP1, #GG_G2DISP]
 235|   decode_RD RC, INS
 236|  br_auth TMP0
 237|.endmacro
 238|
 239|// Instruction footer.
 240|.if 1
 241|  // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
 242|  .define ins_next, ins_NEXT
 243|  .define ins_next_, ins_NEXT
 244|.else
 245|  // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
 246|  // Affects only certain kinds of benchmarks (and only with -j off).
 247|  .macro ins_next
 248|    b ->ins_next
 249|  .endmacro
 250|  .macro ins_next_
 251|  ->ins_next:
 252|    ins_NEXT
 253|  .endmacro
 254|.endif
 255|
 256|// Call decode and dispatch.
 257|.macro ins_callt
 258|  // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
 259|  ldr PC, LFUNC:CARG3->pc
 260|  ldr INSw, [PC], #4
 261|  add TMP1, GL, INS, uxtb #3
 262|   decode_RA RA, INS
 263|  ldr TMP0, [TMP1, #GG_G2DISP]
 264|   add RA, BASE, RA, lsl #3
 265|  br_auth TMP0
 266|.endmacro
 267|
 268|.macro ins_call
 269|  // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
 270|  str PC, [BASE, FRAME_PC]
 271|  ins_callt
 272|.endmacro
 273|
 274|//-----------------------------------------------------------------------
 275|
 276|// Macros to check the TValue type and extract the GCobj. Branch on failure.
 277|.macro checktp, reg, tp, target
 278|  asr ITYPE, reg, #47
 279|  cmn ITYPE, #-tp
 280|   and reg, reg, #LJ_GCVMASK
 281|  bne target
 282|.endmacro
 283|.macro checktp, dst, reg, tp, target
 284|  asr ITYPE, reg, #47
 285|  cmn ITYPE, #-tp
 286|   and dst, reg, #LJ_GCVMASK
 287|  bne target
 288|.endmacro
 289|.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
 290|.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
 291|.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
 292|.macro checkint, reg, target
 293|  cmp TISNUMhi, reg, lsr #32
 294|  bne target
 295|.endmacro
 296|.macro checknum, reg, target
 297|  cmp TISNUMhi, reg, lsr #32
 298|  bls target
 299|.endmacro
 300|.macro checknumber, reg, target
 301|  cmp TISNUMhi, reg, lsr #32
 302|  blo target
 303|.endmacro
 304|
 305|.macro init_constants
 306|  movn TISNIL, #0
 307|  movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
 308|  movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
 309|.endmacro
 310|
 311|.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
 312|.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
 313|.macro mov_nil, reg; mov reg, TISNIL; .endmacro
 314|.macro cmp_nil, reg; cmp reg, TISNIL; .endmacro
 315|.macro add_TISNUM, dst, src; add dst, src, TISNUM; .endmacro
 316|
 317#define GL_J(field)	(GG_G2J + (int)offsetof(jit_State, field))
 318|
 319#define PC2PROTO(field)  ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
 320|
 321|.macro hotcheck, delta
 322|  lsr CARG1, PC, #1
 323|  and CARG1, CARG1, #126
 324|  add CARG1, CARG1, #GG_G2DISP+GG_DISP2HOT
 325|  ldrh CARG2w, [GL, CARG1]
 326|  subs CARG2, CARG2, #delta
 327|  strh CARG2w, [GL, CARG1]
 328|.endmacro
 329|
 330|.macro hotloop
 331|  hotcheck HOTCOUNT_LOOP
 332|  blo ->vm_hotloop
 333|.endmacro
 334|
 335|.macro hotcall
 336|  hotcheck HOTCOUNT_CALL
 337|  blo ->vm_hotcall
 338|.endmacro
 339|
 340|// Set current VM state.
 341|.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
 342|.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
 343|
 344|// Move table write barrier back. Overwrites mark and tmp.
 345|.macro barrierback, tab, mark, tmp
 346|  ldr tmp, GL->gc.grayagain
 347|   and mark, mark, #~LJ_GC_BLACK	// black2gray(tab)
 348|  str tab, GL->gc.grayagain
 349|   strb mark, tab->marked
 350|  str tmp, tab->gclist
 351|.endmacro
 352|
 353|//-----------------------------------------------------------------------
 354
 355#if !LJ_DUALNUM
 356#error "Only dual-number mode supported for ARM64 target"
 357#endif
 358
 359/* Generate subroutines used by opcodes and other parts of the VM. */
 360/* The .code_sub section should be last to help static branch prediction. */
 361static void build_subroutines(BuildCtx *ctx)
 362{
 363  |.code_sub
 364  |
 365  |//-----------------------------------------------------------------------
 366  |//-- Return handling ----------------------------------------------------
 367  |//-----------------------------------------------------------------------
 368  |
 369  |->vm_returnp:
 370  |  // See vm_return. Also: RB = previous base.
 371  |  tbz PC, #2, ->cont_dispatch	// (PC & FRAME_P) == 0?
 372  |
 373  |  // Return from pcall or xpcall fast func.
 374  |  ldr PC, [RB, FRAME_PC]		// Fetch PC of previous frame.
 375  |   mov_true TMP0
 376  |  mov BASE, RB
 377  |  // Prepending may overwrite the pcall frame, so do it at the end.
 378  |   str TMP0, [RA, #-8]!		// Prepend true to results.
 379  |
 380  |->vm_returnc:
 381  |  adds RC, RC, #8			// RC = (nresults+1)*8.
 382  |  mov CRET1, #LUA_YIELD
 383  |  beq ->vm_unwind_c_eh
 384  |  str RCw, SAVE_MULTRES
 385  |  ands CARG1, PC, #FRAME_TYPE
 386  |  beq ->BC_RET_Z			// Handle regular return to Lua.
 387  |
 388  |->vm_return:
 389  |  // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
 390  |  // CARG1 = PC & FRAME_TYPE
 391  |  and RB, PC, #~FRAME_TYPEP
 392  |   cmp CARG1, #FRAME_C
 393  |  sub RB, BASE, RB			// RB = previous base.
 394  |   bne ->vm_returnp
 395  |
 396  |  str RB, L->base
 397  |   ldrsw CARG2, SAVE_NRES		// CARG2 = nresults+1.
 398  |    mv_vmstate TMP0w, C
 399  |   sub BASE, BASE, #16
 400  |  subs TMP2, RC, #8
 401  |    st_vmstate TMP0w
 402  |  beq >2
 403  |1:
 404  |  subs TMP2, TMP2, #8
 405  |   ldr TMP0, [RA], #8
 406  |   str TMP0, [BASE], #8
 407  |  bne <1
 408  |2:
 409  |  cmp RC, CARG2, lsl #3		// More/less results wanted?
 410  |  bne >6
 411  |3:
 412  |  str BASE, L->top			// Store new top.
 413  |
 414  |->vm_leave_cp:
 415  |  ldr RC, SAVE_CFRAME		// Restore previous C frame.
 416  |   mov CRET1, #0			// Ok return status for vm_pcall.
 417  |  str RC, L->cframe
 418  |
 419  |->vm_leave_unw:
 420  |  restoreregs
 421  |  ret_auth
 422  |
 423  |6:
 424  |  bgt >7				// Less results wanted?
 425  |  // More results wanted. Check stack size and fill up results with nil.
 426  |  ldr CARG3, L->maxstack
 427  |  cmp BASE, CARG3
 428  |  bhs >8
 429  |   str TISNIL, [BASE], #8
 430  |  add RC, RC, #8
 431  |  b <2
 432  |
 433  |7:  // Less results wanted.
 434  |  cbz CARG2, <3			// LUA_MULTRET+1 case?
 435  |  sub CARG1, RC, CARG2, lsl #3
 436  |  sub BASE, BASE, CARG1		// Shrink top.
 437  |  b <3
 438  |
 439  |8:  // Corner case: need to grow stack for filling up results.
 440  |  // This can happen if:
 441  |  // - A C function grows the stack (a lot).
 442  |  // - The GC shrinks the stack in between.
 443  |  // - A return back from a lua_call() with (high) nresults adjustment.
 444  |  str BASE, L->top			// Save current top held in BASE (yes).
 445  |  mov CARG1, L
 446  |  bl extern lj_state_growstack	// (lua_State *L, int n)
 447  |  ldr BASE, L->top			// Need the (realloced) L->top in BASE.
 448  |  ldrsw CARG2, SAVE_NRES
 449  |  b <2
 450  |
 451  |->vm_unwind_c:			// Unwind C stack, return from vm_pcall.
 452  |  // (void *cframe, int errcode)
 453  |  bti_tailcall
 454  |  add fp, CARG1, # SAVE_FP_LR_
 455  |  mov sp, CARG1
 456  |  mov CRET1, CARG2
 457  |  ldr L, SAVE_L
 458  |  ldr GL, L->glref
 459  |->vm_unwind_c_eh:			// Landing pad for external unwinder.
 460  |  bti_tailcall
 461  |   mv_vmstate TMP0w, C
 462  |   st_vmstate TMP0w
 463  |  b ->vm_leave_unw
 464  |
 465  |->vm_unwind_ff:			// Unwind C stack, return from ff pcall.
 466  |  // (void *cframe)
 467  |  bti_tailcall
 468  |  add fp, CARG1, # SAVE_FP_LR_
 469  |  mov sp, CARG1
 470  |  ldr L, SAVE_L
 471  |    init_constants
 472  |   ldr GL, L->glref			// Setup pointer to global state.
 473  |->vm_unwind_ff_eh:			// Landing pad for external unwinder.
 474  |  bti_tailcall
 475  |    mov RC, #16			// 2 results: false + error message.
 476  |  ldr BASE, L->base
 477  |    mov_false TMP0
 478  |  sub RA, BASE, #8			// Results start at BASE-8.
 479  |  ldr PC, [BASE, FRAME_PC]		// Fetch PC of previous frame.
 480  |    str TMP0, [BASE, #-8]		// Prepend false to error message.
 481  |   st_vmstate ST_INTERP
 482  |  b ->vm_returnc
 483  |
 484  |//-----------------------------------------------------------------------
 485  |//-- Grow stack for calls -----------------------------------------------
 486  |//-----------------------------------------------------------------------
 487  |
 488  |->vm_growstack_c:			// Grow stack for C function.
 489  |  // CARG1 = L
 490  |  mov CARG2, #LUA_MINSTACK
 491  |  b >2
 492  |
 493  |->vm_growstack_l:			// Grow stack for Lua function.
 494  |  // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
 495  |  add RC, BASE, RC
 496  |   sub RA, RA, BASE
 497  |    mov CARG1, L
 498  |  stp BASE, RC, L->base
 499  |   add PC, PC, #4			// Must point after first instruction.
 500  |   lsr CARG2, RA, #3
 501  |2:
 502  |  // L->base = new base, L->top = top
 503  |  str PC, SAVE_PC
 504  |  bl extern lj_state_growstack	// (lua_State *L, int n)
 505  |  ldp BASE, RC, L->base
 506  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
 507  |   sub NARGS8:RC, RC, BASE
 508  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
 509  |  // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
 510  |  ins_callt				// Just retry the call.
 511  |
 512  |//-----------------------------------------------------------------------
 513  |//-- Entry points into the assembler VM ---------------------------------
 514  |//-----------------------------------------------------------------------
 515  |
 516  |->vm_resume:				// Setup C frame and resume thread.
 517  |  // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
 518  |  saveregs
 519  |  mov L, CARG1
 520  |    ldr GL, L->glref			// Setup pointer to global state.
 521  |  mov BASE, CARG2
 522  |   str L, SAVE_L
 523  |  mov PC, #FRAME_CP
 524  |   str wzr, SAVE_NRES
 525  |    add TMP0, sp, #CFRAME_RESUME
 526  |  ldrb TMP1w, L->status
 527  |   str wzr, SAVE_ERRF
 528  |   str L, SAVE_PC			// Any value outside of bytecode is ok.
 529  |   str xzr, SAVE_CFRAME
 530  |    str TMP0, L->cframe
 531  |  cbz TMP1w, >3
 532  |
 533  |  // Resume after yield (like a return).
 534  |  str L, GL->cur_L
 535  |  mov RA, BASE
 536  |   ldp BASE, CARG1, L->base
 537  |    init_constants
 538  |  ldr PC, [BASE, FRAME_PC]
 539  |     strb wzr, L->status
 540  |   sub RC, CARG1, BASE
 541  |  ands CARG1, PC, #FRAME_TYPE
 542  |   add RC, RC, #8
 543  |     st_vmstate ST_INTERP
 544  |   str RCw, SAVE_MULTRES
 545  |  beq ->BC_RET_Z
 546  |  b ->vm_return
 547  |
 548  |->vm_pcall:				// Setup protected C frame and enter VM.
 549  |  // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
 550  |  saveregs
 551  |  mov PC, #FRAME_CP
 552  |  str CARG4w, SAVE_ERRF
 553  |  b >1
 554  |
 555  |->vm_call:				// Setup C frame and enter VM.
 556  |  // (lua_State *L, TValue *base, int nres1)
 557  |  saveregs
 558  |  mov PC, #FRAME_C
 559  |
 560  |1:  // Entry point for vm_pcall above (PC = ftype).
 561  |  ldr RC, L:CARG1->cframe
 562  |   str CARG3w, SAVE_NRES
 563  |    mov L, CARG1
 564  |   str CARG1, SAVE_L
 565  |    ldr GL, L->glref			// Setup pointer to global state.
 566  |     mov BASE, CARG2
 567  |   str CARG1, SAVE_PC		// Any value outside of bytecode is ok.
 568  |  add TMP0, sp, #0
 569  |   str RC, SAVE_CFRAME
 570  |  str TMP0, L->cframe		// Add our C frame to cframe chain.
 571  |
 572  |3:  // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
 573  |  str L, GL->cur_L
 574  |  ldp RB, CARG1, L->base		// RB = old base (for vmeta_call).
 575  |  add PC, PC, BASE
 576  |    init_constants
 577  |  sub PC, PC, RB			// PC = frame delta + frame type
 578  |   sub NARGS8:RC, CARG1, BASE
 579  |    st_vmstate ST_INTERP
 580  |
 581  |->vm_call_dispatch:
 582  |  // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
 583  |  ldr CARG3, [BASE, FRAME_FUNC]
 584  |  checkfunc CARG3, ->vmeta_call
 585  |
 586  |->vm_call_dispatch_f:
 587  |  ins_call
 588  |  // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
 589  |
 590  |->vm_cpcall:				// Setup protected C frame, call C.
 591  |  // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
 592  |  saveregs
 593  |  mov L, CARG1
 594  |   ldr RA, L:CARG1->stack
 595  |  str CARG1, SAVE_L
 596  |    ldr GL, L->glref			// Setup pointer to global state.
 597  |   ldr RB, L->top
 598  |  str CARG1, SAVE_PC			// Any value outside of bytecode is ok.
 599  |  ldr RC, L->cframe
 600  |   sub RA, RA, RB			// Compute -savestack(L, L->top).
 601  |   str RAw, SAVE_NRES		// Neg. delta means cframe w/o frame.
 602  |  str wzr, SAVE_ERRF			// No error function.
 603  |  add TMP0, sp, #0
 604  |   str RC, SAVE_CFRAME
 605  |  str TMP0, L->cframe		// Add our C frame to cframe chain.
 606  |    str L, GL->cur_L
 607  |  blr_auth CARG4		// (lua_State *L, lua_CFunction func, void *ud)
 608  |  mov BASE, CRET1
 609  |   mov PC, #FRAME_CP
 610  |  cbnz BASE, <3			// Else continue with the call.
 611  |  b ->vm_leave_cp			// No base? Just remove C frame.
 612  |
 613  |//-----------------------------------------------------------------------
 614  |//-- Metamethod handling ------------------------------------------------
 615  |//-----------------------------------------------------------------------
 616  |
 617  |//-- Continuation dispatch ----------------------------------------------
 618  |
 619  |->cont_dispatch:
 620  |  // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
 621  |  ldr LFUNC:CARG3, [RB, FRAME_FUNC]
 622  |    ldr CARG1, [BASE, #-32]		// Get continuation.
 623  |   mov CARG4, BASE
 624  |   mov BASE, RB			// Restore caller BASE.
 625  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
 626  |.if FFI
 627  |    cmp CARG1, #1
 628  |.endif
 629  |   ldr PC, [CARG4, #-24]		// Restore PC from [cont|PC].
 630  |    add TMP0, RA, RC
 631  |    str TISNIL, [TMP0, #-8]		// Ensure one valid arg.
 632  |.if FFI
 633  |    bls >1
 634  |.endif
 635  |  ldr CARG3, LFUNC:CARG3->pc
 636  |  ldr KBASE, [CARG3, #PC2PROTO(k)]
 637  |  // BASE = base, RA = resultptr, CARG4 = meta base
 638  |    br_auth CARG1
 639  |
 640  |.if FFI
 641  |1:
 642  |  beq ->cont_ffi_callback		// cont = 1: return from FFI callback.
 643  |  // cont = 0: tailcall from C function.
 644  |   sub CARG4, CARG4, #32
 645  |   sub RC, CARG4, BASE
 646  |  b ->vm_call_tail
 647  |.endif
 648  |
 649  |->cont_cat:				// RA = resultptr, CARG4 = meta base
 650  |  bti_jump
 651  |  ldr INSw, [PC, #-4]
 652  |   sub CARG2, CARG4, #32
 653  |   ldr TMP0, [RA]
 654  |     str BASE, L->base
 655  |  decode_RB RB, INS
 656  |   decode_RA RA, INS
 657  |  add TMP1, BASE, RB, lsl #3
 658  |  subs TMP1, CARG2, TMP1
 659  |  beq >1
 660  |   str TMP0, [CARG2]
 661  |  lsr CARG3, TMP1, #3
 662  |  b ->BC_CAT_Z
 663  |
 664  |1:
 665  |   str TMP0, [BASE, RA, lsl #3]
 666  |  b ->cont_nop
 667  |
 668  |//-- Table indexing metamethods -----------------------------------------
 669  |
 670  |->vmeta_tgets1:
 671  |  movn CARG4, #~LJ_TSTR
 672  |   add CARG2, BASE, RB, lsl #3
 673  |  add CARG4, STR:RC, CARG4, lsl #47
 674  |  b >2
 675  |
 676  |->vmeta_tgets:
 677  |  movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
 678  |  str CARG2, GL->tmptv
 679  |  add CARG2, GL, #offsetof(global_State, tmptv)
 680  |2:
 681  |   add CARG3, sp, TMPDofs
 682  |  str CARG4, TMPD
 683  |  b >1
 684  |
 685  |->vmeta_tgetb:			// RB = table, RC = index
 686  |  add_TISNUM RC, RC
 687  |   add CARG2, BASE, RB, lsl #3
 688  |   add CARG3, sp, TMPDofs
 689  |  str RC, TMPD
 690  |  b >1
 691  |
 692  |->vmeta_tgetv:			// RB = table, RC = key
 693  |  add CARG2, BASE, RB, lsl #3
 694  |   add CARG3, BASE, RC, lsl #3
 695  |1:
 696  |   str BASE, L->base
 697  |  mov CARG1, L
 698  |   str PC, SAVE_PC
 699  |  bl extern lj_meta_tget		// (lua_State *L, TValue *o, TValue *k)
 700  |  // Returns TValue * (finished) or NULL (metamethod).
 701  |  cbz CRET1, >3
 702  |  ldr TMP0, [CRET1]
 703  |  str TMP0, [BASE, RA, lsl #3]
 704  |  ins_next
 705  |
 706  |3:  // Call __index metamethod.
 707  |  // BASE = base, L->top = new base, stack = cont/func/t/k
 708  |   sub TMP1, BASE, #FRAME_CONT
 709  |  ldr BASE, L->top
 710  |    mov NARGS8:RC, #16		// 2 args for func(t, k).
 711  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
 712  |    str PC, [BASE, #-24]		// [cont|PC]
 713  |   sub PC, BASE, TMP1
 714  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
 715  |  b ->vm_call_dispatch_f
 716  |
 717  |->vmeta_tgetr:
 718  |  sxtw CARG2, TMP1w
 719  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
 720  |  // Returns cTValue * or NULL.
 721  |  mov_nil TMP0
 722  |  cbz CRET1, ->BC_TGETR_Z
 723  |  ldr TMP0, [CRET1]
 724  |  b ->BC_TGETR_Z
 725  |
 726  |//-----------------------------------------------------------------------
 727  |
 728  |->vmeta_tsets1:
 729  |  movn CARG4, #~LJ_TSTR
 730  |   add CARG2, BASE, RB, lsl #3
 731  |  add CARG4, STR:RC, CARG4, lsl #47
 732  |  b >2
 733  |
 734  |->vmeta_tsets:
 735  |  movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
 736  |  str CARG2, GL->tmptv
 737  |  add CARG2, GL, #offsetof(global_State, tmptv)
 738  |2:
 739  |   add CARG3, sp, TMPDofs
 740  |  str CARG4, TMPD
 741  |  b >1
 742  |
 743  |->vmeta_tsetb:			// RB = table, RC = index
 744  |  add_TISNUM RC, RC
 745  |   add CARG2, BASE, RB, lsl #3
 746  |   add CARG3, sp, TMPDofs
 747  |  str RC, TMPD
 748  |  b >1
 749  |
 750  |->vmeta_tsetv:
 751  |  add CARG2, BASE, RB, lsl #3
 752  |   add CARG3, BASE, RC, lsl #3
 753  |1:
 754  |   str BASE, L->base
 755  |  mov CARG1, L
 756  |   str PC, SAVE_PC
 757  |  bl extern lj_meta_tset		// (lua_State *L, TValue *o, TValue *k)
 758  |  // Returns TValue * (finished) or NULL (metamethod).
 759  |   ldr TMP0, [BASE, RA, lsl #3]
 760  |  cbz CRET1, >3
 761  |  // NOBARRIER: lj_meta_tset ensures the table is not black.
 762  |   str TMP0, [CRET1]
 763  |  ins_next
 764  |
 765  |3:  // Call __newindex metamethod.
 766  |  // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
 767  |   sub TMP1, BASE, #FRAME_CONT
 768  |  ldr BASE, L->top
 769  |    mov NARGS8:RC, #24		// 3 args for func(t, k, v).
 770  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
 771  |   str TMP0, [BASE, #16]		// Copy value to third argument.
 772  |    str PC, [BASE, #-24]		// [cont|PC]
 773  |   sub PC, BASE, TMP1
 774  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
 775  |  b ->vm_call_dispatch_f
 776  |
 777  |->vmeta_tsetr:
 778  |  sxtw CARG3, TMP1w
 779  |  str BASE, L->base
 780  |  mov CARG1, L
 781  |  str PC, SAVE_PC
 782  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
 783  |  // Returns TValue *.
 784  |  b ->BC_TSETR_Z
 785  |
 786  |//-- Comparison metamethods ---------------------------------------------
 787  |
 788  |->vmeta_comp:
 789  |  add CARG2, BASE, RA, lsl #3
 790  |   sub PC, PC, #4
 791  |  add CARG3, BASE, RC, lsl #3
 792  |   str BASE, L->base
 793  |  mov CARG1, L
 794  |   str PC, SAVE_PC
 795  |  uxtb CARG4w, INSw
 796  |  bl extern lj_meta_comp  // (lua_State *L, TValue *o1, *o2, int op)
 797  |  // Returns 0/1 or TValue * (metamethod).
 798  |3:
 799  |  cmp CRET1, #1
 800  |  bhi ->vmeta_binop
 801  |4:
 802  |   ldrh RBw, [PC, # OFS_RD]
 803  |    add PC, PC, #4
 804  |   add RB, PC, RB, lsl #2
 805  |   sub RB, RB, #0x20000
 806  |  csel PC, PC, RB, lo
 807  |->cont_nop:
 808  |  bti_jump
 809  |  ins_next
 810  |
 811  |->cont_ra:				// RA = resultptr
 812  |  bti_jump
 813  |  ldr INSw, [PC, #-4]
 814  |   ldr TMP0, [RA]
 815  |  decode_RA TMP1, INS
 816  |   str TMP0, [BASE, TMP1, lsl #3]
 817  |  b ->cont_nop
 818  |
 819  |->cont_condt:			// RA = resultptr
 820  |  bti_jump
 821  |  ldr TMP0, [RA]
 822  |   mov_true TMP1
 823  |  cmp TMP1, TMP0			// Branch if result is true.
 824  |  b <4
 825  |
 826  |->cont_condf:			// RA = resultptr
 827  |  bti_jump
 828  |  ldr TMP0, [RA]
 829  |   mov_false TMP1
 830  |  cmp TMP0, TMP1			// Branch if result is false.
 831  |  b <4
 832  |
 833  |->vmeta_equal:
 834  |  // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
 835  |  and TAB:CARG3, CARG3, #LJ_GCVMASK
 836  |  sub PC, PC, #4
 837  |   str BASE, L->base
 838  |   mov CARG1, L
 839  |  str PC, SAVE_PC
 840  |  bl extern lj_meta_equal  // (lua_State *L, GCobj *o1, *o2, int ne)
 841  |  // Returns 0/1 or TValue * (metamethod).
 842  |  b <3
 843  |
 844  |->vmeta_equal_cd:
 845  |.if FFI
 846  |  sub PC, PC, #4
 847  |   str BASE, L->base
 848  |   mov CARG1, L
 849  |   mov CARG2, INS
 850  |  str PC, SAVE_PC
 851  |  bl extern lj_meta_equal_cd		// (lua_State *L, BCIns op)
 852  |  // Returns 0/1 or TValue * (metamethod).
 853  |  b <3
 854  |.endif
 855  |
 856  |->vmeta_istype:
 857  |  sub PC, PC, #4
 858  |   str BASE, L->base
 859  |   mov CARG1, L
 860  |   mov CARG2, RA
 861  |   mov CARG3, RC
 862  |  str PC, SAVE_PC
 863  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
 864  |  b ->cont_nop
 865  |
 866  |//-- Arithmetic metamethods ---------------------------------------------
 867  |
 868  |->vmeta_arith_vn:
 869  |  add CARG3, BASE, RB, lsl #3
 870  |   add CARG4, KBASE, RC, lsl #3
 871  |  b >1
 872  |
 873  |->vmeta_arith_nv:
 874  |  add CARG4, BASE, RB, lsl #3
 875  |   add CARG3, KBASE, RC, lsl #3
 876  |  b >1
 877  |
 878  |->vmeta_unm:
 879  |  add CARG3, BASE, RC, lsl #3
 880  |  mov CARG4, CARG3
 881  |  b >1
 882  |
 883  |->vmeta_arith_vv:
 884  |  add CARG3, BASE, RB, lsl #3
 885  |   add CARG4, BASE, RC, lsl #3
 886  |1:
 887  |  uxtb CARG5w, INSw
 888  |   add CARG2, BASE, RA, lsl #3
 889  |    str BASE, L->base
 890  |   mov CARG1, L
 891  |    str PC, SAVE_PC
 892  |  bl extern lj_meta_arith  // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
 893  |  // Returns NULL (finished) or TValue * (metamethod).
 894  |  cbz CRET1, ->cont_nop
 895  |
 896  |  // Call metamethod for binary op.
 897  |->vmeta_binop:
 898  |  // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
 899  |  sub TMP1, CRET1, BASE
 900  |   str PC, [CRET1, #-24]		// [cont|PC]
 901  |  add PC, TMP1, #FRAME_CONT
 902  |  mov BASE, CRET1
 903  |   mov NARGS8:RC, #16		// 2 args for func(o1, o2).
 904  |  b ->vm_call_dispatch
 905  |
 906  |->vmeta_len:
 907  |  add CARG2, BASE, RC, lsl #3
 908#if LJ_52
 909  |  mov TAB:RC, TAB:CARG1		// Save table (ignored for other types).
 910#endif
 911  |   str BASE, L->base
 912  |  mov CARG1, L
 913  |   str PC, SAVE_PC
 914  |  bl extern lj_meta_len		// (lua_State *L, TValue *o)
 915  |  // Returns NULL (retry) or TValue * (metamethod base).
 916#if LJ_52
 917  |  cbnz CRET1, ->vmeta_binop		// Binop call for compatibility.
 918  |  mov TAB:CARG1, TAB:RC
 919  |  b ->BC_LEN_Z
 920#else
 921  |  b ->vmeta_binop			// Binop call for compatibility.
 922#endif
 923  |
 924  |//-- Call metamethod ----------------------------------------------------
 925  |
 926  |->vmeta_call:			// Resolve and call __call metamethod.
 927  |  // RB = old base, BASE = new base, RC = nargs*8
 928  |  mov CARG1, L
 929  |   str RB, L->base			// This is the callers base!
 930  |  sub CARG2, BASE, #16
 931  |   str PC, SAVE_PC
 932  |  add CARG3, BASE, NARGS8:RC
 933  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
 934  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
 935  |   add NARGS8:RC, NARGS8:RC, #8	// Got one more argument now.
 936  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
 937  |  ins_call
 938  |
 939  |->vmeta_callt:			// Resolve __call for BC_CALLT.
 940  |  // BASE = old base, RA = new base, RC = nargs*8
 941  |  mov CARG1, L
 942  |   str BASE, L->base
 943  |  sub CARG2, RA, #16
 944  |   str PC, SAVE_PC
 945  |  add CARG3, RA, NARGS8:RC
 946  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
 947  |  ldr TMP1, [RA, FRAME_FUNC]		// Guaranteed to be a function here.
 948  |   ldr PC, [BASE, FRAME_PC]
 949  |   add NARGS8:RC, NARGS8:RC, #8	// Got one more argument now.
 950  |  and LFUNC:CARG3, TMP1, #LJ_GCVMASK
 951  |  b ->BC_CALLT2_Z
 952  |
 953  |//-- Argument coercion for 'for' statement ------------------------------
 954  |
 955  |->vmeta_for:
 956  |  mov CARG1, L
 957  |   str BASE, L->base
 958  |  mov CARG2, RA
 959  |   str PC, SAVE_PC
 960  |  bl extern lj_meta_for	// (lua_State *L, TValue *base)
 961  |  ldr INSw, [PC, #-4]
 962  |.if JIT
 963  |   uxtb TMP0w, INSw
 964  |.endif
 965  |  decode_RA RA, INS
 966  |  decode_RD RC, INS
 967  |.if JIT
 968  |   cmp TMP0, #BC_JFORI
 969  |   beq =>BC_JFORI
 970  |.endif
 971  |  b =>BC_FORI
 972  |
 973  |//-----------------------------------------------------------------------
 974  |//-- Fast functions -----------------------------------------------------
 975  |//-----------------------------------------------------------------------
 976  |
 977  |.macro .ffunc, name
 978  |->ff_ .. name:
 979  |  bti_jump
 980  |.endmacro
 981  |
 982  |.macro .ffunc_1, name
 983  |->ff_ .. name:
 984  |  bti_jump
 985  |  ldr CARG1, [BASE]
 986  |   cmp NARGS8:RC, #8
 987  |   blo ->fff_fallback
 988  |.endmacro
 989  |
 990  |.macro .ffunc_2, name
 991  |->ff_ .. name:
 992  |  bti_jump
 993  |  ldp CARG1, CARG2, [BASE]
 994  |   cmp NARGS8:RC, #16
 995  |   blo ->fff_fallback
 996  |.endmacro
 997  |
 998  |.macro .ffunc_n, name
 999  |  .ffunc name
1000  |  ldr CARG1, [BASE]
1001  |   cmp NARGS8:RC, #8
1002  |  ldr FARG1, [BASE]
1003  |   blo ->fff_fallback
1004  |  checknum CARG1, ->fff_fallback
1005  |.endmacro
1006  |
1007  |.macro .ffunc_nn, name
1008  |  .ffunc name
1009  |  ldp CARG1, CARG2, [BASE]
1010  |   cmp NARGS8:RC, #16
1011  |  ldp FARG1, FARG2, [BASE]
1012  |   blo ->fff_fallback
1013  |  checknum CARG1, ->fff_fallback
1014  |  checknum CARG2, ->fff_fallback
1015  |.endmacro
1016  |
1017  |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
1018  |.macro ffgccheck
1019  |  ldp CARG1, CARG2, GL->gc.total	// Assumes threshold follows total.
1020  |  cmp CARG1, CARG2
1021  |  blt >1
1022  |  bl ->fff_gcstep
1023  |1:
1024  |.endmacro
1025  |
1026  |//-- Base library: checks -----------------------------------------------
1027  |
1028  |.ffunc_1 assert
1029  |   ldr PC, [BASE, FRAME_PC]
1030  |  mov_false TMP1
1031  |  cmp CARG1, TMP1
1032  |  bhs ->fff_fallback
1033  |  str CARG1, [BASE, #-16]
1034  |  sub RB, BASE, #8
1035  |  subs RA, NARGS8:RC, #8
1036  |   add RC, NARGS8:RC, #8		// Compute (nresults+1)*8.
1037  |  cbz RA, ->fff_res			// Done if exactly 1 argument.
1038  |1:
1039  |   ldr CARG1, [RB, #16]
1040  |  sub RA, RA, #8
1041  |   str CARG1, [RB], #8
1042  |  cbnz RA, <1
1043  |  b ->fff_res
1044  |
1045  |.ffunc_1 type
1046  |  mov TMP0, #~LJ_TISNUM
1047  |  asr ITYPE, CARG1, #47
1048  |  cmn ITYPE, #~LJ_TISNUM
1049  |  csinv TMP1, TMP0, ITYPE, lo
1050  |  add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
1051  |  ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
1052  |  b ->fff_restv
1053  |
1054  |//-- Base library: getters and setters ---------------------------------
1055  |
1056  |.ffunc_1 getmetatable
1057  |  asr ITYPE, CARG1, #47
1058  |  cmn ITYPE, #-LJ_TTAB
1059  |  ccmn ITYPE, #-LJ_TUDATA, #4, ne
1060  |   and TAB:CARG1, CARG1, #LJ_GCVMASK
1061  |  bne >6
1062  |1:  // Field metatable must be at same offset for GCtab and GCudata!
1063  |  ldr TAB:RB, TAB:CARG1->metatable
1064  |2:
1065  |   mov_nil CARG1
1066  |   ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
1067  |  cbz TAB:RB, ->fff_restv
1068  |  ldr TMP1w, TAB:RB->hmask
1069  |   ldr TMP2w, STR:RC->sid
1070  |    ldr NODE:CARG3, TAB:RB->node
1071  |  and TMP1w, TMP1w, TMP2w		// idx = str->sid & tab->hmask
1072  |  add TMP1, TMP1, TMP1, lsl #1
1073  |  movn CARG4, #~LJ_TSTR
1074  |    add NODE:CARG3, NODE:CARG3, TMP1, lsl #3  // node = tab->node + idx*3*8
1075  |  add CARG4, STR:RC, CARG4, lsl #47	// Tagged key to look for.
1076  |3:  // Rearranged logic, because we expect _not_ to find the key.
1077  |  ldp CARG1, TMP0, NODE:CARG3->val
1078  |   ldr NODE:CARG3, NODE:CARG3->next
1079  |  cmp TMP0, CARG4
1080  |  beq >5
1081  |  cbnz NODE:CARG3, <3
1082  |4:
1083  |  mov CARG1, RB			// Use metatable as default result.
1084  |  movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
1085  |  b ->fff_restv
1086  |5:
1087  |  cmp_nil TMP0
1088  |  bne ->fff_restv
1089  |  b <4
1090  |
1091  |6:
1092  |  movn TMP0, #~LJ_TISNUM
1093  |  cmp ITYPE, TMP0
1094  |  csel ITYPE, ITYPE, TMP0, hs
1095  |  sub TMP1, GL, ITYPE, lsl #3
1096  |  ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
1097  |  b <2
1098  |
1099  |.ffunc_2 setmetatable
1100  |  // Fast path: no mt for table yet and not clearing the mt.
1101  |  checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1102  |   ldr TAB:TMP0, TAB:TMP1->metatable
1103  |  asr ITYPE, CARG2, #47
1104  |   ldrb TMP2w, TAB:TMP1->marked
1105  |  cmn ITYPE, #-LJ_TTAB
1106  |    and TAB:CARG2, CARG2, #LJ_GCVMASK
1107  |  ccmp TAB:TMP0, #0, #0, eq
1108  |  bne ->fff_fallback
1109  |    str TAB:CARG2, TAB:TMP1->metatable
1110  |   tbz TMP2w, #2, ->fff_restv	// isblack(table)
1111  |  barrierback TAB:TMP1, TMP2w, TMP0
1112  |  b ->fff_restv
1113  |
1114  |.ffunc rawget
1115  |  ldr CARG2, [BASE]
1116  |   cmp NARGS8:RC, #16
1117  |   blo ->fff_fallback
1118  |  checktab CARG2, ->fff_fallback
1119  |   mov CARG1, L
1120  |   add CARG3, BASE, #8
1121  |  bl extern lj_tab_get  // (lua_State *L, GCtab *t, cTValue *key)
1122  |  // Returns cTValue *.
1123  |  ldr CARG1, [CRET1]
1124  |  b ->fff_restv
1125  |
1126  |//-- Base library: conversions ------------------------------------------
1127  |
1128  |.ffunc tonumber
1129  |  // Only handles the number case inline (without a base argument).
1130  |  ldr CARG1, [BASE]
1131  |   cmp NARGS8:RC, #8
1132  |   bne ->fff_fallback
1133  |  checknumber CARG1, ->fff_fallback
1134  |  b ->fff_restv
1135  |
1136  |.ffunc_1 tostring
1137  |  // Only handles the string or number case inline.
1138  |  asr ITYPE, CARG1, #47
1139  |  cmn ITYPE, #-LJ_TSTR
1140  |  // A __tostring method in the string base metatable is ignored.
1141  |  beq ->fff_restv
1142  |  // Handle numbers inline, unless a number base metatable is present.
1143  |  ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
1144  |   str BASE, L->base
1145  |  cmn ITYPE, #-LJ_TISNUM
1146  |  ccmp TMP1, #0, #0, ls
1147  |   str PC, SAVE_PC			// Redundant (but a defined value).
1148  |  bne ->fff_fallback
1149  |  ffgccheck
1150  |  mov CARG1, L
1151  |  mov CARG2, BASE
1152  |  bl extern lj_strfmt_number		// (lua_State *L, cTValue *o)
1153  |  // Returns GCstr *.
1154  |   movn TMP1, #~LJ_TSTR
1155  |  ldr BASE, L->base
1156  |   add CARG1, CARG1, TMP1, lsl #47
1157  |  b ->fff_restv
1158  |
1159  |//-- Base library: iterators -------------------------------------------
1160  |
1161  |.ffunc_1 next
1162  |  checktp CARG1, LJ_TTAB, ->fff_fallback
1163  |  str TISNIL, [BASE, NARGS8:RC]	// Set missing 2nd arg to nil.
1164  |  ldr PC, [BASE, FRAME_PC]
1165  |  add CARG2, BASE, #8
1166  |  sub CARG3, BASE, #16
1167  |  bl extern lj_tab_next		// (GCtab *t, cTValue *key, TValue *o)
1168  |  // Returns 1=found, 0=end, -1=error.
1169  |   mov RC, #(2+1)*8
1170  |  tbnz CRET1w, #31, ->fff_fallback	// Invalid key.
1171  |  cbnz CRET1, ->fff_res		// Found key/value.
1172  |  // End of traversal: return nil.
1173  |  str TISNIL, [BASE, #-16]
1174  |  b ->fff_res1
1175  |
1176  |.ffunc_1 pairs
1177  |  checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1178#if LJ_52
1179  |  ldr TAB:CARG2, TAB:TMP1->metatable
1180#endif
1181  |   ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1182  |    ldr PC, [BASE, FRAME_PC]
1183#if LJ_52
1184  |  cbnz TAB:CARG2, ->fff_fallback
1185#endif
1186  |  mov RC, #(3+1)*8
1187  |  stp CFUNC:CARG4, CARG1, [BASE, #-16]
1188  |   str TISNIL, [BASE]
1189  |  b ->fff_res
1190  |
1191  |.ffunc_2 ipairs_aux
1192  |  checktab CARG1, ->fff_fallback
1193  |   checkint CARG2, ->fff_fallback
1194  |  ldr TMP1w, TAB:CARG1->asize
1195  |   ldr CARG3, TAB:CARG1->array
1196  |    ldr TMP0w, TAB:CARG1->hmask
1197  |  add CARG2w, CARG2w, #1
1198  |  cmp CARG2w, TMP1w
1199  |    ldr PC, [BASE, FRAME_PC]
1200  |     add_TISNUM TMP2, CARG2
1201  |   mov RC, #(0+1)*8
1202  |     str TMP2, [BASE, #-16]
1203  |  bhs >2				// Not in array part?
1204  |  ldr TMP0, [CARG3, CARG2, lsl #3]
1205  |1:
1206  |   mov TMP1, #(2+1)*8
1207  |   cmp_nil TMP0
1208  |  str TMP0, [BASE, #-8]
1209  |   csel RC, RC, TMP1, eq
1210  |  b ->fff_res
1211  |2:  // Check for empty hash part first. Otherwise call C function.
1212  |  cbz TMP0w, ->fff_res
1213  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
1214  |  // Returns cTValue * or NULL.
1215  |  cbz CRET1, ->fff_res
1216  |  ldr TMP0, [CRET1]
1217  |  b <1
1218  |
1219  |.ffunc_1 ipairs
1220  |  checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1221#if LJ_52
1222  |  ldr TAB:CARG2, TAB:TMP1->metatable
1223#endif
1224  |   ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1225  |    ldr PC, [BASE, FRAME_PC]
1226#if LJ_52
1227  |  cbnz TAB:CARG2, ->fff_fallback
1228#endif
1229  |  mov RC, #(3+1)*8
1230  |  stp CFUNC:CARG4, CARG1, [BASE, #-16]
1231  |   str TISNUM, [BASE]
1232  |  b ->fff_res
1233  |
1234  |//-- Base library: catch errors ----------------------------------------
1235  |
1236  |.ffunc pcall
1237  |  ldr TMP1, L->maxstack
1238  |  add TMP2, BASE, NARGS8:RC
1239  |  cmp TMP1, TMP2
1240  |  blo ->fff_fallback
1241  |   cmp NARGS8:RC, #8
1242  |  ldrb TMP0w, GL->hookmask
1243  |   blo ->fff_fallback
1244  |   sub NARGS8:RC, NARGS8:RC, #8
1245  |    mov RB, BASE
1246  |    add BASE, BASE, #16
1247  |  ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1248  |  add PC, TMP0, #16+FRAME_PCALL
1249  |   beq ->vm_call_dispatch
1250  |1:
1251  |   add TMP2, BASE, NARGS8:RC
1252  |2:
1253  |   ldr TMP0, [TMP2, #-16]
1254  |   str TMP0, [TMP2, #-8]!
1255  |  cmp TMP2, BASE
1256  |  bne <2
1257  |  b ->vm_call_dispatch
1258  |
1259  |.ffunc xpcall
1260  |  ldr TMP1, L->maxstack
1261  |  add TMP2, BASE, NARGS8:RC
1262  |  cmp TMP1, TMP2
1263  |  blo ->fff_fallback
1264  |     ldp CARG1, CARG2, [BASE]
1265  |  ldrb TMP0w, GL->hookmask
1266  |   subs NARGS8:TMP1, NARGS8:RC, #16
1267  |   blo ->fff_fallback
1268  |    mov RB, BASE
1269  |     asr ITYPE, CARG2, #47
1270  |  ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1271  |     cmn ITYPE, #-LJ_TFUNC
1272  |  add PC, TMP0, #24+FRAME_PCALL
1273  |     bne ->fff_fallback		// Traceback must be a function.
1274  |   mov NARGS8:RC, NARGS8:TMP1
1275  |    add BASE, BASE, #24
1276  |     stp CARG2, CARG1, [RB]		// Swap function and traceback.
1277  |   cbz NARGS8:RC, ->vm_call_dispatch
1278  |  b <1
1279  |
1280  |//-- Coroutine library --------------------------------------------------
1281  |
1282  |.macro coroutine_resume_wrap, resume
1283  |.if resume
1284  |.ffunc_1 coroutine_resume
1285  |  checktp CARG1, LJ_TTHREAD, ->fff_fallback
1286  |.else
1287  |.ffunc coroutine_wrap_aux
1288  |  ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1289  |  and L:CARG1, CARG1, #LJ_GCVMASK
1290  |.endif
1291  |   ldr PC, [BASE, FRAME_PC]
1292  |     str BASE, L->base
1293  |  ldp RB, CARG2, L:CARG1->base
1294  |   ldrb TMP1w, L:CARG1->status
1295  |  add TMP0, CARG2, TMP1
1296  |   str PC, SAVE_PC
1297  |  cmp TMP0, RB
1298  |  beq ->fff_fallback
1299  |   cmp TMP1, #LUA_YIELD
1300  |    add TMP0, CARG2, #8
1301  |   csel CARG2, CARG2, TMP0, hs
1302  |   ldr CARG4, L:CARG1->maxstack
1303  |   add CARG3, CARG2, NARGS8:RC
1304  |    ldr RB, L:CARG1->cframe
1305  |   ccmp CARG3, CARG4, #2, ls
1306  |    ccmp RB, #0, #2, ls
1307  |    bhi ->fff_fallback
1308  |.if resume
1309  |  sub CARG3, CARG3, #8		// Keep resumed thread in stack for GC.
1310  |  add BASE, BASE, #8
1311  |  sub NARGS8:RC, NARGS8:RC, #8
1312  |.endif
1313  |  str CARG3, L:CARG1->top
1314  |  str BASE, L->top
1315  |  cbz NARGS8:RC, >3
1316  |2:  // Move args to coroutine.
1317  |   ldr TMP0, [BASE, RB]
1318  |  cmp RB, NARGS8:RC
1319  |   str TMP0, [CARG2, RB]
1320  |   add RB, RB, #8
1321  |  bne <2
1322  |3:
1323  |  mov CARG3, #0
1324  |   mov L:RA, L:CARG1
1325  |  mov CARG4, #0
1326  |  bl ->vm_resume			// (lua_State *L, TValue *base, 0, 0)
1327  |  // Returns thread status.
1328  |4:
1329  |  ldp CARG3, CARG4, L:RA->base
1330  |   cmp CRET1, #LUA_YIELD
1331  |  ldr BASE, L->base
1332  |    str L, GL->cur_L
1333  |    st_vmstate ST_INTERP
1334  |   bhi >8
1335  |  sub RC, CARG4, CARG3
1336  |   ldr CARG1, L->maxstack
1337  |   add CARG2, BASE, RC
1338  |  cbz RC, >6				// No results?
1339  |  cmp CARG2, CARG1
1340  |   mov RB, #0
1341  |  bhi >9				// Need to grow stack?
1342  |
1343  |  sub CARG4, RC, #8
1344  |   str CARG3, L:RA->top		// Clear coroutine stack.
1345  |5:  // Move results from coroutine.
1346  |   ldr TMP0, [CARG3, RB]
1347  |  cmp RB, CARG4
1348  |   str TMP0, [BASE, RB]
1349  |   add RB, RB, #8
1350  |  bne <5
1351  |6:
1352  |.if resume
1353  |  mov_true TMP1
1354  |   add RC, RC, #16
1355  |7:
1356  |  str TMP1, [BASE, #-8]		// Prepend true/false to results.
1357  |   sub RA, BASE, #8
1358  |.else
1359  |   mov RA, BASE
1360  |   add RC, RC, #8
1361  |.endif
1362  |  ands CARG1, PC, #FRAME_TYPE
1363  |   str PC, SAVE_PC
1364  |   str RCw, SAVE_MULTRES
1365  |  beq ->BC_RET_Z
1366  |  b ->vm_return
1367  |
1368  |8:  // Coroutine returned with error (at co->top-1).
1369  |.if resume
1370  |  ldr TMP0, [CARG4, #-8]!
1371  |   mov_false TMP1
1372  |    mov RC, #(2+1)*8
1373  |  str CARG4, L:RA->top		// Remove error from coroutine stack.
1374  |  str TMP0, [BASE]			// Copy error message.
1375  |  b <7
1376  |.else
1377  |  mov CARG1, L
1378  |  mov CARG2, L:RA
1379  |  bl extern lj_ffh_coroutine_wrap_err  // (lua_State *L, lua_State *co)
1380  |  // Never returns.
1381  |.endif
1382  |
1383  |9:  // Handle stack expansion on return from yield.
1384  |  mov CARG1, L
1385  |  lsr CARG2, RC, #3
1386  |  bl extern lj_state_growstack	// (lua_State *L, int n)
1387  |  mov CRET1, #0
1388  |  b <4
1389  |.endmacro
1390  |
1391  |  coroutine_resume_wrap 1		// coroutine.resume
1392  |  coroutine_resume_wrap 0		// coroutine.wrap
1393  |
1394  |.ffunc coroutine_yield
1395  |  ldr TMP0, L->cframe
1396  |   add TMP1, BASE, NARGS8:RC
1397  |    mov CRET1, #LUA_YIELD
1398  |   stp BASE, TMP1, L->base
1399  |  tbz TMP0, #0, ->fff_fallback
1400  |   str xzr, L->cframe
1401  |    strb CRET1w, L->status
1402  |  b ->vm_leave_unw
1403  |
1404  |//-- Math library -------------------------------------------------------
1405  |
1406  |.macro math_round, func, round
1407  |  .ffunc math_ .. func
1408  |  ldr CARG1, [BASE]
1409  |   cmp NARGS8:RC, #8
1410  |  ldr d0, [BASE]
1411  |   blo ->fff_fallback
1412  |  cmp TISNUMhi, CARG1, lsr #32
1413  |  beq ->fff_restv
1414  |  blo ->fff_fallback
1415  |  round d0, d0
1416  |  b ->fff_resn
1417  |.endmacro
1418  |
1419  |  math_round floor, frintm
1420  |  math_round ceil, frintp
1421  |
1422  |.ffunc_1 math_abs
1423  |  checknumber CARG1, ->fff_fallback
1424  |  and CARG1, CARG1, #U64x(7fffffff,ffffffff)
1425  |  bne ->fff_restv
1426  |  eor CARG2w, CARG1w, CARG1w, asr #31
1427  |   movz CARG3, #0x41e0, lsl #48	// 2^31.
1428  |  subs CARG1w, CARG2w, CARG1w, asr #31
1429  |   add_TISNUM CARG1, CARG1
1430  |  csel CARG1, CARG1, CARG3, pl
1431  |  // Fallthrough.
1432  |
1433  |->fff_restv:
1434  |  // CARG1 = TValue result.
1435  |  ldr PC, [BASE, FRAME_PC]
1436  |  str CARG1, [BASE, #-16]
1437  |->fff_res1:
1438  |  // PC = return.
1439  |  mov RC, #(1+1)*8
1440  |->fff_res:
1441  |  // RC = (nresults+1)*8, PC = return.
1442  |  ands CARG1, PC, #FRAME_TYPE
1443  |   str RCw, SAVE_MULTRES
1444  |   sub RA, BASE, #16
1445  |  bne ->vm_return
1446  |  ldr INSw, [PC, #-4]
1447  |  decode_RB RB, INS
1448  |5:
1449  |  cmp RC, RB, lsl #3			// More results expected?
1450  |  blo >6
1451  |  decode_RA TMP1, INS
1452  |  // Adjust BASE. KBASE is assumed to be set for the calling frame.
1453  |  sub BASE, RA, TMP1, lsl #3
1454  |  ins_next
1455  |
1456  |6:  // Fill up results with nil.
1457  |  add TMP1, RA, RC
1458  |   add RC, RC, #8
1459  |  str TISNIL, [TMP1, #-8]
1460  |  b <5
1461  |
1462  |.macro math_extern, func
1463  |  .ffunc_n math_ .. func
1464  |  bl extern func
1465  |  b ->fff_resn
1466  |.endmacro
1467  |
1468  |.macro math_extern2, func
1469  |  .ffunc_nn math_ .. func
1470  |  bl extern func
1471  |  b ->fff_resn
1472  |.endmacro
1473  |
1474  |.ffunc_n math_sqrt
1475  |  fsqrt d0, d0
1476  |->fff_resn:
1477  |  ldr PC, [BASE, FRAME_PC]
1478  |  str d0, [BASE, #-16]
1479  |  b ->fff_res1
1480  |
1481  |.ffunc math_log
1482  |  ldr CARG1, [BASE]
1483  |   cmp NARGS8:RC, #8
1484  |  ldr FARG1, [BASE]
1485  |   bne ->fff_fallback			// Need exactly 1 argument.
1486  |  checknum CARG1, ->fff_fallback
1487  |  bl extern log
1488  |  b ->fff_resn
1489  |
1490  |  math_extern log10
1491  |  math_extern exp
1492  |  math_extern sin
1493  |  math_extern cos
1494  |  math_extern tan
1495  |  math_extern asin
1496  |  math_extern acos
1497  |  math_extern atan
1498  |  math_extern sinh
1499  |  math_extern cosh
1500  |  math_extern tanh
1501  |  math_extern2 pow
1502  |  math_extern2 atan2
1503  |  math_extern2 fmod
1504  |
1505  |.ffunc_2 math_ldexp
1506  |  ldr FARG1, [BASE]
1507  |  checknum CARG1, ->fff_fallback
1508  |  checkint CARG2, ->fff_fallback
1509  |  sxtw CARG1, CARG2w
1510  |  bl extern ldexp			// (double x, int exp)
1511  |  b ->fff_resn
1512  |
1513  |.ffunc_n math_frexp
1514  |  add CARG1, sp, TMPDofs
1515  |  bl extern frexp
1516  |   ldr CARG2w, TMPD
1517  |    ldr PC, [BASE, FRAME_PC]
1518  |  str d0, [BASE, #-16]
1519  |    mov RC, #(2+1)*8
1520  |   add_TISNUM CARG2, CARG2
1521  |   str CARG2, [BASE, #-8]
1522  |  b ->fff_res
1523  |
1524  |.ffunc_n math_modf
1525  |  sub CARG1, BASE, #16
1526  |   ldr PC, [BASE, FRAME_PC]
1527  |  bl extern modf
1528  |   mov RC, #(2+1)*8
1529  |  str d0, [BASE, #-8]
1530  |  b ->fff_res
1531  |
1532  |.macro math_minmax, name, cond, fcond
1533  |  .ffunc_1 name
1534  |   add RB, BASE, RC
1535  |   add RA, BASE, #8
1536  |  checkint CARG1, >4
1537  |1:  // Handle integers.
1538  |  ldr CARG2, [RA]
1539  |   cmp RA, RB
1540  |   bhs ->fff_restv
1541  |  checkint CARG2, >3
1542  |  cmp CARG1w, CARG2w
1543  |   add RA, RA, #8
1544  |  csel CARG1, CARG2, CARG1, cond
1545  |  b <1
1546  |3:  // Convert intermediate result to number and continue below.
1547  |  scvtf d0, CARG1w
1548  |  blo ->fff_fallback
1549  |  ldr d1, [RA]
1550  |  b >6
1551  |
1552  |4:
1553  |  ldr d0, [BASE]
1554  |  blo ->fff_fallback
1555  |5:  // Handle numbers.
1556  |  ldr CARG2, [RA]
1557  |  ldr d1, [RA]
1558  |   cmp RA, RB
1559  |   bhs ->fff_resn
1560  |  checknum CARG2, >7
1561  |6:
1562  |  fcmp d0, d1
1563  |   add RA, RA, #8
1564  |  fcsel d0, d1, d0, fcond
1565  |  b <5
1566  |7:  // Convert integer to number and continue above.
1567  |  scvtf d1, CARG2w
1568  |  blo ->fff_fallback
1569  |  b <6
1570  |.endmacro
1571  |
1572  |  math_minmax math_min, gt, pl
1573  |  math_minmax math_max, lt, le
1574  |
1575  |//-- String library -----------------------------------------------------
1576  |
1577  |.ffunc string_byte			// Only handle the 1-arg case here.
1578  |  ldp PC, CARG1, [BASE, FRAME_PC]
1579  |   cmp NARGS8:RC, #8
1580  |  asr ITYPE, CARG1, #47
1581  |  ccmn ITYPE, #-LJ_TSTR, #0, eq
1582  |   and STR:CARG1, CARG1, #LJ_GCVMASK
1583  |  bne ->fff_fallback
1584  |  ldrb TMP0w, STR:CARG1[1]		// Access is always ok (NUL at end).
1585  |   ldr CARG3w, STR:CARG1->len
1586  |  add_TISNUM TMP0, TMP0
1587  |  str TMP0, [BASE, #-16]
1588  |  mov RC, #(0+1)*8
1589  |   cbz CARG3, ->fff_res
1590  |  b ->fff_res1
1591  |
1592  |.ffunc string_char			// Only handle the 1-arg case here.
1593  |  ffgccheck
1594  |  ldp PC, CARG1, [BASE, FRAME_PC]
1595  |  cmp CARG1w, #255
1596  |   ccmp NARGS8:RC, #8, #0, ls		// Need exactly 1 argument.
1597  |  bne ->fff_fallback
1598  |  checkint CARG1, ->fff_fallback
1599  |  mov CARG3, #1
1600  |  // Point to the char inside the integer in the stack slot.
1601  |.if ENDIAN_LE
1602  |  mov CARG2, BASE
1603  |.else
1604  |  add CARG2, BASE, #7
1605  |.endif
1606  |->fff_newstr:
1607  |  // CARG2 = str, CARG3 = len.
1608  |   str BASE, L->base
1609  |  mov CARG1, L
1610  |   str PC, SAVE_PC
1611  |  bl extern lj_str_new		// (lua_State *L, char *str, size_t l)
1612  |->fff_resstr:
1613  |  // Returns GCstr *.
1614  |  ldr BASE, L->base
1615  |   movn TMP1, #~LJ_TSTR
1616  |  add CARG1, CARG1, TMP1, lsl #47
1617  |  b ->fff_restv
1618  |
1619  |.ffunc string_sub
1620  |  ffgccheck
1621  |  ldr CARG1, [BASE]
1622  |    ldr CARG3, [BASE, #16]
1623  |   cmp NARGS8:RC, #16
1624  |    movn RB, #0
1625  |   beq >1
1626  |   blo ->fff_fallback
1627  |    checkint CARG3, ->fff_fallback
1628  |    sxtw RB, CARG3w
1629  |1:
1630  |  ldr CARG2, [BASE, #8]
1631  |  checkstr CARG1, ->fff_fallback
1632  |   ldr TMP1w, STR:CARG1->len
1633  |  checkint CARG2, ->fff_fallback
1634  |  sxtw CARG2, CARG2w
1635  |  // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
1636  |   add TMP2, RB, TMP1
1637  |   cmp RB, #0
1638  |  add TMP0, CARG2, TMP1
1639  |   csinc RB, RB, TMP2, ge		// if (end < 0) end += len+1
1640  |  cmp CARG2, #0
1641  |  csinc CARG2, CARG2, TMP0, ge	// if (start < 0) start += len+1
1642  |   cmp RB, #0
1643  |   csel RB, RB, xzr, ge		// if (end < 0) end = 0
1644  |  cmp CARG2, #1
1645  |  csinc CARG2, CARG2, xzr, ge	// if (start < 1) start = 1
1646  |   cmp RB, TMP1
1647  |   csel RB, RB, TMP1, le		// if (end > len) end = len
1648  |  add CARG1, STR:CARG1, #sizeof(GCstr)-1
1649  |   subs CARG3, RB, CARG2		// len = end - start
1650  |  add CARG2, CARG1, CARG2
1651  |   add CARG3, CARG3, #1		// len += 1
1652  |   bge ->fff_newstr
1653  |  add STR:CARG1, GL, #offsetof(global_State, strempty)
1654  |   movn TMP1, #~LJ_TSTR
1655  |  add CARG1, CARG1, TMP1, lsl #47
1656  |  b ->fff_restv
1657  |
1658  |.macro ffstring_op, name
1659  |  .ffunc string_ .. name
1660  |  ffgccheck
1661  |  ldr CARG2, [BASE]
1662  |   cmp NARGS8:RC, #8
1663  |  asr ITYPE, CARG2, #47
1664  |  ccmn ITYPE, #-LJ_TSTR, #0, hs
1665  |   and STR:CARG2, CARG2, #LJ_GCVMASK
1666  |  bne ->fff_fallback
1667  |  ldr TMP0, GL->tmpbuf.b
1668  |   add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
1669  |   str BASE, L->base
1670  |   str PC, SAVE_PC
1671  |   str L, GL->tmpbuf.L
1672  |  str TMP0, GL->tmpbuf.w
1673  |  bl extern lj_buf_putstr_ .. name
1674  |  bl extern lj_buf_tostr
1675  |  b ->fff_resstr
1676  |.endmacro
1677  |
1678  |ffstring_op reverse
1679  |ffstring_op lower
1680  |ffstring_op upper
1681  |
1682  |//-- Bit library --------------------------------------------------------
1683  |
1684  |// FP number to bit conversion. Clobbers CARG1-CARG2, FARG1-FARG2.
1685  |->vm_tobit_fb:
1686  |  bls ->fff_fallback
1687  |  fmov FARG1, CARG1
1688  |  movz CARG2, #0x4338, lsl #48
1689  |  fmov FARG2, CARG2
1690  |  fadd FARG1, FARG1, FARG2
1691  |  fmov CARG1w, s0
1692  |  br lr
1693  |
1694  |.macro .ffunc_bit, name
1695  |  .ffunc_1 bit_..name
1696  |  adr lr, >1
1697  |  checkint CARG1, ->vm_tobit_fb
1698  |1:
1699  |  bti_jump
1700  |.endmacro
1701  |
1702  |.macro .ffunc_bit_op, name, ins
1703  |  .ffunc_bit name
1704  |  mov RA, #8
1705  |  mov TMP0w, CARG1w
1706  |  adr lr, >2
1707  |1:
1708  |  ldr CARG1, [BASE, RA]
1709  |   cmp RA, NARGS8:RC
1710  |    add RA, RA, #8
1711  |   bge >9
1712  |  checkint CARG1, ->vm_tobit_fb
1713  |2:
1714  |  bti_jump
1715  |  ins TMP0w, TMP0w, CARG1w
1716  |  b <1
1717  |.endmacro
1718  |
1719  |.ffunc_bit_op band, and
1720  |.ffunc_bit_op bor, orr
1721  |.ffunc_bit_op bxor, eor
1722  |
1723  |.ffunc_bit tobit
1724  |  mov TMP0w, CARG1w
1725  |9:  // Label reused by .ffunc_bit_op users.
1726  |  add_TISNUM CARG1, TMP0
1727  |  b ->fff_restv
1728  |
1729  |.ffunc_bit bswap
1730  |  rev TMP0w, CARG1w
1731  |  add_TISNUM CARG1, TMP0
1732  |  b ->fff_restv
1733  |
1734  |.ffunc_bit bnot
1735  |  mvn TMP0w, CARG1w
1736  |  add_TISNUM CARG1, TMP0
1737  |  b ->fff_restv
1738  |
1739  |.macro .ffunc_bit_sh, name, ins, shmod
1740  |  .ffunc bit_..name
1741  |  ldp TMP0, CARG1, [BASE]
1742  |   cmp NARGS8:RC, #16
1743  |   blo ->fff_fallback
1744  |  adr lr, >1
1745  |  checkint CARG1, ->vm_tobit_fb
1746  |1:
1747  |  bti_jump
1748  |.if shmod == 0
1749  |  mov TMP1, CARG1
1750  |.else
1751  |  neg TMP1, CARG1
1752  |.endif
1753  |  mov CARG1, TMP0
1754  |  adr lr, >2
1755  |  checkint CARG1, ->vm_tobit_fb
1756  |2:
1757  |  bti_jump
1758  |  ins TMP0w, CARG1w, TMP1w
1759  |  add_TISNUM CARG1, TMP0
1760  |  b ->fff_restv
1761  |.endmacro
1762  |
1763  |.ffunc_bit_sh lshift, lsl, 0
1764  |.ffunc_bit_sh rshift, lsr, 0
1765  |.ffunc_bit_sh arshift, asr, 0
1766  |.ffunc_bit_sh rol, ror, 1
1767  |.ffunc_bit_sh ror, ror, 0
1768  |
1769  |//-----------------------------------------------------------------------
1770  |
1771  |->fff_fallback:			// Call fast function fallback handler.
1772  |  // BASE = new base, RC = nargs*8
1773  |   ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC]	// Fallback may overwrite PC.
1774  |  ldr TMP2, L->maxstack
1775  |  add TMP1, BASE, NARGS8:RC
1776  |  stp BASE, TMP1, L->base
1777  |   and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1778  |  add TMP1, TMP1, #8*LUA_MINSTACK
1779  |   ldr CARG3, CFUNC:CARG3->f
1780  |    str PC, SAVE_PC			// Redundant (but a defined value).
1781  |  cmp TMP1, TMP2
1782  |   mov CARG1, L
1783  |  bhi >5				// Need to grow stack.
1784  |   blr_auth CARG3			// (lua_State *L)
1785  |  // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1786  |   ldr BASE, L->base
1787  |  cmp CRET1w, #0
1788  |   lsl RC, CRET1, #3
1789  |   sub RA, BASE, #16
1790  |  bgt ->fff_res			// Returned nresults+1?
1791  |1:  // Returned 0 or -1: retry fast path.
1792  |   ldr CARG1, L->top
1793  |    ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1794  |   sub NARGS8:RC, CARG1, BASE
1795  |  bne ->vm_call_tail			// Returned -1?
1796  |    and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1797  |  ins_callt				// Returned 0: retry fast path.
1798  |
1799  |// Reconstruct previous base for vmeta_call during tailcall.
1800  |->vm_call_tail:
1801  |  ands TMP0, PC, #FRAME_TYPE
1802  |   and TMP1, PC, #~FRAME_TYPEP
1803  |  bne >3
1804  |  ldrb RAw, [PC, #-4+OFS_RA]
1805  |  lsl RA, RA, #3
1806  |  add TMP1, RA, #16
1807  |3:
1808  |  sub RB, BASE, TMP1
1809  |  b ->vm_call_dispatch		// Resolve again for tailcall.
1810  |
1811  |5:  // Grow stack for fallback handler.
1812  |  mov CARG2, #LUA_MINSTACK
1813  |  bl extern lj_state_growstack	// (lua_State *L, int n)
1814  |  ldr BASE, L->base
1815  |  cmp CARG1, CARG1			// Set zero-flag to force retry.
1816  |  b <1
1817  |
1818  |->fff_gcstep:			// Call GC step function.
1819  |  // BASE = new base, RC = nargs*8
1820  |  sp_auth
1821  |   add CARG2, BASE, NARGS8:RC	// Calculate L->top.
1822  |  mov RA, lr
1823  |   stp BASE, CARG2, L->base
1824  |   str PC, SAVE_PC			// Redundant (but a defined value).
1825  |  mov CARG1, L
1826  |  bl extern lj_gc_step		// (lua_State *L)
1827  |  ldp BASE, CARG2, L->base
1828  |   ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1829  |  mov lr, RA				// Help return address predictor.
1830  |  sub NARGS8:RC, CARG2, BASE		// Calculate nargs*8.
1831  |   and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1832  |  ret_auth
1833  |
1834  |//-----------------------------------------------------------------------
1835  |//-- Special dispatch targets -------------------------------------------
1836  |//-----------------------------------------------------------------------
1837  |
1838  |->vm_record:				// Dispatch target for recording phase.
1839  |.if JIT
1840  |  bti_jump
1841  |  ldrb CARG1w, GL->hookmask
1842  |  tst CARG1, #HOOK_VMEVENT		// No recording while in vmevent.
1843  |  bne >5
1844  |  // Decrement the hookcount for consistency, but always do the call.
1845  |   ldr CARG2w, GL->hookcount
1846  |  tst CARG1, #HOOK_ACTIVE
1847  |  bne >1
1848  |   sub CARG2w, CARG2w, #1
1849  |  tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
1850  |  beq >1
1851  |   str CARG2w, GL->hookcount
1852  |  b >1
1853  |.endif
1854  |
1855  |->vm_rethook:			// Dispatch target for return hooks.
1856  |  bti_jump
1857  |  ldrb TMP2w, GL->hookmask
1858  |  tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1	// Hook already active?
1859  |5:  // Re-dispatch to static ins.
1860  |  ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1861  |  br_auth TMP0
1862  |
1863  |->vm_inshook:			// Dispatch target for instr/line hooks.
1864  |  bti_jump
1865  |  ldrb TMP2w, GL->hookmask
1866  |   ldr TMP3w, GL->hookcount
1867  |  tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5	// Hook already active?
1868  |  tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
1869  |  beq <5
1870  |   sub TMP3w, TMP3w, #1
1871  |   str TMP3w, GL->hookcount
1872  |   cbz TMP3w, >1
1873  |  tbz TMP2w, #LUA_HOOKLINE, <5
1874  |1:
1875  |  mov CARG1, L
1876  |   str BASE, L->base
1877  |  mov CARG2, PC
1878  |  // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1879  |  bl extern lj_dispatch_ins		// (lua_State *L, const BCIns *pc)
1880  |3:
1881  |  ldr BASE, L->base
1882  |4:  // Re-dispatch to static ins.
1883  |  ldr INSw, [PC, #-4]
1884  |  add TMP1, GL, INS, uxtb #3
1885  |   decode_RA RA, INS
1886  |  ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1887  |   decode_RD RC, INS
1888  |  br_auth TMP0
1889  |
1890  |->cont_hook:				// Continue from hook yield.
1891  |  bti_jump
1892  |  ldr CARG1, [CARG4, #-40]
1893  |   add PC, PC, #4
1894  |  str CARG1w, SAVE_MULTRES		// Restore MULTRES for *M ins.
1895  |  b <4
1896  |
1897  |->vm_hotloop:			// Hot loop counter underflow.
1898  |.if JIT
1899  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Same as curr_topL(L).
1900  |   add CARG1, GL, #GG_G2DISP+GG_DISP2J
1901  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1902  |   str PC, SAVE_PC
1903  |  ldr CARG3, LFUNC:CARG3->pc
1904  |   mov CARG2, PC
1905  |   str L, [GL, #GL_J(L)]
1906  |  ldrb CARG3w, [CARG3, #PC2PROTO(framesize)]
1907  |   str BASE, L->base
1908  |  add CARG3, BASE, CARG3, lsl #3
1909  |  str CARG3, L->top
1910  |  bl extern lj_trace_hot		// (jit_State *J, const BCIns *pc)
1911  |  b <3
1912  |.endif
1913  |
1914  |->vm_callhook:			// Dispatch target for call hooks.
1915  |  bti_jump
1916  |  mov CARG2, PC
1917  |.if JIT
1918  |  b >1
1919  |.endif
1920  |
1921  |->vm_hotcall:			// Hot call counter underflow.
1922  |.if JIT
1923  |  orr CARG2, PC, #1
1924  |1:
1925  |.endif
1926  |  add TMP1, BASE, NARGS8:RC
1927  |   str PC, SAVE_PC
1928  |   mov CARG1, L
1929  |   sub RA, RA, BASE
1930  |  stp BASE, TMP1, L->base
1931  |  bl extern lj_dispatch_call		// (lua_State *L, const BCIns *pc)
1932  |  // Returns ASMFunction.
1933  |  ldp BASE, TMP1, L->base
1934  |   str xzr, SAVE_PC			// Invalidate for subsequent line hook.
1935  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1936  |  add RA, BASE, RA
1937  |  sub NARGS8:RC, TMP1, BASE
1938  |   ldr INSw, [PC, #-4]
1939  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1940  |  br_auth CRET1
1941  |
1942  |->cont_stitch:			// Trace stitching.
1943  |.if JIT
1944  |  // RA = resultptr, CARG4 = meta base
1945  |  bti_jump
1946  |   ldr RBw, SAVE_MULTRES
1947  |  ldr INSw, [PC, #-4]
1948  |    ldr TRACE:CARG3, [CARG4, #-40]	// Save previous trace.
1949  |   subs RB, RB, #8
1950  |  decode_RA RC, INS			// Call base.
1951  |    and CARG3, CARG3, #LJ_GCVMASK
1952  |   beq >2
1953  |1:  // Move results down.
1954  |  ldr CARG1, [RA], #8
1955  |   subs RB, RB, #8
1956  |  str CARG1, [BASE, RC, lsl #3]
1957  |    add RC, RC, #1
1958  |   bne <1
1959  |2:
1960  |   decode_RA RA, INS
1961  |   decode_RB RB, INS
1962  |   add RA, RA, RB
1963  |3:
1964  |   cmp RA, RC
1965  |   bhi >9				// More results wanted?
1966  |
1967  |  ldrh RAw, TRACE:CARG3->traceno
1968  |  ldrh RCw, TRACE:CARG3->link
1969  |  cmp RCw, RAw
1970  |  beq ->cont_nop			// Blacklisted.
1971  |  cmp RCw, #0
1972  |  bne =>BC_JLOOP			// Jump to stitched trace.
1973  |
1974  |  // Stitch a new trace to the previous trace.
1975  |  mov CARG1, #GL_J(exitno)
1976  |  str RAw, [GL, CARG1]
1977  |  mov CARG1, #GL_J(L)
1978  |  str L, [GL, CARG1]
1979  |  str BASE, L->base
1980  |  add CARG1, GL, #GG_G2J
1981  |  mov CARG2, PC
1982  |  bl extern lj_dispatch_stitch	// (jit_State *J, const BCIns *pc)
1983  |  ldr BASE, L->base
1984  |  b ->cont_nop
1985  |
1986  |9:  // Fill up results with nil.
1987  |  str TISNIL, [BASE, RC, lsl #3]
1988  |  add RC, RC, #1
1989  |  b <3
1990  |.endif
1991  |
1992  |->vm_profhook:			// Dispatch target for profiler hook.
1993#if LJ_HASPROFILE
1994  |  bti_jump
1995  |  mov CARG1, L
1996  |   str BASE, L->base
1997  |  mov CARG2, PC
1998  |  bl extern lj_dispatch_profile	// (lua_State *L, const BCIns *pc)
1999  |  // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
2000  |  ldr BASE, L->base
2001  |  sub PC, PC, #4
2002  |  b ->cont_nop
2003#endif
2004  |
2005  |//-----------------------------------------------------------------------
2006  |//-- Trace exit handler -------------------------------------------------
2007  |//-----------------------------------------------------------------------
2008  |
2009  |.macro savex_, a, b
2010  |  stp d..a, d..b, [sp, #a*8]
2011  |  stp x..a, x..b, [sp, #32*8+a*8]
2012  |.endmacro
2013  |
2014  |->vm_exit_handler:
2015  |.if JIT
2016  |  bti_call
2017  |  sub     sp, sp, #(64*8)
2018  |  savex_, 0, 1
2019  |  savex_, 2, 3
2020  |  savex_, 4, 5
2021  |  savex_, 6, 7
2022  |  savex_, 8, 9
2023  |  savex_, 10, 11
2024  |  savex_, 12, 13
2025  |  savex_, 14, 15
2026  |  savex_, 16, 17
2027  |  savex_, 18, 19
2028  |  savex_, 20, 21
2029  |  savex_, 22, 23
2030  |  savex_, 24, 25
2031  |  savex_, 26, 27
2032  |  savex_, 28, 29
2033  |  stp d30, d31, [sp, #30*8]
2034  |  ldr CARG1, [sp, #64*8]	// Load original value of lr.
2035  |   add CARG3, sp, #64*8	// Recompute original value of sp.
2036  |    mv_vmstate CARG4w, EXIT
2037  |   stp xzr, CARG3, [sp, #62*8]	// Store 0/sp in RID_LR/RID_SP.
2038  |  sub CARG1, CARG1, lr
2039  |   ldr L, GL->cur_L
2040  |  lsr CARG1, CARG1, #2
2041  |   ldr BASE, GL->jit_base
2042  |  sub CARG1, CARG1, #2
2043  |   ldr CARG2w, [lr]		// Load trace number.
2044  |    st_vmstate CARG4w
2045  |.if ENDIAN_BE
2046  |   rev32 CARG2, CARG2
2047  |.endif
2048  |   str BASE, L->base
2049  |  ubfx CARG2w, CARG2w, #5, #16
2050  |  str CARG1w, [GL, #GL_J(exitno)]
2051  |   str CARG2w, [GL, #GL_J(parent)]
2052  |   str L, [GL, #GL_J(L)]
2053  |  str xzr, GL->jit_base
2054  |  add CARG1, GL, #GG_G2J
2055  |  mov CARG2, sp
2056  |  bl extern lj_trace_exit		// (jit_State *J, ExitState *ex)
2057  |  // Returns MULTRES (unscaled) or negated error code.
2058  |  ldr CARG2, L->cframe
2059  |   ldr BASE, L->base
2060  |  and sp, CARG2, #CFRAME_RAWMASK
2061  |   ldr PC, SAVE_PC			// Get SAVE_PC.
2062  |  str L, SAVE_L			// Set SAVE_L (on-trace resume/yield).
2063  |  b >1
2064  |.endif
2065  |
2066  |->vm_exit_interp:
2067  |  bti_jump
2068  |  // CARG1 = MULTRES or negated error code, BASE, PC and GL set.
2069  |.if JIT
2070  |  ldr L, SAVE_L
2071  |1:
2072  |   init_constants
2073  |  cmn CARG1w, #LUA_ERRERR
2074  |  bhs >9				// Check for error from exit.
2075  |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2076  |   lsl RC, CARG1, #3
2077  |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2078  |   str RCw, SAVE_MULTRES
2079  |   str BASE, L->base
2080  |  ldr CARG2, LFUNC:CARG2->pc
2081  |   str xzr, GL->jit_base
2082  |    mv_vmstate CARG4w, INTERP
2083  |  ldr KBASE, [CARG2, #PC2PROTO(k)]
2084  |  // Modified copy of ins_next which handles function header dispatch, too.
2085  |  ldrb RBw, [PC, # OFS_OP]
2086  |   ldr INSw, [PC], #4
2087  |    st_vmstate CARG4w
2088  |  cmn CARG1w, #17			// Static dispatch?
2089  |  beq >5
2090  |  cmp RBw, #BC_FUNCC+2		// Fast function?
2091  |   add TMP1, GL, INS, uxtb #3
2092  |  bhs >4
2093  |2:
2094  |  cmp RBw, #BC_FUNCF			// Function header?
2095  |  add TMP0, GL, RB, uxtb #3
2096  |  ldr RB, [TMP0, #GG_G2DISP]
2097  |   decode_RA RA, INS
2098  |   lsr TMP0, INS, #16
2099  |   csel RC, TMP0, RC, lo
2100  |   blo >3
2101  |   ldr CARG3, [BASE, FRAME_FUNC]
2102  |   sub RC, RC, #8
2103  |   add RA, BASE, RA, lsl #3	// Yes: RA = BASE+framesize*8, RC = nargs*8
2104  |   and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2105  |3:
2106  |  br_auth RB
2107  |
2108  |4:  // Check frame below fast function.
2109  |  ldr CARG1, [BASE, FRAME_PC]
2110  |  ands CARG2, CARG1, #FRAME_TYPE
2111  |  bne <2			// Trace stitching continuation?
2112  |  // Otherwise set KBASE for Lua function below fast function.
2113  |  ldr CARG3w, [CARG1, #-4]
2114  |  decode_RA CARG1, CARG3
2115  |  sub CARG2, BASE, CARG1, lsl #3
2116  |  ldr LFUNC:CARG3, [CARG2, #-32]
2117  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2118  |  ldr CARG3, LFUNC:CARG3->pc
2119  |  ldr KBASE, [CARG3, #PC2PROTO(k)]
2120  |  b <2
2121  |
2122  |5:  // Dispatch to static entry of original ins replaced by BC_JLOOP.
2123  |  ldr RA, [GL, #GL_J(trace)]
2124  |  decode_RD RC, INS
2125  |  ldr TRACE:RA, [RA, RC, lsl #3]
2126  |  ldr INSw, TRACE:RA->startins
2127  |  add TMP0, GL, INS, uxtb #3
2128  |   decode_RA RA, INS
2129  |  ldr RB, [TMP0, #GG_G2DISP+GG_DISP2STATIC]
2130  |   decode_RD RC, INS
2131  |  br_auth RB
2132  |
2133  |9:  // Rethrow error from the right C frame.
2134  |  neg CARG2w, CARG1w
2135  |  mov CARG1, L
2136  |  bl extern lj_err_trace		// (lua_State *L, int errcode)
2137  |.endif
2138  |
2139  |//-----------------------------------------------------------------------
2140  |//-- Math helper functions ----------------------------------------------
2141  |//-----------------------------------------------------------------------
2142  |
2143  |  // int lj_vm_modi(int dividend, int divisor);
2144  |->vm_modi:
2145  |  bti_call
2146  |    eor CARG4w, CARG1w, CARG2w
2147  |    cmp CARG4w, #0
2148  |  eor CARG3w, CARG1w, CARG1w, asr #31
2149  |   eor CARG4w, CARG2w, CARG2w, asr #31
2150  |  sub CARG3w, CARG3w, CARG1w, asr #31
2151  |   sub CARG4w, CARG4w, CARG2w, asr #31
2152  |  udiv CARG1w, CARG3w, CARG4w
2153  |  msub CARG1w, CARG1w, CARG4w, CARG3w
2154  |    ccmp CARG1w, #0, #4, mi
2155  |    sub CARG3w, CARG1w, CARG4w
2156  |    csel CARG1w, CARG1w, CARG3w, eq
2157  |  eor CARG3w, CARG1w, CARG2w
2158  |  cmp CARG3w, #0
2159  |  cneg CARG1w, CARG1w, mi
2160  |  ret
2161  |
2162  |//-----------------------------------------------------------------------
2163  |//-- Number conversion functions ----------------------------------------
2164  |//-----------------------------------------------------------------------
2165  |
2166  |// int64_t lj_vm_num2int_check(double x)
2167  |->vm_num2int_check:
2168  |  fcvtzs CRET1w, FARG1
2169  |  scvtf FARG2, CRET1w
2170  |  fcmp FARG2, FARG1
2171  |  bne >1
2172  |  ret
2173  |1:
2174  |  mov CRET1, #0x8000000080000000
2175  |  ret
2176  |
2177  |// int64_t lj_vm_num2i64(double x)
2178  |->vm_num2i64:
2179  |  fcvtzs CRET1, FARG1
2180  |  ret
2181  |
2182  |// uint64_t lj_vm_num2u64(double x)
2183  |->vm_num2u64:
2184  |  fcvtzs CRET1, FARG1
2185  |  fcvtzu CARG2, FARG1
2186  |  cmn CRET1, #1			// Set overflow if CRET1 == INT64_MAX.
2187  |  csel CRET1, CRET1, CARG2, vc	// No overflow ? i64 : u64.
2188  |  ret
2189  |
2190  |// int32_t lj_vm_tobit(double x)
2191  |->vm_tobit:
2192  |  movz CRET1, #0x4338, lsl #48	// 2^52 + 2^51.
2193  |  fmov FARG2, CRET1
2194  |  fadd FARG1, FARG1, FARG2
2195  |  fmov CRET1w, s0
2196  |  ret
2197  |
2198  |//-----------------------------------------------------------------------
2199  |//-- Miscellaneous functions --------------------------------------------
2200  |//-----------------------------------------------------------------------
2201  |
2202  |.define NEXT_TAB,		TAB:CARG1
2203  |.define NEXT_RES,		CARG1
2204  |.define NEXT_IDX,		CARG2w
2205  |.define NEXT_LIM,		CARG3w
2206  |.define NEXT_TMP0,		TMP0
2207  |.define NEXT_TMP0w,		TMP0w
2208  |.define NEXT_TMP1,		TMP1
2209  |.define NEXT_TMP1w,		TMP1w
2210  |.define NEXT_RES_PTR,	sp
2211  |.define NEXT_RES_VAL,	[sp]
2212  |.define NEXT_RES_KEY,	[sp, #8]
2213  |
2214  |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2215  |// Next idx returned in CRET2w.
2216  |->vm_next:
2217  |.if JIT
2218  |  bti_call
2219  |  ldr NEXT_LIM, NEXT_TAB->asize
2220  |   ldr NEXT_TMP1, NEXT_TAB->array
2221  |1:  // Traverse array part.
2222  |  subs NEXT_TMP0w, NEXT_IDX, NEXT_LIM
2223  |  bhs >5				// Index points after array part?
2224  |  ldr NEXT_TMP0, [NEXT_TMP1, NEXT_IDX, uxtw #3]
2225  |  cmn NEXT_TMP0, #-LJ_TNIL
2226  |   cinc NEXT_IDX, NEXT_IDX, eq
2227  |  beq <1				// Skip holes in array part.
2228  |  str NEXT_TMP0, NEXT_RES_VAL
2229  |   movz NEXT_TMP0w, #(LJ_TISNUM>>1)&0xffff, lsl #16
2230  |   stp NEXT_IDX, NEXT_TMP0w, NEXT_RES_KEY
2231  |  add NEXT_IDX, NEXT_IDX, #1
2232  |  mov NEXT_RES, NEXT_RES_PTR
2233  |4:
2234  |  ret
2235  |
2236  |5:  // Traverse hash part.
2237  |  ldr NEXT_TMP1w, NEXT_TAB->hmask
2238  |   ldr NODE:NEXT_RES, NEXT_TAB->node
2239  |   add NEXT_TMP0w, NEXT_TMP0w, NEXT_TMP0w, lsl #1
2240  |  add NEXT_LIM, NEXT_LIM, NEXT_TMP1w
2241  |   add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP0w, uxtw #3
2242  |6:
2243  |  cmp NEXT_IDX, NEXT_LIM
2244  |  bhi >9
2245  |  ldr NEXT_TMP0, NODE:NEXT_RES->val
2246  |  cmn NEXT_TMP0, #-LJ_TNIL
2247  |   add NEXT_IDX, NEXT_IDX, #1
2248  |  bne <4
2249  |  // Skip holes in hash part.
2250  |  add NODE:NEXT_RES, NODE:NEXT_RES, #sizeof(Node)
2251  |  b <6
2252  |
2253  |9:  // End of iteration. Set the key to nil (not the value).
2254  |  movn NEXT_TMP0, #0
2255  |  str NEXT_TMP0, NEXT_RES_KEY
2256  |  mov NEXT_RES, NEXT_RES_PTR
2257  |  ret
2258  |.endif
2259  |
2260  |//-----------------------------------------------------------------------
2261  |//-- FFI helper functions -----------------------------------------------
2262  |//-----------------------------------------------------------------------
2263  |
2264  |// Handler for callback functions.
2265  |// Saveregs already performed. Callback slot number in w9, g in x10.
2266  |->vm_ffi_callback:
2267  |.if FFI
2268  |.type CTSTATE, CTState, PC
2269  |  saveregs
2270  |  ldr CTSTATE, GL:x10->ctype_state
2271  |  mov GL, x10
2272  |    add x10, sp, # CFRAME_SPACE
2273  |  str w9, CTSTATE->cb.slot
2274  |  stp x0, x1, CTSTATE->cb.gpr[0]
2275  |   stp d0, d1, CTSTATE->cb.fpr[0]
2276  |  stp x2, x3, CTSTATE->cb.gpr[2]
2277  |   stp d2, d3, CTSTATE->cb.fpr[2]
2278  |  stp x4, x5, CTSTATE->cb.gpr[4]
2279  |   stp d4, d5, CTSTATE->cb.fpr[4]
2280  |  stp x6, x7, CTSTATE->cb.gpr[6]
2281  |   stp d6, d7, CTSTATE->cb.fpr[6]
2282  |    str x10, CTSTATE->cb.stack
2283  |  mov CARG1, CTSTATE
2284  |   str CTSTATE, SAVE_PC		// Any value outside of bytecode is ok.
2285  |  mov CARG2, sp
2286  |  bl extern lj_ccallback_enter	// (CTState *cts, void *cf)
2287  |  // Returns lua_State *.
2288  |  ldp BASE, RC, L:CRET1->base
2289  |   init_constants
2290  |   mov L, CRET1
2291  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2292  |  sub RC, RC, BASE
2293  |   st_vmstate ST_INTERP
2294  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2295  |  ins_callt
2296  |.endif
2297  |
2298  |->cont_ffi_callback:			// Return from FFI callback.
2299  |.if FFI
2300  |  ldr CTSTATE, GL->ctype_state
2301  |   stp BASE, CARG4, L->base
2302  |  str L, CTSTATE->L
2303  |  mov CARG1, CTSTATE
2304  |  mov CARG2, RA
2305  |  bl extern lj_ccallback_leave       // (CTState *cts, TValue *o)
2306  |  ldp x0, x1, CTSTATE->cb.gpr[0]
2307  |   ldp d0, d1, CTSTATE->cb.fpr[0]
2308  |  b ->vm_leave_unw
2309  |.endif
2310  |
2311  |->vm_ffi_call:			// Call C function via FFI.
2312  |  // Caveat: needs special frame unwinding, see below.
2313  |.if FFI
2314  |  .type CCSTATE, CCallState, x19
2315  |  sp_auth
2316  |  stp_unwind CCSTATE, x20, [sp, #-32]!
2317  |  stp fp, lr, [sp, #16]
2318  |  add fp, sp, #16
2319  |  mov CCSTATE, x0
2320  |  ldr TMP0w, CCSTATE:x0->spadj
2321  |   ldrb TMP1w, CCSTATE->nsp
2322  |    add TMP2, CCSTATE, #offsetof(CCallState, stack)
2323  |   subs TMP1, TMP1, #8
2324  |    ldr TMP3, CCSTATE->func
2325  |  sub sp, sp, TMP0
2326  |   bmi >2
2327  |1:  // Copy stack slots
2328  |  ldr TMP0, [TMP2, TMP1]
2329  |  str TMP0, [sp, TMP1]
2330  |  subs TMP1, TMP1, #8
2331  |  bpl <1
2332  |2:
2333  |  ldp x0, x1, CCSTATE->gpr[0]
2334  |   ldp d0, d1, CCSTATE->fpr[0]
2335  |  ldp x2, x3, CCSTATE->gpr[2]
2336  |   ldp d2, d3, CCSTATE->fpr[2]
2337  |  ldp x4, x5, CCSTATE->gpr[4]
2338  |   ldp d4, d5, CCSTATE->fpr[4]
2339  |  ldp x6, x7, CCSTATE->gpr[6]
2340  |   ldp d6, d7, CCSTATE->fpr[6]
2341  |  ldr x8, CCSTATE->retp
2342  |  blr_auth TMP3
2343  |  sub sp, fp, #16
2344  |  stp x0, x1, CCSTATE->gpr[0]
2345  |   stp d0, d1, CCSTATE->fpr[0]
2346  |   stp d2, d3, CCSTATE->fpr[2]
2347  |  ldp fp, lr, [sp, #16]
2348  |  ldp_unwind CCSTATE, x20, [sp], #32
2349  |  ret_auth
2350  |.endif
2351  |// Note: vm_ffi_call must be the last function in this object file!
2352  |
2353  |//-----------------------------------------------------------------------
2354}
2355
2356/* Generate the code for a single instruction. */
2357static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2358{
2359  int vk = 0;
2360  |=>defop:
2361
2362  switch (op) {
2363#if !LJ_HASJIT
2364  case BC_FORL:
2365  case BC_JFORI:
2366  case BC_JFORL:
2367  case BC_ITERL:
2368  case BC_JITERL:
2369  case BC_LOOP:
2370  case BC_JLOOP:
2371  case BC_FUNCF:
2372  case BC_JFUNCF:
2373  case BC_JFUNCV:
2374#endif
2375  case BC_FUNCV:  /* NYI: compiled vararg functions. */
2376    break;  /* Avoid redundant bti instructions. */
2377  default:
2378    |  bti_jump
2379    break;
2380  }
2381
2382  switch (op) {
2383
2384  /* -- Comparison ops ---------------------------------------------------- */
2385
2386  /* Remember: all ops branch for a true comparison, fall through otherwise. */
2387
2388  case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2389    |  // RA = src1, RC = src2, JMP with RC = target
2390    |  ldr CARG1, [BASE, RA, lsl #3]
2391    |    ldrh RBw, [PC, # OFS_RD]
2392    |   ldr CARG2, [BASE, RC, lsl #3]
2393    |    add PC, PC, #4
2394    |    add RB, PC, RB, lsl #2
2395    |    sub RB, RB, #0x20000
2396    |  checkint CARG1, >3
2397    |   checkint CARG2, >4
2398    |  cmp CARG1w, CARG2w
2399    if (op == BC_ISLT) {
2400      |  csel PC, RB, PC, lt
2401    } else if (op == BC_ISGE) {
2402      |  csel PC, RB, PC, ge
2403    } else if (op == BC_ISLE) {
2404      |  csel PC, RB, PC, le
2405    } else {
2406      |  csel PC, RB, PC, gt
2407    }
2408    |1:
2409    |  ins_next
2410    |
2411    |3:  // RA not int.
2412    |    ldr FARG1, [BASE, RA, lsl #3]
2413    |  blo ->vmeta_comp
2414    |    ldr FARG2, [BASE, RC, lsl #3]
2415    |   cmp TISNUMhi, CARG2, lsr #32
2416    |   bhi >5
2417    |   bne ->vmeta_comp
2418    |  // RA number, RC int.
2419    |  scvtf FARG2, CARG2w
2420    |  b >5
2421    |
2422    |4:  // RA int, RC not int
2423    |    ldr FARG2, [BASE, RC, lsl #3]
2424    |   blo ->vmeta_comp
2425    |  // RA int, RC number.
2426    |  scvtf FARG1, CARG1w
2427    |
2428    |5:  // RA number, RC number
2429    |  fcmp FARG1, FARG2
2430    |  // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2431    if (op == BC_ISLT) {
2432      |  csel PC, RB, PC, lo
2433    } else if (op == BC_ISGE) {
2434      |  csel PC, RB, PC, hs
2435    } else if (op == BC_ISLE) {
2436      |  csel PC, RB, PC, ls
2437    } else {
2438      |  csel PC, RB, PC, hi
2439    }
2440    |  b <1
2441    break;
2442
2443  case BC_ISEQV: case BC_ISNEV:
2444    vk = op == BC_ISEQV;
2445    |  // RA = src1, RC = src2, JMP with RC = target
2446    |  ldr CARG1, [BASE, RA, lsl #3]
2447    |   add RC, BASE, RC, lsl #3
2448    |    ldrh RBw, [PC, # OFS_RD]
2449    |   ldr CARG3, [RC]
2450    |    add PC, PC, #4
2451    |    add RB, PC, RB, lsl #2
2452    |    sub RB, RB, #0x20000
2453    |  asr ITYPE, CARG3, #47
2454    |  cmn ITYPE, #-LJ_TISNUM
2455    if (vk) {
2456      |  bls ->BC_ISEQN_Z
2457    } else {
2458      |  bls ->BC_ISNEN_Z
2459    }
2460    |  // RC is not a number.
2461    |   asr TMP0, CARG1, #47
2462    |.if FFI
2463    |  // Check if RC or RA is a cdata.
2464    |  cmn ITYPE, #-LJ_TCDATA
2465    |   ccmn TMP0, #-LJ_TCDATA, #4, ne
2466    |  beq ->vmeta_equal_cd
2467    |.endif
2468    |  cmp CARG1, CARG3
2469    |  bne >2
2470    |  // Tag and value are equal.
2471    if (vk) {
2472      |->BC_ISEQV_Z:
2473      |  mov PC, RB			// Perform branch.
2474    }
2475    |1:
2476    |  ins_next
2477    |
2478    |2:  // Check if the tags are the same and it's a table or userdata.
2479    |  cmp ITYPE, TMP0
2480    |  ccmn ITYPE, #-LJ_TISTABUD, #2, eq
2481    if (vk) {
2482      |  bhi <1
2483    } else {
2484      |  bhi ->BC_ISEQV_Z		// Reuse code from opposite instruction.
2485    }
2486    |  // Different tables or userdatas. Need to check __eq metamethod.
2487    |  // Field metatable must be at same offset for GCtab and GCudata!
2488    |  and TAB:CARG2, CARG1, #LJ_GCVMASK
2489    |  ldr TAB:TMP2, TAB:CARG2->metatable
2490    if (vk) {
2491      |  cbz TAB:TMP2, <1		// No metatable?
2492      |  ldrb TMP1w, TAB:TMP2->nomm
2493      |   mov CARG4, #0			// ne = 0
2494      |  tbnz TMP1w, #MM_eq, <1		// 'no __eq' flag set: done.
2495    } else {
2496      |  cbz TAB:TMP2, ->BC_ISEQV_Z	// No metatable?
2497      |  ldrb TMP1w, TAB:TMP2->nomm
2498      |   mov CARG4, #1			// ne = 1.
2499      |  tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z	// 'no __eq' flag set: done.
2500    }
2501    |  b ->vmeta_equal
2502    break;
2503
2504  case BC_ISEQS: case BC_ISNES:
2505    vk = op == BC_ISEQS;
2506    |  // RA = src, RC = str_const (~), JMP with RC = target
2507    |  ldr CARG1, [BASE, RA, lsl #3]
2508    |   mvn RC, RC
2509    |    ldrh RBw, [PC, # OFS_RD]
2510    |   ldr CARG2, [KBASE, RC, lsl #3]
2511    |    add PC, PC, #4
2512    |   movn TMP0, #~LJ_TSTR
2513    |.if FFI
2514    |  asr ITYPE, CARG1, #47
2515    |.endif
2516    |    add RB, PC, RB, lsl #2
2517    |   add CARG2, CARG2, TMP0, lsl #47
2518    |    sub RB, RB, #0x20000
2519    |.if FFI
2520    |  cmn ITYPE, #-LJ_TCDATA
2521    |  beq ->vmeta_equal_cd
2522    |.endif
2523    |  cmp CARG1, CARG2
2524    if (vk) {
2525      |  csel PC, RB, PC, eq
2526    } else {
2527      |  csel PC, RB, PC, ne
2528    }
2529    |  ins_next
2530    break;
2531
2532  case BC_ISEQN: case BC_ISNEN:
2533    vk = op == BC_ISEQN;
2534    |  // RA = src, RC = num_const (~), JMP with RC = target
2535    |  ldr CARG1, [BASE, RA, lsl #3]
2536    |   add RC, KBASE, RC, lsl #3
2537    |    ldrh RBw, [PC, # OFS_RD]
2538    |   ldr CARG3, [RC]
2539    |    add PC, PC, #4
2540    |    add RB, PC, RB, lsl #2
2541    |    sub RB, RB, #0x20000
2542    if (vk) {
2543      |->BC_ISEQN_Z:
2544    } else {
2545      |->BC_ISNEN_Z:
2546    }
2547    |  checkint CARG1, >4
2548    |   checkint CARG3, >6
2549    |  cmp CARG1w, CARG3w
2550    |1:
2551    if (vk) {
2552      |  csel PC, RB, PC, eq
2553      |2:
2554    } else {
2555      |2:
2556      |  csel PC, RB, PC, ne
2557    }
2558    |3:
2559    |  ins_next
2560    |
2561    |4:  // RA not int.
2562    |.if FFI
2563    |  blo >7
2564    |.else
2565    |  blo <2
2566    |.endif
2567    |    ldr FARG1, [BASE, RA, lsl #3]
2568    |    ldr FARG2, [RC]
2569    |   cmp TISNUMhi, CARG3, lsr #32
2570    |   bne >5
2571    |  // RA number, RC int.
2572    |  scvtf FARG2, CARG3w
2573    |5:
2574    |  // RA number, RC number.
2575    |  fcmp FARG1, FARG2
2576    |  b <1
2577    |
2578    |6:  // RA int, RC number
2579    |  ldr FARG2, [RC]
2580    |  scvtf FARG1, CARG1w
2581    |  fcmp FARG1, FARG2
2582    |  b <1
2583    |
2584    |.if FFI
2585    |7:
2586    |  asr ITYPE, CARG1, #47
2587    |  cmn ITYPE, #-LJ_TCDATA
2588    |  bne <2
2589    |  b ->vmeta_equal_cd
2590    |.endif
2591    break;
2592
2593  case BC_ISEQP: case BC_ISNEP:
2594    vk = op == BC_ISEQP;
2595    |  // RA = src, RC = primitive_type (~), JMP with RC = target
2596    |  ldr TMP0, [BASE, RA, lsl #3]
2597    |   ldrh RBw, [PC, # OFS_RD]
2598    |   add PC, PC, #4
2599    |  add RC, RC, #1
2600    |   add RB, PC, RB, lsl #2
2601    |.if FFI
2602    |  asr ITYPE, TMP0, #47
2603    |  cmn ITYPE, #-LJ_TCDATA
2604    |  beq ->vmeta_equal_cd
2605    |  cmn RC, ITYPE
2606    |.else
2607    |  cmn RC, TMP0, asr #47
2608    |.endif
2609    |   sub RB, RB, #0x20000
2610    if (vk) {
2611      |  csel PC, RB, PC, eq
2612    } else {
2613      |  csel PC, RB, PC, ne
2614    }
2615    |  ins_next
2616    break;
2617
2618  /* -- Unary test and copy ops ------------------------------------------- */
2619
2620  case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2621    |  // RA = dst or unused, RC = src, JMP with RC = target
2622    |   ldrh RBw, [PC, # OFS_RD]
2623    |  ldr TMP0, [BASE, RC, lsl #3]
2624    |   add PC, PC, #4
2625    |  mov_false TMP1
2626    |   add RB, PC, RB, lsl #2
2627    |  cmp TMP0, TMP1
2628    |   sub RB, RB, #0x20000
2629    if (op == BC_ISTC || op == BC_IST) {
2630      if (op == BC_ISTC) {
2631	|  csel RA, RA, RC, lo
2632      }
2633      |  csel PC, RB, PC, lo
2634    } else {
2635      if (op == BC_ISFC) {
2636	|  csel RA, RA, RC, hs
2637      }
2638      |  csel PC, RB, PC, hs
2639    }
2640    if (op == BC_ISTC || op == BC_ISFC) {
2641      |  str TMP0, [BASE, RA, lsl #3]
2642    }
2643    |  ins_next
2644    break;
2645
2646  case BC_ISTYPE:
2647    |  // RA = src, RC = -type
2648    |  ldr TMP0, [BASE, RA, lsl #3]
2649    |  cmn RC, TMP0, asr #47
2650    |  bne ->vmeta_istype
2651    |  ins_next
2652    break;
2653  case BC_ISNUM:
2654    |  // RA = src, RC = -(TISNUM-1)
2655    |  ldr TMP0, [BASE, RA]
2656    |  checknum TMP0, ->vmeta_istype
2657    |  ins_next
2658    break;
2659
2660  /* -- Unary ops --------------------------------------------------------- */
2661
2662  case BC_MOV:
2663    |  // RA = dst, RC = src
2664    |  ldr TMP0, [BASE, RC, lsl #3]
2665    |  str TMP0, [BASE, RA, lsl #3]
2666    |  ins_next
2667    break;
2668  case BC_NOT:
2669    |  // RA = dst, RC = src
2670    |  ldr TMP0, [BASE, RC, lsl #3]
2671    |   mov_false TMP1
2672    |   mov_true TMP2
2673    |  cmp TMP0, TMP1
2674    |  csel TMP0, TMP1, TMP2, lo
2675    |  str TMP0, [BASE, RA, lsl #3]
2676    |  ins_next
2677    break;
2678  case BC_UNM:
2679    |  // RA = dst, RC = src
2680    |  ldr TMP0, [BASE, RC, lsl #3]
2681    |  asr ITYPE, TMP0, #47
2682    |  cmn ITYPE, #-LJ_TISNUM
2683    |  bhi ->vmeta_unm
2684    |  eor TMP0, TMP0, #U64x(80000000,00000000)
2685    |  bne >5
2686    |  negs TMP0w, TMP0w
2687    |   movz CARG3, #0x41e0, lsl #48	// 2^31.
2688    |   add_TISNUM TMP0, TMP0
2689    |  csel TMP0, TMP0, CARG3, vc
2690    |   movz CARG3, #0x8000, lsl #48	// -0.
2691    |  csel TMP0, TMP0, CARG3, ne
2692    |5:
2693    |  str TMP0, [BASE, RA, lsl #3]
2694    |  ins_next
2695    break;
2696  case BC_LEN:
2697    |  // RA = dst, RC = src
2698    |  ldr CARG1, [BASE, RC, lsl #3]
2699    |  asr ITYPE, CARG1, #47
2700    |  cmn ITYPE, #-LJ_TSTR
2701    |   and CARG1, CARG1, #LJ_GCVMASK
2702    |  bne >2
2703    |  ldr CARG1w, STR:CARG1->len
2704    |1:
2705    |  add_TISNUM CARG1, CARG1
2706    |  str CARG1, [BASE, RA, lsl #3]
2707    |  ins_next
2708    |
2709    |2:
2710    |  cmn ITYPE, #-LJ_TTAB
2711    |  bne ->vmeta_len
2712#if LJ_52
2713    |  ldr TAB:CARG2, TAB:CARG1->metatable
2714    |  cbnz TAB:CARG2, >9
2715    |3:
2716#endif
2717    |->BC_LEN_Z:
2718    |  bl extern lj_tab_len		// (GCtab *t)
2719    |  // Returns uint32_t (but less than 2^31).
2720    |  b <1
2721    |
2722#if LJ_52
2723    |9:
2724    |  ldrb TMP1w, TAB:CARG2->nomm
2725    |  tbnz TMP1w, #MM_len, <3		// 'no __len' flag set: done.
2726    |  b ->vmeta_len
2727#endif
2728    break;
2729
2730  /* -- Binary ops -------------------------------------------------------- */
2731
2732    |.macro ins_arithcheck_int, target
2733    |  checkint CARG1, target
2734    |  checkint CARG2, target
2735    |.endmacro
2736    |
2737    |.macro ins_arithcheck_num, target
2738    |  checknum CARG1, target
2739    |  checknum CARG2, target
2740    |.endmacro
2741    |
2742    |.macro ins_arithcheck_nzdiv, target
2743    |  cbz CARG2w, target
2744    |.endmacro
2745    |
2746    |.macro ins_arithhead
2747    ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2748    ||if (vk == 1) {
2749    |   and RC, RC, #255
2750    |    decode_RB RB, INS
2751    ||} else {
2752    |   decode_RB RB, INS
2753    |    and RC, RC, #255
2754    ||}
2755    |.endmacro
2756    |
2757    |.macro ins_arithload, reg1, reg2
2758    |  // RA = dst, RB = src1, RC = src2 | num_const
2759    ||switch (vk) {
2760    ||case 0:
2761    |   ldr reg1, [BASE, RB, lsl #3]
2762    |    ldr reg2, [KBASE, RC, lsl #3]
2763    ||  break;
2764    ||case 1:
2765    |   ldr reg1, [KBASE, RC, lsl #3]
2766    |    ldr reg2, [BASE, RB, lsl #3]
2767    ||  break;
2768    ||default:
2769    |   ldr reg1, [BASE, RB, lsl #3]
2770    |    ldr reg2, [BASE, RC, lsl #3]
2771    ||  break;
2772    ||}
2773    |.endmacro
2774    |
2775    |.macro ins_arithfallback, ins
2776    ||switch (vk) {
2777    ||case 0:
2778    |   ins ->vmeta_arith_vn
2779    ||  break;
2780    ||case 1:
2781    |   ins ->vmeta_arith_nv
2782    ||  break;
2783    ||default:
2784    |   ins ->vmeta_arith_vv
2785    ||  break;
2786    ||}
2787    |.endmacro
2788    |
2789    |.macro ins_arithmod, res, reg1, reg2
2790    |  fdiv d2, reg1, reg2
2791    |  frintm d2, d2
2792    |  // Cannot use fmsub, because FMA is not enabled by default.
2793    |  fmul d2, d2, reg2
2794    |  fsub res, reg1, d2
2795    |.endmacro
2796    |
2797    |.macro ins_arithdn, intins, fpins
2798    |  ins_arithhead
2799    |  ins_arithload CARG1, CARG2
2800    |  ins_arithcheck_int >5
2801    |.if "intins" == "smull"
2802    |  smull CARG1, CARG1w, CARG2w
2803    |  cmp CARG1, CARG1, sxtw
2804    |   mov CARG1w, CARG1w
2805    |  ins_arithfallback bne
2806    |.elif "intins" == "ins_arithmodi"
2807    |  ins_arithfallback ins_arithcheck_nzdiv
2808    |  bl ->vm_modi
2809    |.else
2810    |  intins CARG1w, CARG1w, CARG2w
2811    |  ins_arithfallback bvs
2812    |.endif
2813    |  add_TISNUM CARG1, CARG1
2814    |  str CARG1, [BASE, RA, lsl #3]
2815    |4:
2816    |  ins_next
2817    |
2818    |5:  // FP variant.
2819    |  ins_arithload FARG1, FARG2
2820    |  ins_arithfallback ins_arithcheck_num
2821    |  fpins FARG1, FARG1, FARG2
2822    |  str FARG1, [BASE, RA, lsl #3]
2823    |  b <4
2824    |.endmacro
2825    |
2826    |.macro ins_arithfp, fpins
2827    |  ins_arithhead
2828    |  ins_arithload CARG1, CARG2
2829    |  ins_arithload FARG1, FARG2
2830    |  ins_arithfallback ins_arithcheck_num
2831    |.if "fpins" == "fpow"
2832    |  bl extern pow
2833    |.else
2834    |  fpins FARG1, FARG1, FARG2
2835    |.endif
2836    |  str FARG1, [BASE, RA, lsl #3]
2837    |  ins_next
2838    |.endmacro
2839
2840  case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2841    |  ins_arithdn adds, fadd
2842    break;
2843  case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2844    |  ins_arithdn subs, fsub
2845    break;
2846  case BC_MULVN: case BC_MULNV: case BC_MULVV:
2847    |  ins_arithdn smull, fmul
2848    break;
2849  case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2850    |  ins_arithfp fdiv
2851    break;
2852  case BC_MODVN: case BC_MODNV: case BC_MODVV:
2853    |  ins_arithdn ins_arithmodi, ins_arithmod
2854    break;
2855  case BC_POW:
2856    |  // NYI: (partial) integer arithmetic.
2857    |  ins_arithfp fpow
2858    break;
2859
2860  case BC_CAT:
2861    |  decode_RB RB, INS
2862    |   and RC, RC, #255
2863    |  // RA = dst, RB = src_start, RC = src_end
2864    |   str BASE, L->base
2865    |  sub CARG3, RC, RB
2866    |  add CARG2, BASE, RC, lsl #3
2867    |->BC_CAT_Z:
2868    |  // RA = dst, CARG2 = top-1, CARG3 = left
2869    |  mov CARG1, L
2870    |   str PC, SAVE_PC
2871    |  bl extern lj_meta_cat		// (lua_State *L, TValue *top, int left)
2872    |  // Returns NULL (finished) or TValue * (metamethod).
2873    |  ldrb RBw, [PC, #-4+OFS_RB]
2874    |   ldr BASE, L->base
2875    |   cbnz CRET1, ->vmeta_binop
2876    |  ldr TMP0, [BASE, RB, lsl #3]
2877    |  str TMP0, [BASE, RA, lsl #3]	// Copy result to RA.
2878    |  ins_next
2879    break;
2880
2881  /* -- Constant ops ------------------------------------------------------ */
2882
2883  case BC_KSTR:
2884    |  // RA = dst, RC = str_const (~)
2885    |  mvn RC, RC
2886    |  ldr TMP0, [KBASE, RC, lsl #3]
2887    |   movn TMP1, #~LJ_TSTR
2888    |  add TMP0, TMP0, TMP1, lsl #47
2889    |  str TMP0, [BASE, RA, lsl #3]
2890    |  ins_next
2891    break;
2892  case BC_KCDATA:
2893    |.if FFI
2894    |  // RA = dst, RC = cdata_const (~)
2895    |  mvn RC, RC
2896    |  ldr TMP0, [KBASE, RC, lsl #3]
2897    |   movn TMP1, #~LJ_TCDATA
2898    |  add TMP0, TMP0, TMP1, lsl #47
2899    |  str TMP0, [BASE, RA, lsl #3]
2900    |  ins_next
2901    |.endif
2902    break;
2903  case BC_KSHORT:
2904    |  // RA = dst, RC = int16_literal
2905    |  sxth RCw, RCw
2906    |  add_TISNUM TMP0, RC
2907    |  str TMP0, [BASE, RA, lsl #3]
2908    |  ins_next
2909    break;
2910  case BC_KNUM:
2911    |  // RA = dst, RC = num_const
2912    |  ldr TMP0, [KBASE, RC, lsl #3]
2913    |  str TMP0, [BASE, RA, lsl #3]
2914    |  ins_next
2915    break;
2916  case BC_KPRI:
2917    |  // RA = dst, RC = primitive_type (~)
2918    |  mvn TMP0, RC, lsl #47
2919    |  str TMP0, [BASE, RA, lsl #3]
2920    |  ins_next
2921    break;
2922  case BC_KNIL:
2923    |  // RA = base, RC = end
2924    |  add RA, BASE, RA, lsl #3
2925    |   add RC, BASE, RC, lsl #3
2926    |  str TISNIL, [RA], #8
2927    |1:
2928    |   cmp RA, RC
2929    |  str TISNIL, [RA], #8
2930    |   blt <1
2931    |  ins_next_
2932    break;
2933
2934  /* -- Upvalue and function ops ------------------------------------------ */
2935
2936  case BC_UGET:
2937    |  // RA = dst, RC = uvnum
2938    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2939    |   add RC, RC, #offsetof(GCfuncL, uvptr)/8
2940    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2941    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
2942    |  ldr CARG2, UPVAL:CARG2->v
2943    |  ldr TMP0, [CARG2]
2944    |  str TMP0, [BASE, RA, lsl #3]
2945    |  ins_next
2946    break;
2947  case BC_USETV:
2948    |  // RA = uvnum, RC = src
2949    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2950    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
2951    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2952    |  ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2953    |   ldr CARG3, [BASE, RC, lsl #3]
2954    |    ldr CARG2, UPVAL:CARG1->v
2955    |  ldrb TMP2w, UPVAL:CARG1->marked
2956    |  ldrb TMP0w, UPVAL:CARG1->closed
2957    |    asr ITYPE, CARG3, #47
2958    |   str CARG3, [CARG2]
2959    |    add ITYPE, ITYPE, #-LJ_TISGCV
2960    |  tst TMP2w, #LJ_GC_BLACK		// isblack(uv)
2961    |  ccmp TMP0w, #0, #4, ne		// && uv->closed
2962    |    ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne	// && tvisgcv(v)
2963    |  bhi >2
2964    |1:
2965    |  ins_next
2966    |
2967    |2:  // Check if new value is white.
2968    |  and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
2969    |  ldrb TMP1w, GCOBJ:CARG3->gch.marked
2970    |  tst TMP1w, #LJ_GC_WHITES		// iswhite(str)
2971    |  beq <1
2972    |  // Crossed a write barrier. Move the barrier forward.
2973    |  mov CARG1, GL
2974    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
2975    |  b <1
2976    break;
2977  case BC_USETS:
2978    |  // RA = uvnum, RC = str_const (~)
2979    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2980    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
2981    |    mvn RC, RC
2982    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2983    |  ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2984    |   ldr STR:CARG3, [KBASE, RC, lsl #3]
2985    |   movn TMP0, #~LJ_TSTR
2986    |    ldr CARG2, UPVAL:CARG1->v
2987    |  ldrb TMP2w, UPVAL:CARG1->marked
2988    |   add TMP0, STR:CARG3, TMP0, lsl #47
2989    |    ldrb TMP1w, STR:CARG3->marked
2990    |   str TMP0, [CARG2]
2991    |  tbnz TMP2w, #2, >2		// isblack(uv)
2992    |1:
2993    |  ins_next
2994    |
2995    |2:  // Check if string is white and ensure upvalue is closed.
2996    |  ldrb TMP0w, UPVAL:CARG1->closed
2997    |    tst TMP1w, #LJ_GC_WHITES	// iswhite(str)
2998    |  ccmp TMP0w, #0, #4, ne
2999    |  beq <1
3000    |  // Crossed a write barrier. Move the barrier forward.
3001    |  mov CARG1, GL
3002    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
3003    |  b <1
3004    break;
3005  case BC_USETN:
3006    |  // RA = uvnum, RC = num_const
3007    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3008    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
3009    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
3010    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
3011    |   ldr TMP0, [KBASE, RC, lsl #3]
3012    |  ldr CARG2, UPVAL:CARG2->v
3013    |   str TMP0, [CARG2]
3014    |  ins_next
3015    break;
3016  case BC_USETP:
3017    |  // RA = uvnum, RC = primitive_type (~)
3018    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
3019    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
3020    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
3021    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
3022    |   mvn TMP0, RC, lsl #47
3023    |  ldr CARG2, UPVAL:CARG2->v
3024    |   str TMP0, [CARG2]
3025    |  ins_next
3026    break;
3027
3028  case BC_UCLO:
3029    |  // RA = level, RC = target
3030    |  ldr CARG3, L->openupval
3031    |   add RC, PC, RC, lsl #2
3032    |    str BASE, L->base
3033    |   sub PC, RC, #0x20000
3034    |  cbz CARG3, >1
3035    |  mov CARG1, L
3036    |  add CARG2, BASE, RA, lsl #3
3037    |  bl extern lj_func_closeuv	// (lua_State *L, TValue *level)
3038    |  ldr BASE, L->base
3039    |1:
3040    |  ins_next
3041    break;
3042
3043  case BC_FNEW:
3044    |  // RA = dst, RC = proto_const (~) (holding function prototype)
3045    |  mvn RC, RC
3046    |   str BASE, L->base
3047    |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
3048    |    str PC, SAVE_PC
3049    |   ldr CARG2, [KBASE, RC, lsl #3]
3050    |    mov CARG1, L
3051    |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
3052    |  // (lua_State *L, GCproto *pt, GCfuncL *parent)
3053    |  bl extern lj_func_newL_gc
3054    |  // Returns GCfuncL *.
3055    |  ldr BASE, L->base
3056    |   movn TMP0, #~LJ_TFUNC
3057    |   add CRET1, CRET1, TMP0, lsl #47
3058    |  str CRET1, [BASE, RA, lsl #3]
3059    |  ins_next
3060    break;
3061
3062  /* -- Table ops --------------------------------------------------------- */
3063
3064  case BC_TNEW:
3065  case BC_TDUP:
3066    |  // RA = dst, RC = (hbits|asize) | tab_const (~)
3067    |  ldp CARG3, CARG4, GL->gc.total	// Assumes threshold follows total.
3068    |   str BASE, L->base
3069    |   str PC, SAVE_PC
3070    |   mov CARG1, L
3071    |  cmp CARG3, CARG4
3072    |  bhs >5
3073    |1:
3074    if (op == BC_TNEW) {
3075      |  and CARG2, RC, #0x7ff
3076      |   lsr CARG3, RC, #11
3077      |  cmp CARG2, #0x7ff
3078      |  mov TMP0, #0x801
3079      |  csel CARG2, CARG2, TMP0, ne
3080      |  bl extern lj_tab_new  // (lua_State *L, int32_t asize, uint32_t hbits)
3081      |  // Returns GCtab *.
3082    } else {
3083      |  mvn RC, RC
3084      |  ldr CARG2, [KBASE, RC, lsl #3]
3085      |  bl extern lj_tab_dup  // (lua_State *L, Table *kt)
3086      |  // Returns GCtab *.
3087    }
3088    |  ldr BASE, L->base
3089    |   movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
3090    |  str CRET1, [BASE, RA, lsl #3]
3091    |  ins_next
3092    |
3093    |5:
3094    |  bl extern lj_gc_step_fixtop  // (lua_State *L)
3095    |  mov CARG1, L
3096    |  b <1
3097    break;
3098
3099  case BC_GGET:
3100    |  // RA = dst, RC = str_const (~)
3101  case BC_GSET:
3102    |  // RA = src, RC = str_const (~)
3103    |  ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3104    |   mvn RC, RC
3105    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3106    |  ldr TAB:CARG2, LFUNC:CARG1->env
3107    |   ldr STR:RC, [KBASE, RC, lsl #3]
3108    if (op == BC_GGET) {
3109      |  b ->BC_TGETS_Z
3110    } else {
3111      |  b ->BC_TSETS_Z
3112    }
3113    break;
3114
3115  case BC_TGETV:
3116    |  decode_RB RB, INS
3117    |   and RC, RC, #255
3118    |  // RA = dst, RB = table, RC = key
3119    |  ldr CARG2, [BASE, RB, lsl #3]
3120    |   ldr TMP1, [BASE, RC, lsl #3]
3121    |  checktab CARG2, ->vmeta_tgetv
3122    |  checkint TMP1, >9		// Integer key?
3123    |  ldr CARG3, TAB:CARG2->array
3124    |   ldr CARG1w, TAB:CARG2->asize
3125    |  add CARG3, CARG3, TMP1, uxtw #3
3126    |   cmp TMP1w, CARG1w		// In array part?
3127    |   bhs ->vmeta_tgetv
3128    |  ldr TMP0, [CARG3]
3129    |  cmp_nil TMP0
3130    |  beq >5
3131    |1:
3132    |  str TMP0, [BASE, RA, lsl #3]
3133    |  ins_next
3134    |
3135    |5:  // Check for __index if table value is nil.
3136    |  ldr TAB:CARG1, TAB:CARG2->metatable
3137    |  cbz TAB:CARG1, <1		// No metatable: done.
3138    |  ldrb TMP1w, TAB:CARG1->nomm
3139    |  tbnz TMP1w, #MM_index, <1	// 'no __index' flag set: done.
3140    |  b ->vmeta_tgetv
3141    |
3142    |9:
3143    |  asr ITYPE, TMP1, #47
3144    |  cmn ITYPE, #-LJ_TSTR		// String key?
3145    |  bne ->vmeta_tgetv
3146    |   and STR:RC, TMP1, #LJ_GCVMASK
3147    |  b ->BC_TGETS_Z
3148    break;
3149  case BC_TGETS:
3150    |  decode_RB RB, INS
3151    |   and RC, RC, #255
3152    |  // RA = dst, RB = table, RC = str_const (~)
3153    |  ldr CARG2, [BASE, RB, lsl #3]
3154    |   mvn RC, RC
3155    |   ldr STR:RC, [KBASE, RC, lsl #3]
3156    |  checktab CARG2, ->vmeta_tgets1
3157    |->BC_TGETS_Z:
3158    |  // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
3159    |  ldr TMP1w, TAB:CARG2->hmask
3160    |   ldr TMP2w, STR:RC->sid
3161    |    ldr NODE:CARG3, TAB:CARG2->node
3162    |  and TMP1w, TMP1w, TMP2w		// idx = str->sid & tab->hmask
3163    |  add TMP1, TMP1, TMP1, lsl #1
3164    |  movn CARG4, #~LJ_TSTR
3165    |    add NODE:CARG3, NODE:CARG3, TMP1, lsl #3  // node = tab->node + idx*3*8
3166    |  add CARG4, STR:RC, CARG4, lsl #47	// Tagged key to look for.
3167    |1:
3168    |  ldp TMP0, CARG1, NODE:CARG3->val
3169    |   ldr NODE:CARG3, NODE:CARG3->next
3170    |  cmp CARG1, CARG4
3171    |  bne >4
3172    |  cmp_nil TMP0
3173    |  beq >5
3174    |3:
3175    |  str TMP0, [BASE, RA, lsl #3]
3176    |  ins_next
3177    |
3178    |4:  // Follow hash chain.
3179    |  cbnz NODE:CARG3, <1
3180    |  // End of hash chain: key not found, nil result.
3181    |   mov_nil TMP0
3182    |
3183    |5:  // Check for __index if table value is nil.
3184    |  ldr TAB:CARG1, TAB:CARG2->metatable
3185    |  cbz TAB:CARG1, <3		// No metatable: done.
3186    |  ldrb TMP1w, TAB:CARG1->nomm
3187    |  tbnz TMP1w, #MM_index, <3	// 'no __index' flag set: done.
3188    |  b ->vmeta_tgets
3189    break;
3190  case BC_TGETB:
3191    |  decode_RB RB, INS
3192    |   and RC, RC, #255
3193    |  // RA = dst, RB = table, RC = index
3194    |  ldr CARG2, [BASE, RB, lsl #3]
3195    |  checktab CARG2, ->vmeta_tgetb
3196    |  ldr CARG3, TAB:CARG2->array
3197    |   ldr CARG1w, TAB:CARG2->asize
3198    |  add CARG3, CARG3, RC, lsl #3
3199    |   cmp RCw, CARG1w			// In array part?
3200    |   bhs ->vmeta_tgetb
3201    |  ldr TMP0, [CARG3]
3202    |  cmp_nil TMP0
3203    |  beq >5
3204    |1:
3205    |  str TMP0, [BASE, RA, lsl #3]
3206    |  ins_next
3207    |
3208    |5:  // Check for __index if table value is nil.
3209    |  ldr TAB:CARG1, TAB:CARG2->metatable
3210    |  cbz TAB:CARG1, <1		// No metatable: done.
3211    |  ldrb TMP1w, TAB:CARG1->nomm
3212    |  tbnz TMP1w, #MM_index, <1	// 'no __index' flag set: done.
3213    |  b ->vmeta_tgetb
3214    break;
3215  case BC_TGETR:
3216    |  decode_RB RB, INS
3217    |   and RC, RC, #255
3218    |  // RA = dst, RB = table, RC = key
3219    |  ldr CARG1, [BASE, RB, lsl #3]
3220    |   ldr TMP1, [BASE, RC, lsl #3]
3221    |  and TAB:CARG1, CARG1, #LJ_GCVMASK
3222    |  ldr CARG3, TAB:CARG1->array
3223    |   ldr TMP2w, TAB:CARG1->asize
3224    |  add CARG3, CARG3, TMP1w, uxtw #3
3225    |   cmp TMP1w, TMP2w		// In array part?
3226    |   bhs ->vmeta_tgetr
3227    |  ldr TMP0, [CARG3]
3228    |->BC_TGETR_Z:
3229    |  str TMP0, [BASE, RA, lsl #3]
3230    |  ins_next
3231    break;
3232
3233  case BC_TSETV:
3234    |  decode_RB RB, INS
3235    |   and RC, RC, #255
3236    |  // RA = src, RB = table, RC = key
3237    |  ldr CARG2, [BASE, RB, lsl #3]
3238    |   ldr TMP1, [BASE, RC, lsl #3]
3239    |  checktab CARG2, ->vmeta_tsetv
3240    |  checkint TMP1, >9		// Integer key?
3241    |  ldr CARG3, TAB:CARG2->array
3242    |   ldr CARG1w, TAB:CARG2->asize
3243    |  add CARG3, CARG3, TMP1, uxtw #3
3244    |   cmp TMP1w, CARG1w		// In array part?
3245    |   bhs ->vmeta_tsetv
3246    |  ldr TMP1, [CARG3]
3247    |   ldr TMP0, [BASE, RA, lsl #3]
3248    |    ldrb TMP2w, TAB:CARG2->marked
3249    |  cmp_nil TMP1			// Previous value is nil?
3250    |  beq >5
3251    |1:
3252    |   str TMP0, [CARG3]
3253    |    tbnz TMP2w, #2, >7		// isblack(table)
3254    |2:
3255    |   ins_next
3256    |
3257    |5:  // Check for __newindex if previous value is nil.
3258    |  ldr TAB:CARG1, TAB:CARG2->metatable
3259    |  cbz TAB:CARG1, <1		// No metatable: done.
3260    |  ldrb TMP1w, TAB:CARG1->nomm
3261    |  tbnz TMP1w, #MM_newindex, <1	// 'no __newindex' flag set: done.
3262    |  b ->vmeta_tsetv
3263    |
3264    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3265    |  barrierback TAB:CARG2, TMP2w, TMP1
3266    |  b <2
3267    |
3268    |9:
3269    |  asr ITYPE, TMP1, #47
3270    |  cmn ITYPE, #-LJ_TSTR		// String key?
3271    |  bne ->vmeta_tsetv
3272    |   and STR:RC, TMP1, #LJ_GCVMASK
3273    |  b ->BC_TSETS_Z
3274    break;
3275  case BC_TSETS:
3276    |  decode_RB RB, INS
3277    |   and RC, RC, #255
3278    |  // RA = dst, RB = table, RC = str_const (~)
3279    |  ldr CARG2, [BASE, RB, lsl #3]
3280    |   mvn RC, RC
3281    |   ldr STR:RC, [KBASE, RC, lsl #3]
3282    |  checktab CARG2, ->vmeta_tsets1
3283    |->BC_TSETS_Z:
3284    |  // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
3285    |  ldr TMP1w, TAB:CARG2->hmask
3286    |   ldr TMP2w, STR:RC->sid
3287    |    ldr NODE:CARG3, TAB:CARG2->node
3288    |  and TMP1w, TMP1w, TMP2w		// idx = str->sid & tab->hmask
3289    |  add TMP1, TMP1, TMP1, lsl #1
3290    |  movn CARG4, #~LJ_TSTR
3291    |    add NODE:CARG3, NODE:CARG3, TMP1, lsl #3  // node = tab->node + idx*3*8
3292    |  add CARG4, STR:RC, CARG4, lsl #47	// Tagged key to look for.
3293    |   strb wzr, TAB:CARG2->nomm	// Clear metamethod cache.
3294    |1:
3295    |  ldp TMP1, CARG1, NODE:CARG3->val
3296    |   ldr NODE:TMP3, NODE:CARG3->next
3297    |    ldrb TMP2w, TAB:CARG2->marked
3298    |  cmp CARG1, CARG4
3299    |  bne >5
3300    |   ldr TMP0, [BASE, RA, lsl #3]
3301    |  cmp_nil TMP1			// Previous value is nil?
3302    |  beq >4
3303    |2:
3304    |   str TMP0, NODE:CARG3->val
3305    |    tbnz TMP2w, #2, >7		// isblack(table)
3306    |3:
3307    |  ins_next
3308    |
3309    |4:  // Check for __newindex if previous value is nil.
3310    |  ldr TAB:CARG1, TAB:CARG2->metatable
3311    |  cbz TAB:CARG1, <2		// No metatable: done.
3312    |  ldrb TMP1w, TAB:CARG1->nomm
3313    |  tbnz TMP1w, #MM_newindex, <2	// 'no __newindex' flag set: done.
3314    |  b ->vmeta_tsets
3315    |
3316    |5:  // Follow hash chain.
3317    |  mov NODE:CARG3, NODE:TMP3
3318    |  cbnz NODE:TMP3, <1
3319    |  // End of hash chain: key not found, add a new one.
3320    |
3321    |  // But check for __newindex first.
3322    |  ldr TAB:CARG1, TAB:CARG2->metatable
3323    |  cbz TAB:CARG1, >6		// No metatable: continue.
3324    |  ldrb TMP1w, TAB:CARG1->nomm
3325    |  // 'no __newindex' flag NOT set: check.
3326    |  tbz TMP1w, #MM_newindex, ->vmeta_tsets
3327    |6:
3328    |  movn TMP1, #~LJ_TSTR
3329    |   str PC, SAVE_PC
3330    |  add TMP0, STR:RC, TMP1, lsl #47
3331    |   str BASE, L->base
3332    |   mov CARG1, L
3333    |  str TMP0, TMPD
3334    |   add CARG3, sp, TMPDofs
3335    |  bl extern lj_tab_newkey		// (lua_State *L, GCtab *t, TValue *k)
3336    |  // Returns TValue *.
3337    |  ldr BASE, L->base
3338    |  ldr TMP0, [BASE, RA, lsl #3]
3339    |  str TMP0, [CRET1]
3340    |  b <3				// No 2nd write barrier needed.
3341    |
3342    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3343    |  barrierback TAB:CARG2, TMP2w, TMP1
3344    |  b <3
3345    break;
3346  case BC_TSETB:
3347    |  decode_RB RB, INS
3348    |   and RC, RC, #255
3349    |  // RA = src, RB = table, RC = index
3350    |  ldr CARG2, [BASE, RB, lsl #3]
3351    |  checktab CARG2, ->vmeta_tsetb
3352    |  ldr CARG3, TAB:CARG2->array
3353    |   ldr CARG1w, TAB:CARG2->asize
3354    |  add CARG3, CARG3, RC, lsl #3
3355    |   cmp RCw, CARG1w			// In array part?
3356    |   bhs ->vmeta_tsetb
3357    |  ldr TMP1, [CARG3]
3358    |   ldr TMP0, [BASE, RA, lsl #3]
3359    |    ldrb TMP2w, TAB:CARG2->marked
3360    |  cmp_nil TMP1			// Previous value is nil?
3361    |  beq >5
3362    |1:
3363    |   str TMP0, [CARG3]
3364    |    tbnz TMP2w, #2, >7		// isblack(table)
3365    |2:
3366    |   ins_next
3367    |
3368    |5:  // Check for __newindex if previous value is nil.
3369    |  ldr TAB:CARG1, TAB:CARG2->metatable
3370    |  cbz TAB:CARG1, <1		// No metatable: done.
3371    |  ldrb TMP1w, TAB:CARG1->nomm
3372    |  tbnz TMP1w, #MM_newindex, <1	// 'no __newindex' flag set: done.
3373    |  b ->vmeta_tsetb
3374    |
3375    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3376    |  barrierback TAB:CARG2, TMP2w, TMP1
3377    |  b <2
3378    break;
3379  case BC_TSETR:
3380    |  decode_RB RB, INS
3381    |   and RC, RC, #255
3382    |  // RA = src, RB = table, RC = key
3383    |  ldr CARG2, [BASE, RB, lsl #3]
3384    |   ldr TMP1, [BASE, RC, lsl #3]
3385    |  and TAB:CARG2, CARG2, #LJ_GCVMASK
3386    |  ldr CARG1, TAB:CARG2->array
3387    |    ldrb TMP2w, TAB:CARG2->marked
3388    |   ldr CARG4w, TAB:CARG2->asize
3389    |  add CARG1, CARG1, TMP1, uxtw #3
3390    |    tbnz TMP2w, #2, >7		// isblack(table)
3391    |2:
3392    |   cmp TMP1w, CARG4w		// In array part?
3393    |   bhs ->vmeta_tsetr
3394    |->BC_TSETR_Z:
3395    |   ldr TMP0, [BASE, RA, lsl #3]
3396    |   str TMP0, [CARG1]
3397    |   ins_next
3398    |
3399    |7:  // Possible table write barrier for the value. Skip valiswhite check.
3400    |  barrierback TAB:CARG2, TMP2w, TMP0
3401    |  b <2
3402    break;
3403
3404  case BC_TSETM:
3405    |  // RA = base (table at base-1), RC = num_const (start index)
3406    |  add RA, BASE, RA, lsl #3
3407    |1:
3408    |   ldr RBw, SAVE_MULTRES
3409    |  ldr TAB:CARG2, [RA, #-8]		// Guaranteed to be a table.
3410    |   ldr TMP1, [KBASE, RC, lsl #3]	// Integer constant is in lo-word.
3411    |    sub RB, RB, #8
3412    |    cbz RB, >4			// Nothing to copy?
3413    |  and TAB:CARG2, CARG2, #LJ_GCVMASK
3414    |  ldr CARG1w, TAB:CARG2->asize
3415    |   add CARG3w, TMP1w, RBw, lsr #3
3416    |   ldr CARG4, TAB:CARG2->array
3417    |  cmp CARG3, CARG1
3418    |    add RB, RA, RB
3419    |  bhi >5
3420    |   add TMP1, CARG4, TMP1w, uxtw #3
3421    |    ldrb TMP2w, TAB:CARG2->marked
3422    |3:  // Copy result slots to table.
3423    |   ldr TMP0, [RA], #8
3424    |   str TMP0, [TMP1], #8
3425    |  cmp RA, RB
3426    |  blo <3
3427    |    tbnz TMP2w, #2, >7		// isblack(table)
3428    |4:
3429    |  ins_next
3430    |
3431    |5:  // Need to resize array part.
3432    |   str BASE, L->base
3433    |  mov CARG1, L
3434    |   str PC, SAVE_PC
3435    |  bl extern lj_tab_reasize		// (lua_State *L, GCtab *t, int nasize)
3436    |  // Must not reallocate the stack.
3437    |  b <1
3438    |
3439    |7:  // Possible table write barrier for any value. Skip valiswhite check.
3440    |  barrierback TAB:CARG2, TMP2w, TMP1
3441    |  b <4
3442    break;
3443
3444  /* -- Calls and vararg handling ----------------------------------------- */
3445
3446  case BC_CALLM:
3447    |  // RA = base, (RB = nresults+1,) RC = extra_nargs
3448    |  ldr TMP0w, SAVE_MULTRES
3449    |  decode_RC8RD NARGS8:RC, RC
3450    |  add NARGS8:RC, NARGS8:RC, TMP0
3451    |  b ->BC_CALL_Z
3452    break;
3453  case BC_CALL:
3454    |  decode_RC8RD NARGS8:RC, RC
3455    |  // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
3456    |->BC_CALL_Z:
3457    |  mov RB, BASE			// Save old BASE for vmeta_call.
3458    |  add BASE, BASE, RA, lsl #3
3459    |  ldr CARG3, [BASE], #16
3460    |   sub NARGS8:RC, NARGS8:RC, #8
3461    |  checkfunc CARG3, ->vmeta_call
3462    |  ins_call
3463    break;
3464
3465  case BC_CALLMT:
3466    |  // RA = base, (RB = 0,) RC = extra_nargs
3467    |  ldr TMP0w, SAVE_MULTRES
3468    |  add NARGS8:RC, TMP0, RC, lsl #3
3469    |  b ->BC_CALLT1_Z
3470    break;
3471  case BC_CALLT:
3472    |  lsl NARGS8:RC, RC, #3
3473    |  // RA = base, (RB = 0,) RC = (nargs+1)*8
3474    |->BC_CALLT1_Z:
3475    |  add RA, BASE, RA, lsl #3
3476    |  ldr TMP1, [RA], #16
3477    |   sub NARGS8:RC, NARGS8:RC, #8
3478    |  checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
3479    |  ldr PC, [BASE, FRAME_PC]
3480    |->BC_CALLT2_Z:
3481    |   mov RB, #0
3482    |   ldrb TMP2w, LFUNC:CARG3->ffid
3483    |  tst PC, #FRAME_TYPE
3484    |  bne >7
3485    |1:
3486    |  str TMP1, [BASE, FRAME_FUNC]	// Copy function down, but keep PC.
3487    |  cbz NARGS8:RC, >3
3488    |2:
3489    |  ldr TMP0, [RA, RB]
3490    |   add TMP1, RB, #8
3491    |   cmp TMP1, NARGS8:RC
3492    |  str TMP0, [BASE, RB]
3493    |    mov RB, TMP1
3494    |   bne <2
3495    |3:
3496    |  cmp TMP2, #1			// (> FF_C) Calling a fast function?
3497    |  bhi >5
3498    |4:
3499    |  ins_callt
3500    |
3501    |5:  // Tailcall to a fast function with a Lua frame below.
3502    |  ldrb RAw, [PC, #-4+OFS_RA]
3503    |  sub CARG1, BASE, RA, lsl #3
3504    |  ldr LFUNC:CARG1, [CARG1, #-32]
3505    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3506    |  ldr CARG1, LFUNC:CARG1->pc
3507    |  ldr KBASE, [CARG1, #PC2PROTO(k)]
3508    |  b <4
3509    |
3510    |7:  // Tailcall from a vararg function.
3511    |  eor PC, PC, #FRAME_VARG
3512    |  tst PC, #FRAME_TYPEP		// Vararg frame below?
3513    |  csel TMP2, RB, TMP2, ne		// Clear ffid if no Lua function below.
3514    |  bne <1
3515    |  sub BASE, BASE, PC
3516    |  ldr PC, [BASE, FRAME_PC]
3517    |  tst PC, #FRAME_TYPE
3518    |  csel TMP2, RB, TMP2, ne		// Clear ffid if no Lua function below.
3519    |  b <1
3520    break;
3521
3522  case BC_ITERC:
3523    |  // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3524    |  add RA, BASE, RA, lsl #3
3525    |  ldr CARG3, [RA, #-24]
3526    |    mov RB, BASE			// Save old BASE for vmeta_call.
3527    |   ldp CARG1, CARG2, [RA, #-16]
3528    |    add BASE, RA, #16
3529    |    mov NARGS8:RC, #16		// Iterators get 2 arguments.
3530    |  str CARG3, [RA]			// Copy callable.
3531    |   stp CARG1, CARG2, [RA, #16]	// Copy state and control var.
3532    |  checkfunc CARG3, ->vmeta_call
3533    |  ins_call
3534    break;
3535
3536  case BC_ITERN:
3537    |.if JIT
3538    |  hotloop
3539    |.endif
3540    |->vm_IITERN:
3541    |  // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3542    |.if JIT
3543    |  bti_jump
3544    |.endif
3545    |  add RA, BASE, RA, lsl #3
3546    |  ldr TAB:RB, [RA, #-16]
3547    |    ldrh TMP3w, [PC, # OFS_RD]
3548    |  ldr CARG1w, [RA, #-8+LO]		// Get index from control var.
3549    |    add PC, PC, #4
3550    |    add TMP3, PC, TMP3, lsl #2
3551    |  and TAB:RB, RB, #LJ_GCVMASK
3552    |    sub TMP3, TMP3, #0x20000
3553    |  ldr TMP1w, TAB:RB->asize
3554    |   ldr CARG2, TAB:RB->array
3555    |1:  // Traverse array part.
3556    |  subs RC, CARG1, TMP1
3557    |   add CARG3, CARG2, CARG1, lsl #3
3558    |  bhs >5				// Index points after array part?
3559    |   ldr TMP0, [CARG3]
3560    |   cmp_nil TMP0
3561    |   cinc CARG1, CARG1, eq		// Skip holes in array part.
3562    |   beq <1
3563    |   add_TISNUM CARG1, CARG1
3564    |   stp CARG1, TMP0, [RA]
3565    |    add CARG1, CARG1, #1
3566    |3:
3567    |    str CARG1w, [RA, #-8+LO]	// Update control var.
3568    |  mov PC, TMP3
3569    |4:
3570    |  ins_next
3571    |
3572    |5:  // Traverse hash part.
3573    |  ldr TMP2w, TAB:RB->hmask
3574    |   ldr NODE:RB, TAB:RB->node
3575    |6:
3576    |   add CARG1, RC, RC, lsl #1
3577    |  cmp RC, TMP2			// End of iteration? Branch to ITERN+1.
3578    |   add NODE:CARG3, NODE:RB, CARG1, lsl #3  // node = tab->node + idx*3*8
3579    |  bhi <4
3580    |  ldp TMP0, CARG1, NODE:CARG3->val
3581    |  cmp_nil TMP0
3582    |   add RC, RC, #1
3583    |  beq <6				// Skip holes in hash part.
3584    |  stp CARG1, TMP0, [RA]
3585    |  add CARG1, RC, TMP1
3586    |  b <3
3587    break;
3588
3589  case BC_ISNEXT:
3590    |  // RA = base, RC = target (points to ITERN)
3591    |  add RA, BASE, RA, lsl #3
3592    |  ldr CFUNC:CARG1, [RA, #-24]
3593    |     add RC, PC, RC, lsl #2
3594    |   ldp TAB:CARG3, CARG4, [RA, #-16]
3595    |     sub RC, RC, #0x20000
3596    |  checkfunc CFUNC:CARG1, >5
3597    |   asr TMP0, TAB:CARG3, #47
3598    |  ldrb TMP1w, CFUNC:CARG1->ffid
3599    |   cmp_nil CARG4
3600    |   ccmn TMP0, #-LJ_TTAB, #0, eq
3601    |  ccmp TMP1w, #FF_next_N, #0, eq
3602    |  bne >5
3603    |  mov TMP0w, #0xfffe7fff		// LJ_KEYINDEX
3604    |  lsl TMP0, TMP0, #32
3605    |  str TMP0, [RA, #-8]		// Initialize control var.
3606    |1:
3607    |     mov PC, RC
3608    |  ins_next
3609    |
3610    |5:  // Despecialize bytecode if any of the checks fail.
3611    |.if JIT
3612    |  ldrb TMP2w, [RC, # OFS_OP]
3613    |.endif
3614    |  mov TMP0, #BC_JMP
3615    |   mov TMP1, #BC_ITERC
3616    |  strb TMP0w, [PC, #-4+OFS_OP]
3617    |.if JIT
3618    |  cmp TMP2w, #BC_ITERN
3619    |  bne >6
3620    |.endif
3621    |   strb TMP1w, [RC, # OFS_OP]
3622    |  b <1
3623    |.if JIT
3624    |6:  // Unpatch JLOOP.
3625    |  ldr RA, [GL, #GL_J(trace)]
3626    |  ldrh TMP2w, [RC, # OFS_RD]
3627    |  ldr TRACE:RA, [RA, TMP2, lsl #3]
3628    |  ldr TMP2w, TRACE:RA->startins
3629    |  bfxil TMP2w, TMP1w, #0, #8
3630    |  str TMP2w, [RC]
3631    |  b <1
3632    |.endif
3633    break;
3634
3635  case BC_VARG:
3636    |  decode_RB RB, INS
3637    |   and RC, RC, #255
3638    |  // RA = base, RB = (nresults+1), RC = numparams
3639    |  ldr TMP1, [BASE, FRAME_PC]
3640    |  add TMP0, BASE, RC, lsl #3
3641    |   add RC, BASE, RA, lsl #3	// RC = destination
3642    |  add TMP0, TMP0, #FRAME_VARG
3643    |   add TMP2, RC, RB, lsl #3
3644    |  sub RA, TMP0, TMP1		// RA = vbase
3645    |  // Note: RA may now be even _above_ BASE if nargs was < numparams.
3646    |   sub TMP3, BASE, #16		// TMP3 = vtop
3647    |  cbz RB, >5
3648    |   sub TMP2, TMP2, #16
3649    |1:  // Copy vararg slots to destination slots.
3650    |  cmp RA, TMP3
3651    |  ldr TMP0, [RA], #8
3652    |  csinv TMP0, TMP0, xzr, lo	// TISNIL = ~xzr
3653    |   cmp RC, TMP2
3654    |  str TMP0, [RC], #8
3655    |   blo <1
3656    |2:
3657    |  ins_next
3658    |
3659    |5:  // Copy all varargs.
3660    |  ldr TMP0, L->maxstack
3661    |   subs TMP2, TMP3, RA
3662    |   csel RB, xzr, TMP2, le		// MULTRES = (max(vtop-vbase,0)+1)*8
3663    |   add RB, RB, #8
3664    |  add TMP1, RC, TMP2
3665    |   str RBw, SAVE_MULTRES
3666    |   ble <2				// Nothing to copy.
3667    |  cmp TMP1, TMP0
3668    |  bhi >7
3669    |6:
3670    |  ldr TMP0, [RA], #8
3671    |  str TMP0, [RC], #8
3672    |  cmp RA, TMP3
3673    |  blo <6
3674    |  b <2
3675    |
3676    |7:  // Grow stack for varargs.
3677    |  lsr CARG2, TMP2, #3
3678    |   stp BASE, RC, L->base
3679    |  mov CARG1, L
3680    |  sub RA, RA, BASE			// Need delta, because BASE may change.
3681    |   str PC, SAVE_PC
3682    |  bl extern lj_state_growstack	// (lua_State *L, int n)
3683    |  ldp BASE, RC, L->base
3684    |  add RA, BASE, RA
3685    |  sub TMP3, BASE, #16
3686    |  b <6
3687    break;
3688
3689  /* -- Returns ----------------------------------------------------------- */
3690
3691  case BC_RETM:
3692    |  // RA = results, RC = extra results
3693    |  ldr TMP0w, SAVE_MULTRES
3694    |   ldr PC, [BASE, FRAME_PC]
3695    |    add RA, BASE, RA, lsl #3
3696    |  add RC, TMP0, RC, lsl #3
3697    |  b ->BC_RETM_Z
3698    break;
3699
3700  case BC_RET:
3701    |  // RA = results, RC = nresults+1
3702    |  ldr PC, [BASE, FRAME_PC]
3703    |   lsl RC, RC, #3
3704    |    add RA, BASE, RA, lsl #3
3705    |->BC_RETM_Z:
3706    |   str RCw, SAVE_MULTRES
3707    |1:
3708    |  ands CARG1, PC, #FRAME_TYPE
3709    |   eor CARG2, PC, #FRAME_VARG
3710    |  bne ->BC_RETV2_Z
3711    |
3712    |->BC_RET_Z:
3713    |  // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
3714    |  ldr INSw, [PC, #-4]
3715    |  subs TMP1, RC, #8
3716    |   sub CARG3, BASE, #16
3717    |  beq >3
3718    |2:
3719    |  ldr TMP0, [RA], #8
3720    |   add BASE, BASE, #8
3721    |   sub TMP1, TMP1, #8
3722    |  str TMP0, [BASE, #-24]
3723    |   cbnz TMP1, <2
3724    |3:
3725    |  decode_RA RA, INS
3726    |  sub CARG4, CARG3, RA, lsl #3
3727    |   decode_RB RB, INS
3728    |  ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
3729    |5:
3730    |  cmp RC, RB, lsl #3		// More results expected?
3731    |  blo >6
3732    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3733    |  mov BASE, CARG4
3734    |  ldr CARG2, LFUNC:CARG1->pc
3735    |  ldr KBASE, [CARG2, #PC2PROTO(k)]
3736    |   ins_next
3737    |
3738    |6:  // Fill up results with nil.
3739    |  add BASE, BASE, #8
3740    |   add RC, RC, #8
3741    |  str TISNIL, [BASE, #-24]
3742    |  b <5
3743    |
3744    |->BC_RETV1_Z:  // Non-standard return case.
3745    |  add RA, BASE, RA, lsl #3
3746    |->BC_RETV2_Z:
3747    |  tst CARG2, #FRAME_TYPEP
3748    |  bne ->vm_return
3749    |  // Return from vararg function: relocate BASE down.
3750    |  sub BASE, BASE, CARG2
3751    |  ldr PC, [BASE, FRAME_PC]
3752    |  b <1
3753    break;
3754
3755  case BC_RET0: case BC_RET1:
3756    |  // RA = results, RC = nresults+1
3757    |  ldr PC, [BASE, FRAME_PC]
3758    |   lsl RC, RC, #3
3759    |   str RCw, SAVE_MULTRES
3760    |  ands CARG1, PC, #FRAME_TYPE
3761    |   eor CARG2, PC, #FRAME_VARG
3762    |  bne ->BC_RETV1_Z
3763    |   ldr INSw, [PC, #-4]
3764    if (op == BC_RET1) {
3765      |  ldr TMP0, [BASE, RA, lsl #3]
3766    }
3767    |  sub CARG4, BASE, #16
3768    |   decode_RA RA, INS
3769    |  sub BASE, CARG4, RA, lsl #3
3770    if (op == BC_RET1) {
3771      |  str TMP0, [CARG4], #8
3772    }
3773    |   decode_RB RB, INS
3774    |  ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3775    |5:
3776    |  cmp RC, RB, lsl #3
3777    |  blo >6
3778    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3779    |  ldr CARG2, LFUNC:CARG1->pc
3780    |  ldr KBASE, [CARG2, #PC2PROTO(k)]
3781    |  ins_next
3782    |
3783    |6:  // Fill up results with nil.
3784    |  add RC, RC, #8
3785    |  str TISNIL, [CARG4], #8
3786    |  b <5
3787    break;
3788
3789  /* -- Loops and branches ------------------------------------------------ */
3790
3791  |.define FOR_IDX,  [RA];      .define FOR_TIDX,  [RA, #4]
3792  |.define FOR_STOP, [RA, #8];  .define FOR_TSTOP, [RA, #12]
3793  |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
3794  |.define FOR_EXT,  [RA, #24]; .define FOR_TEXT,  [RA, #28]
3795
3796  case BC_FORL:
3797    |.if JIT
3798    |  hotloop
3799    |.endif
3800    |  // Fall through. Assumes BC_IFORL follows.
3801    break;
3802
3803  case BC_JFORI:
3804  case BC_JFORL:
3805#if !LJ_HASJIT
3806    break;
3807#endif
3808  case BC_FORI:
3809  case BC_IFORL:
3810    |  // RA = base, RC = target (after end of loop or start of loop)
3811    vk = (op == BC_IFORL || op == BC_JFORL);
3812    |  add RA, BASE, RA, lsl #3
3813    |  ldp CARG1, CARG2, FOR_IDX		// CARG1 = IDX, CARG2 = STOP
3814    |   ldr CARG3, FOR_STEP			// CARG3 = STEP
3815    if (op != BC_JFORL) {
3816      |   add RC, PC, RC, lsl #2
3817      |   sub RC, RC, #0x20000
3818    }
3819    |  checkint CARG1, >5
3820    if (!vk) {
3821      |  checkint CARG2, ->vmeta_for
3822      |   checkint CARG3, ->vmeta_for
3823      |  tbnz CARG3w, #31, >4
3824      |  cmp CARG1w, CARG2w
3825    } else {
3826      |  adds CARG1w, CARG1w, CARG3w
3827      |  bvs >2
3828      |   add_TISNUM TMP0, CARG1
3829      |  tbnz CARG3w, #31, >4
3830      |  cmp CARG1w, CARG2w
3831    }
3832    |1:
3833    if (op == BC_FORI) {
3834      |  csel PC, RC, PC, gt
3835    } else if (op == BC_JFORI) {
3836      |  mov PC, RC
3837      |  ldrh RCw, [RC, #-4+OFS_RD]
3838    } else if (op == BC_IFORL) {
3839      |  csel PC, RC, PC, le
3840    }
3841    if (vk) {
3842      |   str TMP0, FOR_IDX
3843      |   str TMP0, FOR_EXT
3844    } else {
3845      |  str CARG1, FOR_EXT
3846    }
3847    if (op == BC_JFORI || op == BC_JFORL) {
3848      |  ble =>BC_JLOOP
3849    }
3850    |2:
3851    |   ins_next
3852    |
3853    |4:  // Invert check for negative step.
3854    |  cmp CARG2w, CARG1w
3855    |  b <1
3856    |
3857    |5:  // FP loop.
3858    |  ldp d0, d1, FOR_IDX
3859    |  blo ->vmeta_for
3860    if (!vk) {
3861      |  checknum CARG2, ->vmeta_for
3862      |   checknum CARG3, ->vmeta_for
3863      |  str d0, FOR_EXT
3864    } else {
3865      |  ldr d2, FOR_STEP
3866      |  fadd d0, d0, d2
3867    }
3868    |  tbnz CARG3, #63, >7
3869    |  fcmp d0, d1
3870    |6:
3871    if (vk) {
3872      |  str d0, FOR_IDX
3873      |  str d0, FOR_EXT
3874    }
3875    if (op == BC_FORI) {
3876      |  csel PC, RC, PC, hi
3877    } else if (op == BC_JFORI) {
3878      |  ldrh RCw, [RC, #-4+OFS_RD]
3879      |  bls =>BC_JLOOP
3880    } else if (op == BC_IFORL) {
3881      |  csel PC, RC, PC, ls
3882    } else {
3883      |  bls =>BC_JLOOP
3884    }
3885    |  b <2
3886    |
3887    |7:  // Invert check for negative step.
3888    |  fcmp d1, d0
3889    |  b <6
3890    break;
3891
3892  case BC_ITERL:
3893    |.if JIT
3894    |  hotloop
3895    |.endif
3896    |  // Fall through. Assumes BC_IITERL follows.
3897    break;
3898
3899  case BC_JITERL:
3900#if !LJ_HASJIT
3901    break;
3902#endif
3903  case BC_IITERL:
3904    |  // RA = base, RC = target
3905    |  ldr CARG1, [BASE, RA, lsl #3]
3906    |   add TMP1, BASE, RA, lsl #3
3907    |  cmp_nil CARG1
3908    |  beq >1				// Stop if iterator returned nil.
3909    if (op == BC_JITERL) {
3910      |  str CARG1, [TMP1, #-8]
3911      |  b =>BC_JLOOP
3912    } else {
3913      |  add TMP0, PC, RC, lsl #2	// Otherwise save control var + branch.
3914      |  sub PC, TMP0, #0x20000
3915      |  str CARG1, [TMP1, #-8]
3916    }
3917    |1:
3918    |  ins_next
3919    break;
3920
3921  case BC_LOOP:
3922    |  // RA = base, RC = target (loop extent)
3923    |  // Note: RA/RC is only used by trace recorder to determine scope/extent
3924    |  // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3925    |.if JIT
3926    |  hotloop
3927    |.endif
3928    |  // Fall through. Assumes BC_ILOOP follows.
3929    break;
3930
3931  case BC_ILOOP:
3932    |  // RA = base, RC = target (loop extent)
3933    |  ins_next
3934    break;
3935
3936  case BC_JLOOP:
3937    |.if JIT
3938    |  // RA = base (ignored), RC = traceno
3939    |  ldr CARG1, [GL, #GL_J(trace)]
3940    |   st_vmstate wzr  // Traces on ARM64 don't store the trace #, so use 0.
3941    |  ldr TRACE:RC, [CARG1, RC, lsl #3]
3942    |.if PAUTH
3943    |  ldr RA, TRACE:RC->mcauth
3944    |.else
3945    |  ldr RA, TRACE:RC->mcode
3946    |.endif
3947    |   str BASE, GL->jit_base
3948    |   str L, GL->tmpbuf.L
3949    |  sub sp, sp, #16	// See SPS_FIXED. Avoids sp adjust in every root trace.
3950    |.if PAUTH
3951    |  braa RA, RC
3952    |.else
3953    |  br RA
3954    |.endif
3955    |.endif
3956    break;
3957
3958  case BC_JMP:
3959    |  // RA = base (only used by trace recorder), RC = target
3960    |  add RC, PC, RC, lsl #2
3961    |  sub PC, RC, #0x20000
3962    |  ins_next
3963    break;
3964
3965  /* -- Function headers -------------------------------------------------- */
3966
3967  case BC_FUNCF:
3968    |.if JIT
3969    |  hotcall
3970    |.endif
3971  case BC_FUNCV:  /* NYI: compiled vararg functions. */
3972    |  // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3973    break;
3974
3975  case BC_JFUNCF:
3976#if !LJ_HASJIT
3977    break;
3978#endif
3979  case BC_IFUNCF:
3980    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3981    |  ldr CARG1, L->maxstack
3982    |   ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3983    |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
3984    |  cmp RA, CARG1
3985    |  bhi ->vm_growstack_l
3986    |2:
3987    |  cmp NARGS8:RC, TMP1, lsl #3	// Check for missing parameters.
3988    |  blo >3
3989    if (op == BC_JFUNCF) {
3990      |  decode_RD RC, INS
3991      |  b =>BC_JLOOP
3992    } else {
3993      |  ins_next
3994    }
3995    |
3996    |3:  // Clear missing parameters.
3997    |  str TISNIL, [BASE, NARGS8:RC]
3998    |  add NARGS8:RC, NARGS8:RC, #8
3999    |  b <2
4000    break;
4001
4002  case BC_JFUNCV:
4003#if !LJ_HASJIT
4004    break;
4005#endif
4006    |  NYI  // NYI: compiled vararg functions
4007    break;  /* NYI: compiled vararg functions. */
4008
4009  case BC_IFUNCV:
4010    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
4011    |  ldr CARG1, L->maxstack
4012    |   movn TMP0, #~LJ_TFUNC
4013    |   add TMP2, BASE, RC
4014    |   add LFUNC:CARG3, CARG3, TMP0, lsl #47
4015    |  add RA, RA, RC
4016    |  sub CARG1, CARG1, #8
4017    |   add TMP0, RC, #16+FRAME_VARG
4018    |   str LFUNC:CARG3, [TMP2], #8	// Store (tagged) copy of LFUNC.
4019    |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
4020    |  cmp RA, CARG1
4021    |   str TMP0, [TMP2], #8		// Store delta + FRAME_VARG.
4022    |  bhs ->vm_growstack_l
4023    |   sub RC, TMP2, #16
4024    |  ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
4025    |   mov RA, BASE
4026    |   mov BASE, TMP2
4027    |  cbz TMP1, >2
4028    |1:
4029    |  cmp RA, RC			// Less args than parameters?
4030    |  bhs >3
4031    |   ldr TMP0, [RA]
4032    |  sub TMP1, TMP1, #1
4033    |    str TISNIL, [RA], #8		// Clear old fixarg slot (help the GC).
4034    |   str TMP0, [TMP2], #8
4035    |  cbnz TMP1, <1
4036    |2:
4037    |  ins_next
4038    |
4039    |3:
4040    |  sub TMP1, TMP1, #1
4041    |   str TISNIL, [TMP2], #8
4042    |  cbz TMP1, <2
4043    |  b <3
4044    break;
4045
4046  case BC_FUNCC:
4047  case BC_FUNCCW:
4048    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
4049    if (op == BC_FUNCC) {
4050      |  ldr CARG4, CFUNC:CARG3->f
4051    } else {
4052      |  ldr CARG4, GL->wrapf
4053    }
4054    |   add CARG2, RA, NARGS8:RC
4055    |   ldr CARG1, L->maxstack
4056    |  add RC, BASE, NARGS8:RC
4057    |   cmp CARG2, CARG1
4058    |  stp BASE, RC, L->base
4059    if (op == BC_FUNCCW) {
4060      |  ldr CARG2, CFUNC:CARG3->f
4061    }
4062    |    mv_vmstate TMP0w, C
4063    |  mov CARG1, L
4064    |   bhi ->vm_growstack_c		// Need to grow stack.
4065    |    st_vmstate TMP0w
4066    |  blr_auth CARG4			// (lua_State *L [, lua_CFunction f])
4067    |  // Returns nresults.
4068    |  ldp BASE, TMP1, L->base
4069    |    str L, GL->cur_L
4070    |   sbfiz RC, CRET1, #3, #32
4071    |    st_vmstate ST_INTERP
4072    |  ldr PC, [BASE, FRAME_PC]
4073    |   sub RA, TMP1, RC		// RA = L->top - nresults*8
4074    |  b ->vm_returnc
4075    break;
4076
4077  /* ---------------------------------------------------------------------- */
4078
4079  default:
4080    fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
4081    exit(2);
4082    break;
4083  }
4084}
4085
4086static int build_backend(BuildCtx *ctx)
4087{
4088  int op;
4089
4090  dasm_growpc(Dst, BC__MAX);
4091
4092  build_subroutines(ctx);
4093
4094  |.code_op
4095  for (op = 0; op < BC__MAX; op++)
4096    build_ins(ctx, (BCOp)op, op);
4097
4098  return BC__MAX;
4099}
4100
4101/* Emit pseudo frame-info for all assembler functions. */
4102static void emit_asm_debug(BuildCtx *ctx)
4103{
4104  int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
4105  int i;
4106  switch (ctx->mode) {
4107  case BUILD_elfasm:
4108    fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
4109    fprintf(ctx->fp,
4110	".Lframe0:\n"
4111	"\t.long .LECIE0-.LSCIE0\n"
4112	".LSCIE0:\n"
4113	"\t.long 0xffffffff\n"
4114	"\t.byte 0x1\n"
4115	"\t.string \"\"\n"
4116	"\t.uleb128 0x1\n"
4117	"\t.sleb128 -8\n"
4118	"\t.byte 30\n"				/* Return address is in lr. */
4119	"\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n"	/* def_cfa fp 16 */
4120	"\t.align 3\n"
4121	".LECIE0:\n\n");
4122    fprintf(ctx->fp,
4123	".LSFDE0:\n"
4124	"\t.long .LEFDE0-.LASFDE0\n"
4125	".LASFDE0:\n"
4126	"\t.long .Lframe0\n"
4127	"\t.quad .Lbegin\n"
4128	"\t.quad %d\n"
4129	"\t.byte 0x9e\n\t.uleb128 1\n"		/* offset lr */
4130	"\t.byte 0x9d\n\t.uleb128 2\n",		/* offset fp */
4131	fcofs);
4132    for (i = 19; i <= 28; i++)  /* offset x19-x28 */
4133      fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4134    for (i = 8; i <= 15; i++)  /* offset d8-d15 */
4135      fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4136	      64+i, i+(3+(28-19+1)-8));
4137    fprintf(ctx->fp,
4138	"\t.align 3\n"
4139	".LEFDE0:\n\n");
4140#if LJ_HASFFI
4141    fprintf(ctx->fp,
4142	".LSFDE1:\n"
4143	"\t.long .LEFDE1-.LASFDE1\n"
4144	".LASFDE1:\n"
4145	"\t.long .Lframe0\n"
4146	"\t.quad lj_vm_ffi_call\n"
4147	"\t.quad %d\n"
4148	"\t.byte 0x9e\n\t.uleb128 1\n"		/* offset lr */
4149	"\t.byte 0x9d\n\t.uleb128 2\n"		/* offset fp */
4150	"\t.byte 0x93\n\t.uleb128 3\n"		/* offset x19 */
4151	"\t.byte 0x94\n\t.uleb128 4\n"		/* offset x20 */
4152	"\t.align 3\n"
4153	".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4154#endif
4155#if !LJ_NO_UNWIND
4156    fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
4157    fprintf(ctx->fp,
4158	".Lframe1:\n"
4159	"\t.long .LECIE1-.LSCIE1\n"
4160	".LSCIE1:\n"
4161	"\t.long 0\n"
4162	"\t.byte 0x1\n"
4163	"\t.string \"zPR\"\n"
4164	"\t.uleb128 0x1\n"
4165	"\t.sleb128 -8\n"
4166	"\t.byte 30\n"				/* Return address is in lr. */
4167	"\t.uleb128 6\n"			/* augmentation length */
4168	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
4169	"\t.long lj_err_unwind_dwarf-.\n"
4170	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
4171	"\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n"	/* def_cfa fp 16 */
4172	"\t.align 3\n"
4173	".LECIE1:\n\n");
4174    fprintf(ctx->fp,
4175	".LSFDE2:\n"
4176	"\t.long .LEFDE2-.LASFDE2\n"
4177	".LASFDE2:\n"
4178	"\t.long .LASFDE2-.Lframe1\n"
4179	"\t.long .Lbegin-.\n"
4180	"\t.long %d\n"
4181	"\t.uleb128 0\n"			/* augmentation length */
4182	"\t.byte 0x9e\n\t.uleb128 1\n"		/* offset lr */
4183	"\t.byte 0x9d\n\t.uleb128 2\n",		/* offset fp */
4184	fcofs);
4185    for (i = 19; i <= 28; i++)  /* offset x19-x28 */
4186      fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4187    for (i = 8; i <= 15; i++)  /* offset d8-d15 */
4188      fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4189	      64+i, i+(3+(28-19+1)-8));
4190    fprintf(ctx->fp,
4191	"\t.align 3\n"
4192	".LEFDE2:\n\n");
4193#if LJ_HASFFI
4194    fprintf(ctx->fp,
4195	".Lframe2:\n"
4196	"\t.long .LECIE2-.LSCIE2\n"
4197	".LSCIE2:\n"
4198	"\t.long 0\n"
4199	"\t.byte 0x1\n"
4200	"\t.string \"zR\"\n"
4201	"\t.uleb128 0x1\n"
4202	"\t.sleb128 -8\n"
4203	"\t.byte 30\n"				/* Return address is in lr. */
4204	"\t.uleb128 1\n"			/* augmentation length */
4205	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
4206	"\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n"	/* def_cfa fp 16 */
4207	"\t.align 3\n"
4208	".LECIE2:\n\n");
4209    fprintf(ctx->fp,
4210	".LSFDE3:\n"
4211	"\t.long .LEFDE3-.LASFDE3\n"
4212	".LASFDE3:\n"
4213	"\t.long .LASFDE3-.Lframe2\n"
4214	"\t.long lj_vm_ffi_call-.\n"
4215	"\t.long %d\n"
4216	"\t.uleb128 0\n"			/* augmentation length */
4217	"\t.byte 0x9e\n\t.uleb128 1\n"		/* offset lr */
4218	"\t.byte 0x9d\n\t.uleb128 2\n"		/* offset fp */
4219	"\t.byte 0x93\n\t.uleb128 3\n"		/* offset x19 */
4220	"\t.byte 0x94\n\t.uleb128 4\n"		/* offset x20 */
4221	"\t.align 3\n"
4222	".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
4223#endif
4224#endif
4225#if LJ_TARGET_LINUX && LJ_ABI_BRANCH_TRACK
4226    fprintf(ctx->fp,
4227	"\t.section .note.gnu.property,\"a\"\n"
4228	"\t.align 3\n"
4229	"\t.long 4\n"
4230	"\t.long 16\n"
4231	"\t.long 5\n"
4232	"\t.long 0x00554e47\n"
4233	"\t.long 0xc0000000\n"
4234	"\t.long 4\n"
4235	"\t.long 1\n"
4236	"\t.long 0\n");
4237#endif
4238    break;
4239#if !LJ_NO_UNWIND
4240  case BUILD_machasm: {
4241#if LJ_HASFFI
4242    int fcsize = 0;
4243#endif
4244    int j;
4245    fprintf(ctx->fp, "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support\n");
4246    fprintf(ctx->fp,
4247	"EH_frame1:\n"
4248	"\t.set L$set$x,LECIEX-LSCIEX\n"
4249	"\t.long L$set$x\n"
4250	"LSCIEX:\n"
4251	"\t.long 0\n"
4252	"\t.byte 0x1\n"
4253	"\t.ascii \"zPR\\0\"\n"
4254	"\t.uleb128 0x1\n"
4255	"\t.sleb128 -8\n"
4256	"\t.byte 30\n"				/* Return address is in lr. */
4257	"\t.uleb128 6\n"			/* augmentation length */
4258	"\t.byte 0x9b\n"			/* indirect|pcrel|sdata4 */
4259	"\t.long _lj_err_unwind_dwarf@GOT-.\n"
4260	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
4261	"\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n"	/* def_cfa fp 16 */
4262	"\t.align 3\n"
4263	"LECIEX:\n\n");
4264    for (j = 0; j < ctx->nsym; j++) {
4265      const char *name = ctx->sym[j].name;
4266      int32_t size = ctx->sym[j+1].ofs - ctx->sym[j].ofs;
4267      if (size == 0) continue;
4268#if LJ_HASFFI
4269      if (!strcmp(name, "_lj_vm_ffi_call")) { fcsize = size; continue; }
4270#endif
4271      fprintf(ctx->fp,
4272	"LSFDE%d:\n"
4273	"\t.set L$set$%d,LEFDE%d-LASFDE%d\n"
4274	"\t.long L$set$%d\n"
4275	"LASFDE%d:\n"
4276	"\t.long LASFDE%d-EH_frame1\n"
4277	"\t.long %s-.\n"
4278	"\t.long %d\n"
4279	"\t.uleb128 0\n"			/* augmentation length */
4280	"\t.byte 0x9e\n\t.uleb128 1\n"		/* offset lr */
4281	"\t.byte 0x9d\n\t.uleb128 2\n",		/* offset fp */
4282	j, j, j, j, j, j, j, name, size);
4283      for (i = 19; i <= 28; i++)  /* offset x19-x28 */
4284	fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4285      for (i = 8; i <= 15; i++)  /* offset d8-d15 */
4286	fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4287		64+i, i+(3+(28-19+1)-8));
4288      fprintf(ctx->fp,
4289	"\t.align 3\n"
4290	"LEFDE%d:\n\n", j);
4291    }
4292#if LJ_HASFFI
4293    if (fcsize) {
4294      fprintf(ctx->fp,
4295	"EH_frame2:\n"
4296	"\t.set L$set$y,LECIEY-LSCIEY\n"
4297	"\t.long L$set$y\n"
4298	"LSCIEY:\n"
4299	"\t.long 0\n"
4300	"\t.byte 0x1\n"
4301	"\t.ascii \"zR\\0\"\n"
4302	"\t.uleb128 0x1\n"
4303	"\t.sleb128 -8\n"
4304	"\t.byte 30\n"				/* Return address is in lr. */
4305	"\t.uleb128 1\n"			/* augmentation length */
4306	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
4307	"\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n"	/* def_cfa fp 16 */
4308	"\t.align 3\n"
4309	"LECIEY:\n\n");
4310      fprintf(ctx->fp,
4311	"LSFDEY:\n"
4312	"\t.set L$set$yy,LEFDEY-LASFDEY\n"
4313	"\t.long L$set$yy\n"
4314	"LASFDEY:\n"
4315	"\t.long LASFDEY-EH_frame2\n"
4316	"\t.long _lj_vm_ffi_call-.\n"
4317	"\t.long %d\n"
4318	"\t.uleb128 0\n"			/* augmentation length */
4319	"\t.byte 0x9e\n\t.uleb128 1\n"		/* offset lr */
4320	"\t.byte 0x9d\n\t.uleb128 2\n"		/* offset fp */
4321	"\t.byte 0x93\n\t.uleb128 3\n"		/* offset x19 */
4322	"\t.byte 0x94\n\t.uleb128 4\n"		/* offset x20 */
4323	"\t.align 3\n"
4324	"LEFDEY:\n\n", fcsize);
4325    }
4326#endif
4327    }
4328    break;
4329#endif
4330  default:
4331    break;
4332  }
4333}
4334