luna - luajit/src/lj

   1/*
   2** LuaJIT VM tags, values and objects.
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4**
   5** Portions taken verbatim or adapted from the Lua interpreter.
   6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
   7*/
   8
   9#ifndef _LJ_OBJ_H
  10#define _LJ_OBJ_H
  11
  12#include "lua.h"
  13#include "lj_def.h"
  14#include "lj_arch.h"
  15
  16/* -- Memory references --------------------------------------------------- */
  17
  18/* Memory and GC object sizes. */
  19typedef uint32_t MSize;
  20#if LJ_GC64
  21typedef uint64_t GCSize;
  22#else
  23typedef uint32_t GCSize;
  24#endif
  25
  26/* Memory reference */
  27typedef struct MRef {
  28#if LJ_GC64
  29  uint64_t ptr64;	/* True 64 bit pointer. */
  30#else
  31  uint32_t ptr32;	/* Pseudo 32 bit pointer. */
  32#endif
  33} MRef;
  34
  35#if LJ_GC64
  36#define mref(r, t)	((t *)(void *)(r).ptr64)
  37#define mrefu(r)	((r).ptr64)
  38
  39#define setmref(r, p)	((r).ptr64 = (uint64_t)(void *)(p))
  40#define setmrefu(r, u)	((r).ptr64 = (uint64_t)(u))
  41#define setmrefr(r, v)	((r).ptr64 = (v).ptr64)
  42#else
  43#define mref(r, t)	((t *)(void *)(uintptr_t)(r).ptr32)
  44#define mrefu(r)	((r).ptr32)
  45
  46#define setmref(r, p)	((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
  47#define setmrefu(r, u)	((r).ptr32 = (uint32_t)(u))
  48#define setmrefr(r, v)	((r).ptr32 = (v).ptr32)
  49#endif
  50
  51/* -- GC object references ------------------------------------------------ */
  52
  53/* GCobj reference */
  54typedef struct GCRef {
  55#if LJ_GC64
  56  uint64_t gcptr64;	/* True 64 bit pointer. */
  57#else
  58  uint32_t gcptr32;	/* Pseudo 32 bit pointer. */
  59#endif
  60} GCRef;
  61
  62/* Common GC header for all collectable objects. */
  63#define GCHeader	GCRef nextgc; uint8_t marked; uint8_t gct
  64/* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
  65
  66#if LJ_GC64
  67#define gcref(r)	((GCobj *)(r).gcptr64)
  68#define gcrefp(r, t)	((t *)(void *)(r).gcptr64)
  69#define gcrefu(r)	((r).gcptr64)
  70#define gcrefeq(r1, r2)	((r1).gcptr64 == (r2).gcptr64)
  71
  72#define setgcref(r, gc)	((r).gcptr64 = (uint64_t)&(gc)->gch)
  73#define setgcreft(r, gc, it) \
  74  (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
  75#define setgcrefp(r, p)	((r).gcptr64 = (uint64_t)(p))
  76#define setgcrefnull(r)	((r).gcptr64 = 0)
  77#define setgcrefr(r, v)	((r).gcptr64 = (v).gcptr64)
  78#else
  79#define gcref(r)	((GCobj *)(uintptr_t)(r).gcptr32)
  80#define gcrefp(r, t)	((t *)(void *)(uintptr_t)(r).gcptr32)
  81#define gcrefu(r)	((r).gcptr32)
  82#define gcrefeq(r1, r2)	((r1).gcptr32 == (r2).gcptr32)
  83
  84#define setgcref(r, gc)	((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
  85#define setgcrefp(r, p)	((r).gcptr32 = (uint32_t)(uintptr_t)(p))
  86#define setgcrefnull(r)	((r).gcptr32 = 0)
  87#define setgcrefr(r, v)	((r).gcptr32 = (v).gcptr32)
  88#endif
  89
  90#define gcnext(gc)	(gcref((gc)->gch.nextgc))
  91
  92/* IMPORTANT NOTE:
  93**
  94** All uses of the setgcref* macros MUST be accompanied with a write barrier.
  95**
  96** This is to ensure the integrity of the incremental GC. The invariant
  97** to preserve is that a black object never points to a white object.
  98** I.e. never store a white object into a field of a black object.
  99**
 100** It's ok to LEAVE OUT the write barrier ONLY in the following cases:
 101** - The source is not a GC object (NULL).
 102** - The target is a GC root. I.e. everything in global_State.
 103** - The target is a lua_State field (threads are never black).
 104** - The target is a stack slot, see setgcV et al.
 105** - The target is an open upvalue, i.e. pointing to a stack slot.
 106** - The target is a newly created object (i.e. marked white). But make
 107**   sure nothing invokes the GC inbetween.
 108** - The target and the source are the same object (self-reference).
 109** - The target already contains the object (e.g. moving elements around).
 110**
 111** The most common case is a store to a stack slot. All other cases where
 112** a barrier has been omitted are annotated with a NOBARRIER comment.
 113**
 114** The same logic applies for stores to table slots (array part or hash
 115** part). ALL uses of lj_tab_set* require a barrier for the stored value
 116** *and* the stored key, based on the above rules. In practice this means
 117** a barrier is needed if *either* of the key or value are a GC object.
 118**
 119** It's ok to LEAVE OUT the write barrier in the following special cases:
 120** - The stored value is nil. The key doesn't matter because it's either
 121**   not resurrected or lj_tab_newkey() will take care of the key barrier.
 122** - The key doesn't matter if the *previously* stored value is guaranteed
 123**   to be non-nil (because the key is kept alive in the table).
 124** - The key doesn't matter if it's guaranteed not to be part of the table,
 125**   since lj_tab_newkey() takes care of the key barrier. This applies
 126**   trivially to new tables, but watch out for resurrected keys. Storing
 127**   a nil value leaves the key in the table!
 128**
 129** In case of doubt use lj_gc_anybarriert() as it's rather cheap. It's used
 130** by the interpreter for all table stores.
 131**
 132** Note: In contrast to Lua's GC, LuaJIT's GC does *not* specially mark
 133** dead keys in tables. The reference is left in, but it's guaranteed to
 134** be never dereferenced as long as the value is nil. It's ok if the key is
 135** freed or if any object subsequently gets the same address.
 136**
 137** Not destroying dead keys helps to keep key hash slots stable. This avoids
 138** specialization back-off for HREFK when a value flips between nil and
 139** non-nil and the GC gets in the way. It also allows safely hoisting
 140** HREF/HREFK across GC steps. Dead keys are only removed if a table is
 141** resized (i.e. by NEWREF) and xREF must not be CSEd across a resize.
 142**
 143** The trade-off is that a write barrier for tables must take the key into
 144** account, too. Implicitly resurrecting the key by storing a non-nil value
 145** may invalidate the incremental GC invariant.
 146*/
 147
 148/* -- Common type definitions --------------------------------------------- */
 149
 150/* Types for handling bytecodes. Need this here, details in lj_bc.h. */
 151typedef uint32_t BCIns;  /* Bytecode instruction. */
 152typedef uint32_t BCPos;  /* Bytecode position. */
 153typedef uint32_t BCReg;  /* Bytecode register. */
 154typedef int32_t BCLine;  /* Bytecode line number. */
 155
 156/* Internal assembler functions. Never call these directly from C. */
 157typedef void (*ASMFunction)(void);
 158
 159/* Resizable string buffer. Need this here, details in lj_buf.h. */
 160#define SBufHeader	char *w, *e, *b; MRef L
 161typedef struct SBuf {
 162  SBufHeader;
 163} SBuf;
 164
 165/* -- Tags and values ----------------------------------------------------- */
 166
 167/* Frame link. */
 168typedef union {
 169  int32_t ftsz;		/* Frame type and size of previous frame. */
 170  MRef pcr;		/* Or PC for Lua frames. */
 171} FrameLink;
 172
 173/* Tagged value. */
 174typedef LJ_ALIGN(8) union TValue {
 175  uint64_t u64;		/* 64 bit pattern overlaps number. */
 176  lua_Number n;		/* Number object overlaps split tag/value object. */
 177#if LJ_GC64
 178  GCRef gcr;		/* GCobj reference with tag. */
 179  int64_t it64;
 180  struct {
 181    LJ_ENDIAN_LOHI(
 182      int32_t i;	/* Integer value. */
 183    , uint32_t it;	/* Internal object tag. Must overlap MSW of number. */
 184    )
 185  };
 186#else
 187  struct {
 188    LJ_ENDIAN_LOHI(
 189      union {
 190	GCRef gcr;	/* GCobj reference (if any). */
 191	int32_t i;	/* Integer value. */
 192      };
 193    , uint32_t it;	/* Internal object tag. Must overlap MSW of number. */
 194    )
 195  };
 196#endif
 197#if LJ_FR2
 198  int64_t ftsz;		/* Frame type and size of previous frame, or PC. */
 199#else
 200  struct {
 201    LJ_ENDIAN_LOHI(
 202      GCRef func;	/* Function for next frame (or dummy L). */
 203    , FrameLink tp;	/* Link to previous frame. */
 204    )
 205  } fr;
 206#endif
 207  struct {
 208    LJ_ENDIAN_LOHI(
 209      uint32_t lo;	/* Lower 32 bits of number. */
 210    , uint32_t hi;	/* Upper 32 bits of number. */
 211    )
 212  } u32;
 213} TValue;
 214
 215typedef const TValue cTValue;
 216
 217#define tvref(r)	(mref(r, TValue))
 218
 219/* More external and GCobj tags for internal objects. */
 220#define LAST_TT		LUA_TTHREAD
 221#define LUA_TPROTO	(LAST_TT+1)
 222#define LUA_TCDATA	(LAST_TT+2)
 223
 224/* Internal object tags.
 225**
 226** Format for 32 bit GC references (!LJ_GC64):
 227**
 228** Internal tags overlap the MSW of a number object (must be a double).
 229** Interpreted as a double these are special NaNs. The FPU only generates
 230** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
 231** for use as internal tags. Small negative numbers are used to shorten the
 232** encoding of type comparisons (reg/mem against sign-ext. 8 bit immediate).
 233**
 234**                  ---MSW---.---LSW---
 235** primitive types |  itype  |         |
 236** lightuserdata   |  itype  |  void * |  (32 bit platforms)
 237** lightuserdata   |ffff|seg|    ofs   |  (64 bit platforms)
 238** GC objects      |  itype  |  GCRef  |
 239** int (LJ_DUALNUM)|  itype  |   int   |
 240** number           -------double------
 241**
 242** Format for 64 bit GC references (LJ_GC64):
 243**
 244** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
 245** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
 246** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
 247**
 248**                     ------MSW------.------LSW------
 249** primitive types    |1..1|itype|1..................1|
 250** GC objects         |1..1|itype|-------GCRef--------|
 251** lightuserdata      |1..1|itype|seg|------ofs-------|
 252** int (LJ_DUALNUM)   |1..1|itype|0..0|-----int-------|
 253** number              ------------double-------------
 254**
 255** ORDER LJ_T
 256** Primitive types nil/false/true must be first, lightuserdata next.
 257** GC objects are at the end, table/userdata must be lowest.
 258** Also check lj_ir.h for similar ordering constraints.
 259*/
 260#define LJ_TNIL			(~0u)
 261#define LJ_TFALSE		(~1u)
 262#define LJ_TTRUE		(~2u)
 263#define LJ_TLIGHTUD		(~3u)
 264#define LJ_TSTR			(~4u)
 265#define LJ_TUPVAL		(~5u)
 266#define LJ_TTHREAD		(~6u)
 267#define LJ_TPROTO		(~7u)
 268#define LJ_TFUNC		(~8u)
 269#define LJ_TTRACE		(~9u)
 270#define LJ_TCDATA		(~10u)
 271#define LJ_TTAB			(~11u)
 272#define LJ_TUDATA		(~12u)
 273/* This is just the canonical number type used in some places. */
 274#define LJ_TNUMX		(~13u)
 275
 276/* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
 277#if LJ_64 && !LJ_GC64
 278#define LJ_TISNUM		0xfffeffffu
 279#else
 280#define LJ_TISNUM		LJ_TNUMX
 281#endif
 282#define LJ_TISTRUECOND		LJ_TFALSE
 283#define LJ_TISPRI		LJ_TTRUE
 284#define LJ_TISGCV		(LJ_TSTR+1)
 285#define LJ_TISTABUD		LJ_TTAB
 286
 287/* Type marker for slot holding a traversal index. Must be lightuserdata. */
 288#define LJ_KEYINDEX		0xfffe7fffu
 289
 290#if LJ_GC64
 291#define LJ_GCVMASK		(((uint64_t)1 << 47) - 1)
 292#endif
 293
 294#if LJ_64
 295/* To stay within 47 bits, lightuserdata is segmented. */
 296#define LJ_LIGHTUD_BITS_SEG	8
 297#define LJ_LIGHTUD_BITS_LO	(47 - LJ_LIGHTUD_BITS_SEG)
 298#endif
 299
 300/* -- String object ------------------------------------------------------- */
 301
 302typedef uint32_t StrHash;	/* String hash value. */
 303typedef uint32_t StrID;		/* String ID. */
 304
 305/* String object header. String payload follows. */
 306typedef struct GCstr {
 307  GCHeader;
 308  uint8_t reserved;	/* Used by lexer for fast lookup of reserved words. */
 309  uint8_t hashalg;	/* Hash algorithm. */
 310  StrID sid;		/* Interned string ID. */
 311  StrHash hash;		/* Hash of string. */
 312  MSize len;		/* Size of string. */
 313} GCstr;
 314
 315#define strref(r)	(&gcref((r))->str)
 316#define strdata(s)	((const char *)((s)+1))
 317#define strdatawr(s)	((char *)((s)+1))
 318#define strVdata(o)	strdata(strV(o))
 319
 320/* -- Userdata object ----------------------------------------------------- */
 321
 322/* Userdata object. Payload follows. */
 323typedef struct GCudata {
 324  GCHeader;
 325  uint8_t udtype;	/* Userdata type. */
 326  uint8_t unused2;
 327  GCRef env;		/* Should be at same offset in GCfunc. */
 328  MSize len;		/* Size of payload. */
 329  GCRef metatable;	/* Must be at same offset in GCtab. */
 330  uint32_t align1;	/* To force 8 byte alignment of the payload. */
 331} GCudata;
 332
 333/* Userdata types. */
 334enum {
 335  UDTYPE_USERDATA,	/* Regular userdata. */
 336  UDTYPE_IO_FILE,	/* I/O library FILE. */
 337  UDTYPE_FFI_CLIB,	/* FFI C library namespace. */
 338  UDTYPE_BUFFER,	/* String buffer. */
 339  UDTYPE__MAX
 340};
 341
 342#define uddata(u)	((void *)((u)+1))
 343#define sizeudata(u)	(sizeof(struct GCudata)+(u)->len)
 344
 345/* -- C data object ------------------------------------------------------- */
 346
 347/* C data object. Payload follows. */
 348typedef struct GCcdata {
 349  GCHeader;
 350  uint16_t ctypeid;	/* C type ID. */
 351} GCcdata;
 352
 353/* Prepended to variable-sized or realigned C data objects. */
 354typedef struct GCcdataVar {
 355  uint16_t offset;	/* Offset to allocated memory (relative to GCcdata). */
 356  uint16_t extra;	/* Extra space allocated (incl. GCcdata + GCcdatav). */
 357  MSize len;		/* Size of payload. */
 358} GCcdataVar;
 359
 360#define cdataptr(cd)	((void *)((cd)+1))
 361#define cdataisv(cd)	((cd)->marked & 0x80)
 362#define cdatav(cd)	((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar)))
 363#define cdatavlen(cd)	check_exp(cdataisv(cd), cdatav(cd)->len)
 364#define sizecdatav(cd)	(cdatavlen(cd) + cdatav(cd)->extra)
 365#define memcdatav(cd)	((void *)((char *)(cd) - cdatav(cd)->offset))
 366
 367/* -- Prototype object ---------------------------------------------------- */
 368
 369#define SCALE_NUM_GCO	((int32_t)sizeof(lua_Number)/sizeof(GCRef))
 370#define round_nkgc(n)	(((n) + SCALE_NUM_GCO-1) & ~(SCALE_NUM_GCO-1))
 371
 372typedef struct GCproto {
 373  GCHeader;
 374  uint8_t numparams;	/* Number of parameters. */
 375  uint8_t framesize;	/* Fixed frame size. */
 376  MSize sizebc;		/* Number of bytecode instructions. */
 377#if LJ_GC64
 378  uint32_t unused_gc64;
 379#endif
 380  GCRef gclist;
 381  MRef k;		/* Split constant array (points to the middle). */
 382  MRef uv;		/* Upvalue list. local slot|0x8000 or parent uv idx. */
 383  MSize sizekgc;	/* Number of collectable constants. */
 384  MSize sizekn;		/* Number of lua_Number constants. */
 385  MSize sizept;		/* Total size including colocated arrays. */
 386  uint8_t sizeuv;	/* Number of upvalues. */
 387  uint8_t flags;	/* Miscellaneous flags (see below). */
 388  uint16_t trace;	/* Anchor for chain of root traces. */
 389  /* ------ The following fields are for debugging/tracebacks only ------ */
 390  GCRef chunkname;	/* Name of the chunk this function was defined in. */
 391  BCLine firstline;	/* First line of the function definition. */
 392  BCLine numline;	/* Number of lines for the function definition. */
 393  MRef lineinfo;	/* Compressed map from bytecode ins. to source line. */
 394  MRef uvinfo;		/* Upvalue names. */
 395  MRef varinfo;		/* Names and compressed extents of local variables. */
 396} GCproto;
 397
 398/* Flags for prototype. */
 399#define PROTO_CHILD		0x01	/* Has child prototypes. */
 400#define PROTO_VARARG		0x02	/* Vararg function. */
 401#define PROTO_FFI		0x04	/* Uses BC_KCDATA for FFI datatypes. */
 402#define PROTO_NOJIT		0x08	/* JIT disabled for this function. */
 403#define PROTO_ILOOP		0x10	/* Patched bytecode with ILOOP etc. */
 404/* Only used during parsing. */
 405#define PROTO_HAS_RETURN	0x20	/* Already emitted a return. */
 406#define PROTO_FIXUP_RETURN	0x40	/* Need to fixup emitted returns. */
 407/* Top bits used for counting created closures. */
 408#define PROTO_CLCOUNT		0x20	/* Base of saturating 3 bit counter. */
 409#define PROTO_CLC_BITS		3
 410#define PROTO_CLC_POLY		(3*PROTO_CLCOUNT)  /* Polymorphic threshold. */
 411
 412#define PROTO_UV_LOCAL		0x8000	/* Upvalue for local slot. */
 413#define PROTO_UV_IMMUTABLE	0x4000	/* Immutable upvalue. */
 414
 415#define proto_kgc(pt, idx) \
 416  check_exp((uintptr_t)(intptr_t)(idx) >= ~(uintptr_t)(pt)->sizekgc+1u, \
 417	    gcref(mref((pt)->k, GCRef)[(idx)]))
 418#define proto_knumtv(pt, idx) \
 419  check_exp((uintptr_t)(idx) < (pt)->sizekn, &mref((pt)->k, TValue)[(idx)])
 420#define proto_bc(pt)		((BCIns *)((char *)(pt) + sizeof(GCproto)))
 421#define proto_bcpos(pt, pc)	((BCPos)((pc) - proto_bc(pt)))
 422#define proto_uv(pt)		(mref((pt)->uv, uint16_t))
 423
 424#define proto_chunkname(pt)	(strref((pt)->chunkname))
 425#define proto_chunknamestr(pt)	(strdata(proto_chunkname((pt))))
 426#define proto_lineinfo(pt)	(mref((pt)->lineinfo, const void))
 427#define proto_uvinfo(pt)	(mref((pt)->uvinfo, const uint8_t))
 428#define proto_varinfo(pt)	(mref((pt)->varinfo, const uint8_t))
 429
 430/* -- Upvalue object ------------------------------------------------------ */
 431
 432typedef struct GCupval {
 433  GCHeader;
 434  uint8_t closed;	/* Set if closed (i.e. uv->v == &uv->u.value). */
 435  uint8_t immutable;	/* Immutable value. */
 436  union {
 437    TValue tv;		/* If closed: the value itself. */
 438    struct {		/* If open: double linked list, anchored at thread. */
 439      GCRef prev;
 440      GCRef next;
 441    };
 442  };
 443  MRef v;		/* Points to stack slot (open) or above (closed). */
 444  uint32_t dhash;	/* Disambiguation hash: dh1 != dh2 => cannot alias. */
 445} GCupval;
 446
 447#define uvprev(uv_)	(&gcref((uv_)->prev)->uv)
 448#define uvnext(uv_)	(&gcref((uv_)->next)->uv)
 449#define uvval(uv_)	(mref((uv_)->v, TValue))
 450
 451/* -- Function object (closures) ------------------------------------------ */
 452
 453/* Common header for functions. env should be at same offset in GCudata. */
 454#define GCfuncHeader \
 455  GCHeader; uint8_t ffid; uint8_t nupvalues; \
 456  GCRef env; GCRef gclist; MRef pc
 457
 458typedef struct GCfuncC {
 459  GCfuncHeader;
 460  lua_CFunction f;	/* C function to be called. */
 461  TValue upvalue[1];	/* Array of upvalues (TValue). */
 462} GCfuncC;
 463
 464typedef struct GCfuncL {
 465  GCfuncHeader;
 466  GCRef uvptr[1];	/* Array of _pointers_ to upvalue objects (GCupval). */
 467} GCfuncL;
 468
 469typedef union GCfunc {
 470  GCfuncC c;
 471  GCfuncL l;
 472} GCfunc;
 473
 474#define FF_LUA		0
 475#define FF_C		1
 476#define isluafunc(fn)	((fn)->c.ffid == FF_LUA)
 477#define iscfunc(fn)	((fn)->c.ffid == FF_C)
 478#define isffunc(fn)	((fn)->c.ffid > FF_C)
 479#define funcproto(fn) \
 480  check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto)))
 481#define sizeCfunc(n)	(sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n))
 482#define sizeLfunc(n)	(sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n))
 483
 484/* -- Table object -------------------------------------------------------- */
 485
 486/* Hash node. */
 487typedef struct Node {
 488  TValue val;		/* Value object. Must be first field. */
 489  TValue key;		/* Key object. */
 490  MRef next;		/* Hash chain. */
 491#if !LJ_GC64
 492  MRef freetop;		/* Top of free elements (stored in t->node[0]). */
 493#endif
 494} Node;
 495
 496LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
 497
 498typedef struct GCtab {
 499  GCHeader;
 500  uint8_t nomm;		/* Negative cache for fast metamethods. */
 501  int8_t colo;		/* Array colocation. */
 502  MRef array;		/* Array part. */
 503  GCRef gclist;
 504  GCRef metatable;	/* Must be at same offset in GCudata. */
 505  MRef node;		/* Hash part. */
 506  uint32_t asize;	/* Size of array part (keys [0, asize-1]). */
 507  uint32_t hmask;	/* Hash part mask (size of hash part - 1). */
 508#if LJ_GC64
 509  MRef freetop;		/* Top of free elements. */
 510#endif
 511} GCtab;
 512
 513#define sizetabcolo(n)	((n)*sizeof(TValue) + sizeof(GCtab))
 514#define tabref(r)	((GCtab *)gcref((r)))
 515#define noderef(r)	(mref((r), Node))
 516#define nextnode(n)	(mref((n)->next, Node))
 517#if LJ_GC64
 518#define getfreetop(t, n)	(noderef((t)->freetop))
 519#define setfreetop(t, n, v)	(setmref((t)->freetop, (v)))
 520#else
 521#define getfreetop(t, n)	(noderef((n)->freetop))
 522#define setfreetop(t, n, v)	(setmref((n)->freetop, (v)))
 523#endif
 524
 525/* -- State objects ------------------------------------------------------- */
 526
 527/* VM states. */
 528enum {
 529  LJ_VMST_INTERP,	/* Interpreter. */
 530  LJ_VMST_C,		/* C function. */
 531  LJ_VMST_GC,		/* Garbage collector. */
 532  LJ_VMST_EXIT,		/* Trace exit handler. */
 533  LJ_VMST_RECORD,	/* Trace recorder. */
 534  LJ_VMST_OPT,		/* Optimizer. */
 535  LJ_VMST_ASM,		/* Assembler. */
 536  LJ_VMST__MAX
 537};
 538
 539#define setvmstate(g, st)	((g)->vmstate = ~LJ_VMST_##st)
 540
 541/* Metamethods. ORDER MM */
 542#ifdef LJ_HASFFI
 543#define MMDEF_FFI(_) _(new)
 544#else
 545#define MMDEF_FFI(_)
 546#endif
 547
 548#if LJ_52 || LJ_HASFFI
 549#define MMDEF_PAIRS(_) _(pairs) _(ipairs)
 550#else
 551#define MMDEF_PAIRS(_)
 552#define MM_pairs	255
 553#define MM_ipairs	255
 554#endif
 555
 556#define MMDEF(_) \
 557  _(index) _(newindex) _(gc) _(mode) _(eq) _(len) \
 558  /* Only the above (fast) metamethods are negative cached (max. 8). */ \
 559  _(lt) _(le) _(concat) _(call) \
 560  /* The following must be in ORDER ARITH. */ \
 561  _(add) _(sub) _(mul) _(div) _(mod) _(pow) _(unm) \
 562  /* The following are used in the standard libraries. */ \
 563  _(metatable) _(tostring) MMDEF_FFI(_) MMDEF_PAIRS(_)
 564
 565typedef enum {
 566#define MMENUM(name)	MM_##name,
 567MMDEF(MMENUM)
 568#undef MMENUM
 569  MM__MAX,
 570  MM____ = MM__MAX,
 571  MM_FAST = MM_len
 572} MMS;
 573
 574/* GC root IDs. */
 575typedef enum {
 576  GCROOT_MMNAME,	/* Metamethod names. */
 577  GCROOT_MMNAME_LAST = GCROOT_MMNAME + MM__MAX-1,
 578  GCROOT_BASEMT,	/* Metatables for base types. */
 579  GCROOT_BASEMT_NUM = GCROOT_BASEMT + ~LJ_TNUMX,
 580  GCROOT_IO_INPUT,	/* Userdata for default I/O input file. */
 581  GCROOT_IO_OUTPUT,	/* Userdata for default I/O output file. */
 582#if LJ_HASFFI
 583  GCROOT_FFI_FIN,	/* FFI finalizer table. */
 584#endif
 585  GCROOT_MAX
 586} GCRootID;
 587
 588#define basemt_it(g, it)	((g)->gcroot[GCROOT_BASEMT+~(it)])
 589#define basemt_obj(g, o)	((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
 590#define mmname_str(g, mm)	(strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
 591
 592/* Garbage collector state. */
 593typedef struct GCState {
 594  GCSize total;		/* Memory currently allocated. */
 595  GCSize threshold;	/* Memory threshold. */
 596  uint8_t currentwhite;	/* Current white color. */
 597  uint8_t state;	/* GC state. */
 598  uint8_t unused0;
 599#if LJ_64
 600  uint8_t lightudnum;	/* Number of lightuserdata segments - 1. */
 601#else
 602  uint8_t unused1;
 603#endif
 604  MSize sweepstr;	/* Sweep position in string table. */
 605  GCRef root;		/* List of all collectable objects. */
 606  MRef sweep;		/* Sweep position in root list. */
 607  GCRef gray;		/* List of gray objects. */
 608  GCRef grayagain;	/* List of objects for atomic traversal. */
 609  GCRef weak;		/* List of weak tables (to be cleared). */
 610  GCRef mmudata;	/* List of userdata (to be finalized). */
 611  GCSize debt;		/* Debt (how much GC is behind schedule). */
 612  GCSize estimate;	/* Estimate of memory actually in use. */
 613  MSize stepmul;	/* Incremental GC step granularity. */
 614  MSize pause;		/* Pause between successive GC cycles. */
 615#if LJ_64
 616  MRef lightudseg;	/* Upper bits of lightuserdata segments. */
 617#endif
 618} GCState;
 619
 620/* String interning state. */
 621typedef struct StrInternState {
 622  GCRef *tab;		/* String hash table anchors. */
 623  MSize mask;		/* String hash mask (size of hash table - 1). */
 624  MSize num;		/* Number of strings in hash table. */
 625  StrID id;		/* Next string ID. */
 626  uint8_t idreseed;	/* String ID reseed counter. */
 627  uint8_t second;	/* String interning table uses secondary hashing. */
 628  uint8_t unused1;
 629  uint8_t unused2;
 630  LJ_ALIGN(8) uint64_t seed;	/* Random string seed. */
 631} StrInternState;
 632
 633/* Global state, shared by all threads of a Lua universe. */
 634typedef struct global_State {
 635  lua_Alloc allocf;	/* Memory allocator. */
 636  void *allocd;		/* Memory allocator data. */
 637  GCState gc;		/* Garbage collector. */
 638  GCstr strempty;	/* Empty string. */
 639  uint8_t stremptyz;	/* Zero terminator of empty string. */
 640  uint8_t hookmask;	/* Hook mask. */
 641  uint8_t dispatchmode;	/* Dispatch mode. */
 642  uint8_t vmevmask;	/* VM event mask. */
 643  StrInternState str;	/* String interning. */
 644  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
 645  GCRef mainthref;	/* Link to main thread. */
 646  SBuf tmpbuf;		/* Temporary string buffer. */
 647  TValue tmptv, tmptv2;	/* Temporary TValues. */
 648  Node nilnode;		/* Fallback 1-element hash part (nil key and value). */
 649  TValue registrytv;	/* Anchor for registry. */
 650  GCRef vmthref;	/* Link to VM thread. */
 651  GCupval uvhead;	/* Head of double-linked list of all open upvalues. */
 652  int32_t hookcount;	/* Instruction hook countdown. */
 653  int32_t hookcstart;	/* Start count for instruction hook counter. */
 654  lua_Hook hookf;	/* Hook function. */
 655  lua_CFunction wrapf;	/* Wrapper for C function calls. */
 656  lua_CFunction panic;	/* Called as a last resort for errors. */
 657  BCIns bc_cfunc_int;	/* Bytecode for internal C function calls. */
 658  BCIns bc_cfunc_ext;	/* Bytecode for external C function calls. */
 659  GCRef cur_L;		/* Currently executing lua_State. */
 660  MRef jit_base;	/* Current JIT code L->base or NULL. */
 661  MRef ctype_state;	/* Pointer to C type state. */
 662  PRNGState prng;	/* Global PRNG state. */
 663  GCRef gcroot[GCROOT_MAX];  /* GC roots. */
 664} global_State;
 665
 666#define mainthread(g)	(&gcref(g->mainthref)->th)
 667#define vmthread(g)	(&gcref(g->vmthref)->th)
 668#define niltv(L) \
 669  check_exp(tvisnil(&G(L)->nilnode.val), &G(L)->nilnode.val)
 670#define niltvg(g) \
 671  check_exp(tvisnil(&(g)->nilnode.val), &(g)->nilnode.val)
 672
 673/* Hook management. Hook event masks are defined in lua.h. */
 674#define HOOK_EVENTMASK		0x0f
 675#define HOOK_ACTIVE		0x10
 676#define HOOK_ACTIVE_SHIFT	4
 677#define HOOK_VMEVENT		0x20
 678#define HOOK_GC			0x40
 679#define HOOK_PROFILE		0x80
 680#define hook_active(g)		((g)->hookmask & HOOK_ACTIVE)
 681#define hook_enter(g)		((g)->hookmask |= HOOK_ACTIVE)
 682#define hook_entergc(g) \
 683  ((g)->hookmask = ((g)->hookmask | (HOOK_ACTIVE|HOOK_GC)) & ~HOOK_PROFILE)
 684#define hook_vmevent(g)		((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
 685#define hook_leave(g)		((g)->hookmask &= ~HOOK_ACTIVE)
 686#define hook_save(g)		((g)->hookmask & ~HOOK_EVENTMASK)
 687#define hook_restore(g, h) \
 688  ((g)->hookmask = ((g)->hookmask & HOOK_EVENTMASK) | (h))
 689
 690/* Per-thread state object. */
 691struct lua_State {
 692  GCHeader;
 693  uint8_t dummy_ffid;	/* Fake FF_C for curr_funcisL() on dummy frames. */
 694  uint8_t status;	/* Thread status. */
 695  MRef glref;		/* Link to global state. */
 696  GCRef gclist;		/* GC chain. */
 697  TValue *base;		/* Base of currently executing function. */
 698  TValue *top;		/* First free slot in the stack. */
 699  MRef maxstack;	/* Last free slot in the stack. */
 700  MRef stack;		/* Stack base. */
 701  GCRef openupval;	/* List of open upvalues in the stack. */
 702  GCRef env;		/* Thread environment (table of globals). */
 703  void *cframe;		/* End of C stack frame chain. */
 704  MSize stacksize;	/* True stack size (incl. LJ_STACK_EXTRA). */
 705};
 706
 707#define G(L)			(mref(L->glref, global_State))
 708#define registry(L)		(&G(L)->registrytv)
 709
 710/* Macros to access the currently executing (Lua) function. */
 711#if LJ_GC64
 712#define curr_func(L)		(&gcval(L->base-2)->fn)
 713#elif LJ_FR2
 714#define curr_func(L)		(&gcref((L->base-2)->gcr)->fn)
 715#else
 716#define curr_func(L)		(&gcref((L->base-1)->fr.func)->fn)
 717#endif
 718#define curr_funcisL(L)		(isluafunc(curr_func(L)))
 719#define curr_proto(L)		(funcproto(curr_func(L)))
 720#define curr_topL(L)		(L->base + curr_proto(L)->framesize)
 721#define curr_top(L)		(curr_funcisL(L) ? curr_topL(L) : L->top)
 722
 723#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
 724LJ_FUNC_NORET void lj_assert_fail(global_State *g, const char *file, int line,
 725				  const char *func, const char *fmt, ...);
 726#endif
 727
 728/* -- GC object definition and conversions -------------------------------- */
 729
 730/* GC header for generic access to common fields of GC objects. */
 731typedef struct GChead {
 732  GCHeader;
 733  uint8_t unused1;
 734  uint8_t unused2;
 735  GCRef env;
 736  GCRef gclist;
 737  GCRef metatable;
 738} GChead;
 739
 740/* The env field SHOULD be at the same offset for all GC objects. */
 741LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCfuncL, env));
 742LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCudata, env));
 743
 744/* The metatable field MUST be at the same offset for all GC objects. */
 745LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCtab, metatable));
 746LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable));
 747
 748/* The gclist field MUST be at the same offset for all GC objects. */
 749LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist));
 750LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
 751LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist));
 752LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist));
 753
 754typedef union GCobj {
 755  GChead gch;
 756  GCstr str;
 757  GCupval uv;
 758  lua_State th;
 759  GCproto pt;
 760  GCfunc fn;
 761  GCcdata cd;
 762  GCtab tab;
 763  GCudata ud;
 764} GCobj;
 765
 766/* Macros to convert a GCobj pointer into a specific value. */
 767#define gco2str(o)	check_exp((o)->gch.gct == ~LJ_TSTR, &(o)->str)
 768#define gco2uv(o)	check_exp((o)->gch.gct == ~LJ_TUPVAL, &(o)->uv)
 769#define gco2th(o)	check_exp((o)->gch.gct == ~LJ_TTHREAD, &(o)->th)
 770#define gco2pt(o)	check_exp((o)->gch.gct == ~LJ_TPROTO, &(o)->pt)
 771#define gco2func(o)	check_exp((o)->gch.gct == ~LJ_TFUNC, &(o)->fn)
 772#define gco2cd(o)	check_exp((o)->gch.gct == ~LJ_TCDATA, &(o)->cd)
 773#define gco2tab(o)	check_exp((o)->gch.gct == ~LJ_TTAB, &(o)->tab)
 774#define gco2ud(o)	check_exp((o)->gch.gct == ~LJ_TUDATA, &(o)->ud)
 775
 776/* Macro to convert any collectable object into a GCobj pointer. */
 777#define obj2gco(v)	((GCobj *)(v))
 778
 779/* -- TValue getters/setters ---------------------------------------------- */
 780
 781/* Macros to test types. */
 782#if LJ_GC64
 783#define itype(o)	((uint32_t)((o)->it64 >> 47))
 784#define tvisnil(o)	((o)->it64 == -1)
 785#else
 786#define itype(o)	((o)->it)
 787#define tvisnil(o)	(itype(o) == LJ_TNIL)
 788#endif
 789#define tvisfalse(o)	(itype(o) == LJ_TFALSE)
 790#define tvistrue(o)	(itype(o) == LJ_TTRUE)
 791#define tvisbool(o)	(tvisfalse(o) || tvistrue(o))
 792#if LJ_64 && !LJ_GC64
 793#define tvislightud(o)	(((int32_t)itype(o) >> 15) == -2)
 794#else
 795#define tvislightud(o)	(itype(o) == LJ_TLIGHTUD)
 796#endif
 797#define tvisstr(o)	(itype(o) == LJ_TSTR)
 798#define tvisfunc(o)	(itype(o) == LJ_TFUNC)
 799#define tvisthread(o)	(itype(o) == LJ_TTHREAD)
 800#define tvisproto(o)	(itype(o) == LJ_TPROTO)
 801#define tviscdata(o)	(itype(o) == LJ_TCDATA)
 802#define tvistab(o)	(itype(o) == LJ_TTAB)
 803#define tvisudata(o)	(itype(o) == LJ_TUDATA)
 804#define tvisnumber(o)	(itype(o) <= LJ_TISNUM)
 805#define tvisint(o)	(LJ_DUALNUM && itype(o) == LJ_TISNUM)
 806#define tvisnum(o)	(itype(o) < LJ_TISNUM)
 807
 808#define tvistruecond(o)	(itype(o) < LJ_TISTRUECOND)
 809#define tvispri(o)	(itype(o) >= LJ_TISPRI)
 810#define tvistabud(o)	(itype(o) <= LJ_TISTABUD)  /* && !tvisnum() */
 811#define tvisgcv(o)	((itype(o) - LJ_TISGCV) > (LJ_TNUMX - LJ_TISGCV))
 812
 813/* Special macros to test numbers for NaN, +0, -0, +1 and raw equality. */
 814#define tvisnan(o)	((o)->n != (o)->n)
 815#if LJ_64
 816#define tviszero(o)	(((o)->u64 << 1) == 0)
 817#else
 818#define tviszero(o)	(((o)->u32.lo | ((o)->u32.hi << 1)) == 0)
 819#endif
 820#define tvispzero(o)	((o)->u64 == 0)
 821#define tvismzero(o)	((o)->u64 == U64x(80000000,00000000))
 822#define tvispone(o)	((o)->u64 == U64x(3ff00000,00000000))
 823#define rawnumequal(o1, o2)	((o1)->u64 == (o2)->u64)
 824
 825/* Macros to convert type ids. */
 826#if LJ_64 && !LJ_GC64
 827#define itypemap(o) \
 828  (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
 829#else
 830#define itypemap(o)	(tvisnumber(o) ? ~LJ_TNUMX : ~itype(o))
 831#endif
 832
 833/* Macros to get tagged values. */
 834#if LJ_GC64
 835#define gcval(o)	((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
 836#else
 837#define gcval(o)	(gcref((o)->gcr))
 838#endif
 839#define boolV(o)	check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
 840#if LJ_64
 841#define lightudseg(u) \
 842  (((u) >> LJ_LIGHTUD_BITS_LO) & ((1 << LJ_LIGHTUD_BITS_SEG)-1))
 843#define lightudlo(u) \
 844  ((u) & (((uint64_t)1 << LJ_LIGHTUD_BITS_LO) - 1))
 845#define lightudup(p) \
 846  ((uint32_t)(((p) >> LJ_LIGHTUD_BITS_LO) << (LJ_LIGHTUD_BITS_LO-32)))
 847static LJ_AINLINE void *lightudV(global_State *g, cTValue *o)
 848{
 849  uint64_t u = o->u64;
 850  uint64_t seg = lightudseg(u);
 851  uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
 852  lj_assertG(tvislightud(o), "lightuserdata expected");
 853  if (seg == (1 << LJ_LIGHTUD_BITS_SEG)-1) return NULL;
 854  lj_assertG(seg <= g->gc.lightudnum, "bad lightuserdata segment %d", seg);
 855  return (void *)(((uint64_t)segmap[seg] << 32) | lightudlo(u));
 856}
 857#else
 858#define lightudV(g, o)	check_exp(tvislightud(o), gcrefp((o)->gcr, void))
 859#endif
 860#define gcV(o)		check_exp(tvisgcv(o), gcval(o))
 861#define strV(o)		check_exp(tvisstr(o), &gcval(o)->str)
 862#define funcV(o)	check_exp(tvisfunc(o), &gcval(o)->fn)
 863#define threadV(o)	check_exp(tvisthread(o), &gcval(o)->th)
 864#define protoV(o)	check_exp(tvisproto(o), &gcval(o)->pt)
 865#define cdataV(o)	check_exp(tviscdata(o), &gcval(o)->cd)
 866#define tabV(o)		check_exp(tvistab(o), &gcval(o)->tab)
 867#define udataV(o)	check_exp(tvisudata(o), &gcval(o)->ud)
 868#define numV(o)		check_exp(tvisnum(o), (o)->n)
 869#define intV(o)		check_exp(tvisint(o), (int32_t)(o)->i)
 870
 871/* Macros to set tagged values. */
 872#if LJ_GC64
 873#define setitype(o, i)		((o)->it = ((i) << 15))
 874#define setnilV(o)		((o)->it64 = -1)
 875#define setpriV(o, x)		((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
 876#define setboolV(o, x)		((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
 877#else
 878#define setitype(o, i)		((o)->it = (i))
 879#define setnilV(o)		((o)->it = LJ_TNIL)
 880#define setboolV(o, x)		((o)->it = LJ_TFALSE-(uint32_t)(x))
 881#define setpriV(o, i)		(setitype((o), (i)))
 882#endif
 883
 884static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
 885{
 886#if LJ_GC64
 887  o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
 888#elif LJ_64
 889  o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
 890#else
 891  setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
 892#endif
 893}
 894
 895#if LJ_FR2 || LJ_32
 896#define contptr(f)		((void *)(f))
 897#define setcont(o, f)		((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
 898#else
 899#define contptr(f) \
 900  ((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
 901#define setcont(o, f) \
 902  ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
 903#endif
 904
 905static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
 906{
 907  UNUSED(L); UNUSED(o); UNUSED(msg);
 908#if LUA_USE_ASSERT
 909  if (tvisgcv(o)) {
 910    lj_assertL(~itype(o) == gcval(o)->gch.gct,
 911	       "mismatch of TValue type %d vs GC type %d",
 912	       ~itype(o), gcval(o)->gch.gct);
 913    /* Copy of isdead check from lj_gc.h to avoid circular include. */
 914    lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg);
 915  }
 916#endif
 917}
 918
 919static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
 920{
 921#if LJ_GC64
 922  setgcreft(o->gcr, v, itype);
 923#else
 924  setgcref(o->gcr, v); setitype(o, itype);
 925#endif
 926}
 927
 928static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
 929{
 930  setgcVraw(o, v, it);
 931  checklivetv(L, o, "store to dead GC object");
 932}
 933
 934#define define_setV(name, type, tag) \
 935static LJ_AINLINE void name(lua_State *L, TValue *o, const type *v) \
 936{ \
 937  setgcV(L, o, obj2gco(v), tag); \
 938}
 939define_setV(setstrV, GCstr, LJ_TSTR)
 940define_setV(setthreadV, lua_State, LJ_TTHREAD)
 941define_setV(setprotoV, GCproto, LJ_TPROTO)
 942define_setV(setfuncV, GCfunc, LJ_TFUNC)
 943define_setV(setcdataV, GCcdata, LJ_TCDATA)
 944define_setV(settabV, GCtab, LJ_TTAB)
 945define_setV(setudataV, GCudata, LJ_TUDATA)
 946
 947#define setnumV(o, x)		((o)->n = (x))
 948#define setnanV(o)		((o)->u64 = U64x(fff80000,00000000))
 949#define setpinfV(o)		((o)->u64 = U64x(7ff00000,00000000))
 950#define setminfV(o)		((o)->u64 = U64x(fff00000,00000000))
 951
 952static LJ_AINLINE void setintV(TValue *o, int32_t i)
 953{
 954#if LJ_DUALNUM
 955  o->i = (uint32_t)i; setitype(o, LJ_TISNUM);
 956#else
 957  o->n = (lua_Number)i;
 958#endif
 959}
 960
 961static LJ_AINLINE void setint64V(TValue *o, int64_t i)
 962{
 963  if (LJ_DUALNUM && LJ_LIKELY(i == (int64_t)(int32_t)i))
 964    setintV(o, (int32_t)i);
 965  else
 966    setnumV(o, (lua_Number)i);
 967}
 968
 969#if LJ_64
 970#define setintptrV(o, i)	setint64V((o), (i))
 971#else
 972#define setintptrV(o, i)	setintV((o), (i))
 973#endif
 974
 975/* Copy tagged values. */
 976static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
 977{
 978  *o1 = *o2;
 979  checklivetv(L, o1, "copy of dead GC object");
 980}
 981
 982/* -- Number to integer conversion ---------------------------------------- */
 983
 984/*
 985** The C standard leaves many aspects of FP to integer conversions as
 986** undefined behavior. Portability is a mess, hardware support varies,
 987** and modern C compilers are like a box of chocolates -- you never know
 988** what you're gonna get.
 989**
 990** However, we need 100% matching behavior between the interpreter (asm + C),
 991** optimizations (C) and the code generated by the JIT compiler (asm).
 992** Mixing Lua numbers with FFI numbers creates some extra requirements.
 993**
 994** These conversions have been moved to assembler code, even if they seem
 995** trivial, to foil unanticipated C compiler 'optimizations' with the
 996** surrounding code. Only the unchecked double to int32_t conversion
 997** is still in C, because it ought to be pretty safe -- we'll see.
 998**
 999** These macros also serve to document all places where FP to integer
1000** conversions happen.
1001*/
1002
1003/* Unchecked double to int32_t conversion. */
1004#define lj_num2int(n)		((int32_t)(n))
1005
1006/* Unchecked double to arch/os-dependent signed integer type conversion.
1007** This assumes the 32/64-bit signed conversions are NOT range-extended.
1008*/
1009#define lj_num2int_type(n, tp)	((tp)(n))
1010
1011/* Convert a double to int32_t and check for exact conversion.
1012** Returns the zero-extended int32_t on success. -0 is OK, too.
1013** Returns 0x8000000080000000LL on failure (simplifies range checks).
1014*/
1015LJ_ASMF LJ_CONSTF int64_t lj_vm_num2int_check(double x);
1016
1017/* Check for exact conversion only, without storing the result. */
1018#define lj_num2int_ok(x)	(lj_vm_num2int_check((x)) >= 0)
1019
1020/* Check for exact conversion and conditionally store result.
1021** Note: conditions that fail for 0x80000000 may check only the lower
1022** 32 bits. This generates good code for both 32 and 64 bit archs.
1023*/
1024#define lj_num2int_cond(x, i64, i, cond) \
1025  (i64 = lj_vm_num2int_check((x)), cond ? (i = (int32_t)i64, 1) : 0)
1026
1027/* This is the generic check for a full-range int32_t result. */
1028#define lj_num2int_check(x, i64, i) \
1029  lj_num2int_cond((x), i64, i, i64 >= 0)
1030
1031/* Predictable conversion from double to int64_t or uint64_t.
1032** Truncates towards zero. Out-of-range values, NaN and +-Inf return
1033** an arch-dependent result, but do not cause C undefined behavior.
1034** The uint64_t conversion accepts the union of the unsigned + signed range.
1035*/
1036LJ_ASMF LJ_CONSTF int64_t lj_vm_num2i64(double x);
1037LJ_ASMF LJ_CONSTF int64_t lj_vm_num2u64(double x);
1038
1039#define lj_num2i64(x)		(lj_vm_num2i64((x)))
1040#define lj_num2u64(x)		(lj_vm_num2u64((x)))
1041
1042/* Lua BitOp conversion semantics use the 2^52 + 2^51 trick. */
1043LJ_ASMF LJ_CONSTF int32_t lj_vm_tobit(double x);
1044
1045#define lj_num2bit(x)	lj_vm_tobit((x))
1046
1047static LJ_AINLINE int32_t numberVint(cTValue *o)
1048{
1049  if (LJ_LIKELY(tvisint(o)))
1050    return intV(o);
1051  else
1052    return lj_num2int(numV(o));
1053}
1054
1055static LJ_AINLINE lua_Number numberVnum(cTValue *o)
1056{
1057  if (LJ_UNLIKELY(tvisint(o)))
1058    return (lua_Number)intV(o);
1059  else
1060    return numV(o);
1061}
1062
1063/* -- Miscellaneous object handling --------------------------------------- */
1064
1065/* Names and maps for internal and external object tags. */
1066LJ_DATA const char *const lj_obj_typename[1+LUA_TCDATA+1];
1067LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
1068
1069#define lj_typename(o)	(lj_obj_itypename[itypemap(o)])
1070
1071/* Compare two objects without calling metamethods. */
1072LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
1073LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o);
1074
1075#if LJ_ABI_PAUTH
1076#if LJ_TARGET_ARM64
1077#include <ptrauth.h>
1078#define lj_ptr_sign(ptr, ctx) \
1079  ptrauth_sign_unauthenticated((ptr), ptrauth_key_function_pointer, (ctx))
1080#define lj_ptr_strip(ptr) ptrauth_strip((ptr), ptrauth_key_function_pointer)
1081#else
1082#error "No support for pointer authentication for this architecture"
1083#endif
1084#else
1085#define lj_ptr_sign(ptr, ctx) (ptr)
1086#define lj_ptr_strip(ptr) (ptr)
1087#endif
1088
1089#endif