luna - luajit/src/lj

   1/*
   2** Trace recorder for C data operations.
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4*/
   5
   6#define lj_ffrecord_c
   7#define LUA_CORE
   8
   9#include "lj_obj.h"
  10
  11#if LJ_HASJIT && LJ_HASFFI
  12
  13#include "lj_err.h"
  14#include "lj_tab.h"
  15#include "lj_frame.h"
  16#include "lj_ctype.h"
  17#include "lj_cdata.h"
  18#include "lj_cparse.h"
  19#include "lj_cconv.h"
  20#include "lj_carith.h"
  21#include "lj_clib.h"
  22#include "lj_ccall.h"
  23#include "lj_ff.h"
  24#include "lj_ir.h"
  25#include "lj_jit.h"
  26#include "lj_ircall.h"
  27#include "lj_iropt.h"
  28#include "lj_trace.h"
  29#include "lj_record.h"
  30#include "lj_ffrecord.h"
  31#include "lj_snap.h"
  32#include "lj_crecord.h"
  33#include "lj_dispatch.h"
  34#include "lj_strfmt.h"
  35#include "lj_strscan.h"
  36
  37/* Some local macros to save typing. Undef'd at the end. */
  38#define IR(ref)			(&J->cur.ir[(ref)])
  39
  40/* Pass IR on to next optimization in chain (FOLD). */
  41#define emitir(ot, a, b)	(lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  42
  43#define emitconv(a, dt, st, flags) \
  44  emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
  45
  46/* -- C type checks ------------------------------------------------------- */
  47
  48static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
  49{
  50  GCcdata *cd;
  51  TRef trtypeid;
  52  if (!tref_iscdata(tr))
  53    lj_trace_err(J, LJ_TRERR_BADTYPE);
  54  cd = cdataV(o);
  55  /* Specialize to the CTypeID. */
  56  trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
  57  emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
  58  return cd;
  59}
  60
  61/* Specialize to the CTypeID held by a cdata constructor. */
  62static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
  63{
  64  CTypeID id;
  65  lj_assertJ(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID,
  66	     "expected CTypeID cdata");
  67  id = *(CTypeID *)cdataptr(cd);
  68  tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
  69  emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
  70  return id;
  71}
  72
  73static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
  74{
  75  if (tref_isstr(tr)) {
  76    GCstr *s = strV(o);
  77    CPState cp;
  78    CTypeID oldtop;
  79    /* Specialize to the string containing the C type declaration. */
  80    emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
  81    cp.L = J->L;
  82    cp.cts = ctype_cts(J->L);
  83    oldtop = cp.cts->top;
  84    cp.srcname = strdata(s);
  85    cp.p = strdata(s);
  86    cp.param = NULL;
  87    cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
  88    if (lj_cparse(&cp) || cp.cts->top > oldtop)  /* Avoid new struct defs. */
  89      lj_trace_err(J, LJ_TRERR_BADTYPE);
  90    return cp.val.id;
  91  } else {
  92    GCcdata *cd = argv2cdata(J, tr, o);
  93    return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
  94					cd->ctypeid;
  95  }
  96}
  97
  98/* Convert CType to IRType (if possible). */
  99static IRType crec_ct2irt(CTState *cts, CType *ct)
 100{
 101  if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
 102  if (LJ_LIKELY(ctype_isnum(ct->info))) {
 103    if ((ct->info & CTF_FP)) {
 104      if (ct->size == sizeof(double))
 105	return IRT_NUM;
 106      else if (ct->size == sizeof(float))
 107	return IRT_FLOAT;
 108    } else {
 109      uint32_t b = lj_fls(ct->size);
 110      if (b <= 3)
 111	return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
 112    }
 113  } else if (ctype_isptr(ct->info)) {
 114    return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
 115  } else if (ctype_iscomplex(ct->info)) {
 116    if (ct->size == 2*sizeof(double))
 117      return IRT_NUM;
 118    else if (ct->size == 2*sizeof(float))
 119      return IRT_FLOAT;
 120  }
 121  return IRT_CDATA;
 122}
 123
 124/* -- Optimized memory fill and copy -------------------------------------- */
 125
 126/* Maximum length and unroll of inlined copy/fill. */
 127#define CREC_COPY_MAXUNROLL		16
 128#define CREC_COPY_MAXLEN		128
 129
 130#define CREC_FILL_MAXUNROLL		16
 131
 132/* Number of windowed registers used for optimized memory copy. */
 133#if LJ_TARGET_X86
 134#define CREC_COPY_REGWIN		2
 135#elif LJ_TARGET_PPC || LJ_TARGET_MIPS
 136#define CREC_COPY_REGWIN		8
 137#else
 138#define CREC_COPY_REGWIN		4
 139#endif
 140
 141/* List of memory offsets for copy/fill. */
 142typedef struct CRecMemList {
 143  CTSize ofs;		/* Offset in bytes. */
 144  IRType tp;		/* Type of load/store. */
 145  TRef trofs;		/* TRef of interned offset. */
 146  TRef trval;		/* TRef of load value. */
 147} CRecMemList;
 148
 149/* Generate copy list for element-wise struct copy. */
 150static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
 151{
 152  CTypeID fid = ct->sib;
 153  MSize mlp = 0;
 154  while (fid) {
 155    CType *df = ctype_get(cts, fid);
 156    fid = df->sib;
 157    if (ctype_isfield(df->info)) {
 158      CType *cct;
 159      IRType tp;
 160      if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
 161      cct = ctype_rawchild(cts, df);  /* Field type. */
 162      tp = crec_ct2irt(cts, cct);
 163      if (tp == IRT_CDATA) return 0;  /* NYI: aggregates. */
 164      if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 165      ml[mlp].ofs = df->size;
 166      ml[mlp].tp = tp;
 167      mlp++;
 168      if (ctype_iscomplex(cct->info)) {
 169	if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 170	ml[mlp].ofs = df->size + (cct->size >> 1);
 171	ml[mlp].tp = tp;
 172	mlp++;
 173      }
 174    } else if (!ctype_isconstval(df->info)) {
 175      /* NYI: bitfields and sub-structures. */
 176      return 0;
 177    }
 178  }
 179  return mlp;
 180}
 181
 182/* Generate unrolled copy list, from highest to lowest step size/alignment. */
 183static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
 184			      IRType tp)
 185{
 186  CTSize ofs = 0;
 187  MSize mlp = 0;
 188  if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
 189  do {
 190    while (ofs + step <= len) {
 191      if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 192      ml[mlp].ofs = ofs;
 193      ml[mlp].tp = tp;
 194      mlp++;
 195      ofs += step;
 196    }
 197    step >>= 1;
 198    tp -= 2;
 199  } while (ofs < len);
 200  return mlp;
 201}
 202
 203/*
 204** Emit copy list with windowed loads/stores.
 205** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
 206*/
 207static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 208			   TRef trdst, TRef trsrc)
 209{
 210  MSize i, j, rwin = 0;
 211  for (i = 0, j = 0; i < mlp; ) {
 212    TRef trofs = lj_ir_kintp(J, ml[i].ofs);
 213    TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
 214    ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
 215    ml[i].trofs = trofs;
 216    i++;
 217    rwin += (LJ_SOFTFP32 && ml[i].tp == IRT_NUM) ? 2 : 1;
 218    if (rwin >= CREC_COPY_REGWIN || i >= mlp) {  /* Flush buffered stores. */
 219      rwin = 0;
 220      for ( ; j < i; j++) {
 221	TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
 222	emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
 223      }
 224    }
 225  }
 226}
 227
 228/* Optimized memory copy. */
 229static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
 230		      CType *ct)
 231{
 232  if (tref_isk(trlen)) {  /* Length must be constant. */
 233    CRecMemList ml[CREC_COPY_MAXUNROLL];
 234    MSize mlp = 0;
 235    CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
 236    IRType tp = IRT_CDATA;
 237    int needxbar = 0;
 238    if (len == 0) return;  /* Shortcut. */
 239    if (len > CREC_COPY_MAXLEN) goto fallback;
 240    if (ct) {
 241      CTState *cts = ctype_ctsG(J2G(J));
 242      lj_assertJ(ctype_isarray(ct->info) || ctype_isstruct(ct->info),
 243		 "copy of non-aggregate");
 244      if (ctype_isarray(ct->info)) {
 245	CType *cct = ctype_rawchild(cts, ct);
 246	tp = crec_ct2irt(cts, cct);
 247	if (tp == IRT_CDATA) goto rawcopy;
 248	step = lj_ir_type_size[tp];
 249	lj_assertJ((len & (step-1)) == 0, "copy of fractional size");
 250      } else if ((ct->info & CTF_UNION)) {
 251	step = (1u << ctype_align(ct->info));
 252	goto rawcopy;
 253      } else {
 254	mlp = crec_copy_struct(ml, cts, ct);
 255	goto emitcopy;
 256      }
 257    } else {
 258    rawcopy:
 259      needxbar = 1;
 260      if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 261	step = CTSIZE_PTR;
 262    }
 263    mlp = crec_copy_unroll(ml, len, step, tp);
 264  emitcopy:
 265    if (mlp) {
 266      crec_copy_emit(J, ml, mlp, trdst, trsrc);
 267      if (needxbar)
 268	emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 269      return;
 270    }
 271  }
 272fallback:
 273  /* Call memcpy. Always needs a barrier to disable alias analysis. */
 274  lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
 275  emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 276}
 277
 278/* Generate unrolled fill list, from highest to lowest step size/alignment. */
 279static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
 280{
 281  CTSize ofs = 0;
 282  MSize mlp = 0;
 283  IRType tp = IRT_U8 + 2*lj_fls(step);
 284  do {
 285    while (ofs + step <= len) {
 286      if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 287      ml[mlp].ofs = ofs;
 288      ml[mlp].tp = tp;
 289      mlp++;
 290      ofs += step;
 291    }
 292    step >>= 1;
 293    tp -= 2;
 294  } while (ofs < len);
 295  return mlp;
 296}
 297
 298/*
 299** Emit stores for fill list.
 300** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
 301*/
 302static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 303			   TRef trdst, TRef trfill)
 304{
 305  MSize i;
 306  for (i = 0; i < mlp; i++) {
 307    TRef trofs = lj_ir_kintp(J, ml[i].ofs);
 308    TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
 309    emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
 310  }
 311}
 312
 313/* Optimized memory fill. */
 314static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
 315		      CTSize step)
 316{
 317  if (tref_isk(trlen)) {  /* Length must be constant. */
 318    CRecMemList ml[CREC_FILL_MAXUNROLL];
 319    MSize mlp;
 320    CTSize len = (CTSize)IR(tref_ref(trlen))->i;
 321    if (len == 0) return;  /* Shortcut. */
 322    if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 323      step = CTSIZE_PTR;
 324    if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
 325    mlp = crec_fill_unroll(ml, len, step);
 326    if (!mlp) goto fallback;
 327    if (tref_isk(trfill) || ml[0].tp != IRT_U8)
 328      trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
 329    if (ml[0].tp != IRT_U8) {  /* Scatter U8 to U16/U32/U64. */
 330      if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
 331	if (tref_isk(trfill))  /* Pointless on x64 with zero-extended regs. */
 332	  trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
 333	trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
 334			lj_ir_kint64(J, U64x(01010101,01010101)));
 335      } else {
 336	trfill = emitir(IRTI(IR_MUL), trfill,
 337		   lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
 338      }
 339    }
 340    crec_fill_emit(J, ml, mlp, trdst, trfill);
 341  } else {
 342fallback:
 343    /* Call memset. Always needs a barrier to disable alias analysis. */
 344    lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen);  /* Note: arg order! */
 345  }
 346  emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 347}
 348
 349/* -- Convert C type to C type -------------------------------------------- */
 350
 351/*
 352** This code mirrors the code in lj_cconv.c. It performs the same steps
 353** for the trace recorder that lj_cconv.c does for the interpreter.
 354**
 355** One major difference is that we can get away with much fewer checks
 356** here. E.g. checks for casts, constness or correct types can often be
 357** omitted, even if they might fail. The interpreter subsequently throws
 358** an error, which aborts the trace.
 359**
 360** All operations are specialized to their C types, so the on-trace
 361** outcome must be the same as the outcome in the interpreter. If the
 362** interpreter doesn't throw an error, then the trace is correct, too.
 363** Care must be taken not to generate invalid (temporary) IR or to
 364** trigger asserts.
 365*/
 366
 367/* Determine whether a passed number or cdata number is non-zero. */
 368static int crec_isnonzero(CType *s, void *p)
 369{
 370  if (p == (void *)0)
 371    return 0;
 372  if (p == (void *)1)
 373    return 1;
 374  if ((s->info & CTF_FP)) {
 375    if (s->size == sizeof(float))
 376      return (*(float *)p != 0);
 377    else
 378      return (*(double *)p != 0);
 379  } else {
 380    if (s->size == 1)
 381      return (*(uint8_t *)p != 0);
 382    else if (s->size == 2)
 383      return (*(uint16_t *)p != 0);
 384    else if (s->size == 4)
 385      return (*(uint32_t *)p != 0);
 386    else
 387      return (*(uint64_t *)p != 0);
 388  }
 389}
 390
 391static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
 392		       void *svisnz)
 393{
 394  IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
 395  IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
 396  CTSize dsize = d->size, ssize = s->size;
 397  CTInfo dinfo = d->info, sinfo = s->info;
 398
 399  if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
 400    goto err_conv;
 401
 402  /*
 403  ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
 404  ** numbers up to 8 bytes. Otherwise sp holds a pointer.
 405  */
 406
 407  switch (cconv_idx2(dinfo, sinfo)) {
 408  /* Destination is a bool. */
 409  case CCX(B, B):
 410    goto xstore;  /* Source operand is already normalized. */
 411  case CCX(B, I):
 412  case CCX(B, F):
 413    if (st != IRT_CDATA) {
 414      /* Specialize to the result of a comparison against 0. */
 415      TRef zero = (st == IRT_NUM  || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
 416		  (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
 417		  lj_ir_kint(J, 0);
 418      int isnz = crec_isnonzero(s, svisnz);
 419      emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
 420      sp = lj_ir_kint(J, isnz);
 421      goto xstore;
 422    }
 423    goto err_nyi;
 424
 425  /* Destination is an integer. */
 426  case CCX(I, B):
 427  case CCX(I, I):
 428  conv_I_I:
 429    if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 430    /* Extend 32 to 64 bit integer. */
 431    if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
 432      sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
 433		    (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
 434    else if (dsize < 8 && ssize == 8)  /* Truncate from 64 bit integer. */
 435      sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
 436    else if (st == IRT_INT)
 437      sp = lj_opt_narrow_toint(J, sp);
 438  xstore:
 439    if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
 440    if (dp == 0) return sp;
 441    emitir(IRT(IR_XSTORE, dt), dp, sp);
 442    break;
 443  case CCX(I, C):
 444    sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
 445    /* fallthrough */
 446  case CCX(I, F):
 447    if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 448  conv_I_F:
 449#if LJ_SOFTFP || LJ_32
 450    if (st == IRT_FLOAT) {  /* Uncommon. Simplify split backends. */
 451      sp = emitconv(sp, IRT_NUM, IRT_FLOAT, 0);
 452      st = IRT_NUM;
 453    }
 454#endif
 455    if (dsize < 8) {
 456      lj_needsplit(J);
 457      sp = emitconv(sp, IRT_I64, st, IRCONV_ANY);
 458      sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_I64, 0);
 459    } else {
 460      sp = emitconv(sp, dt, st, IRCONV_ANY);
 461    }
 462    goto xstore;
 463  case CCX(I, P):
 464  case CCX(I, A):
 465    sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
 466    ssize = CTSIZE_PTR;
 467    st = IRT_UINTP;
 468    if (((dsize ^ ssize) & 8) == 0) {  /* Must insert no-op type conversion. */
 469      sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
 470      goto xstore;
 471    }
 472    goto conv_I_I;
 473
 474  /* Destination is a floating-point number. */
 475  case CCX(F, B):
 476  case CCX(F, I):
 477  conv_F_I:
 478    if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 479    sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
 480    goto xstore;
 481  case CCX(F, C):
 482    sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
 483    /* fallthrough */
 484  case CCX(F, F):
 485  conv_F_F:
 486    if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 487    if (dt != st) sp = emitconv(sp, dt, st, 0);
 488    goto xstore;
 489
 490  /* Destination is a complex number. */
 491  case CCX(C, I):
 492  case CCX(C, F):
 493    {  /* Clear im. */
 494      TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
 495      emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
 496    }
 497    /* Convert to re. */
 498    if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
 499
 500  case CCX(C, C):
 501    if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 502    {
 503      TRef re, im, ptr;
 504      re = emitir(IRT(IR_XLOAD, st), sp, 0);
 505      ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
 506      im = emitir(IRT(IR_XLOAD, st), ptr, 0);
 507      if (dt != st) {
 508	re = emitconv(re, dt, st, 0);
 509	im = emitconv(im, dt, st, 0);
 510      }
 511      emitir(IRT(IR_XSTORE, dt), dp, re);
 512      ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
 513      emitir(IRT(IR_XSTORE, dt), ptr, im);
 514    }
 515    break;
 516
 517  /* Destination is a vector. */
 518  case CCX(V, I):
 519  case CCX(V, F):
 520  case CCX(V, C):
 521  case CCX(V, V):
 522    goto err_nyi;
 523
 524  /* Destination is a pointer. */
 525  case CCX(P, P):
 526  case CCX(P, A):
 527  case CCX(P, S):
 528    /* There are only 32 bit pointers/addresses on 32 bit machines.
 529    ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
 530    */
 531    goto xstore;
 532  case CCX(P, I):
 533    if (st == IRT_CDATA) goto err_nyi;
 534    if (!LJ_64 && ssize == 8)  /* Truncate from 64 bit integer. */
 535      sp = emitconv(sp, IRT_U32, st, 0);
 536    goto xstore;
 537  case CCX(P, F):
 538    if (st == IRT_CDATA) goto err_nyi;
 539    /* The signed 64 bit conversion is cheaper. */
 540    dt = (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32;
 541    goto conv_I_F;
 542
 543  /* Destination is an array. */
 544  case CCX(A, A):
 545  /* Destination is a struct/union. */
 546  case CCX(S, S):
 547    if (dp == 0) goto err_conv;
 548    crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
 549    break;
 550
 551  default:
 552  err_conv:
 553  err_nyi:
 554    lj_trace_err(J, LJ_TRERR_NYICONV);
 555    break;
 556  }
 557  return 0;
 558}
 559
 560/* -- Convert C type to TValue (load) ------------------------------------- */
 561
 562static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
 563{
 564  CTState *cts = ctype_ctsG(J2G(J));
 565  IRType t = crec_ct2irt(cts, s);
 566  CTInfo sinfo = s->info;
 567  if (ctype_isnum(sinfo)) {
 568    TRef tr;
 569    if (t == IRT_CDATA)
 570      goto err_nyi;  /* NYI: copyval of >64 bit integers. */
 571    tr = emitir(IRT(IR_XLOAD, t), sp, 0);
 572    if (t == IRT_FLOAT || t == IRT_U32) {  /* Keep uint32_t/float as numbers. */
 573      return emitconv(tr, IRT_NUM, t, 0);
 574    } else if (t == IRT_I64 || t == IRT_U64) {  /* Box 64 bit integer. */
 575      sp = tr;
 576      lj_needsplit(J);
 577    } else if ((sinfo & CTF_BOOL)) {
 578      /* Assume not equal to zero. Fixup and emit pending guard later. */
 579      lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
 580      J->postproc = LJ_POST_FIXGUARD;
 581      return TREF_TRUE;
 582    } else {
 583      return tr;
 584    }
 585  } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
 586    sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Box pointers and enums. */
 587  } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
 588    cts->L = J->L;
 589    sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR);  /* Create ref. */
 590  } else if (ctype_iscomplex(sinfo)) {  /* Unbox/box complex. */
 591    ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
 592    TRef ptr, tr1, tr2, dp;
 593    dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
 594    tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
 595    ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
 596    tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
 597    ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
 598    emitir(IRT(IR_XSTORE, t), ptr, tr1);
 599    ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
 600    emitir(IRT(IR_XSTORE, t), ptr, tr2);
 601    return dp;
 602  } else {
 603    /* NYI: copyval of vectors. */
 604  err_nyi:
 605    lj_trace_err(J, LJ_TRERR_NYICONV);
 606  }
 607  /* Box pointer, ref, enum or 64 bit integer. */
 608  return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
 609}
 610
 611/* -- Convert TValue to C type (store) ------------------------------------ */
 612
 613static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
 614{
 615  CTState *cts = ctype_ctsG(J2G(J));
 616  CTypeID sid = CTID_P_VOID;
 617  void *svisnz = 0;
 618  CType *s;
 619  if (LJ_LIKELY(tref_isinteger(sp))) {
 620    sid = CTID_INT32;
 621    svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
 622  } else if (tref_isnum(sp)) {
 623    sid = CTID_DOUBLE;
 624    svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
 625  } else if (tref_isbool(sp)) {
 626    sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
 627    sid = CTID_BOOL;
 628  } else if (tref_isnil(sp)) {
 629    sp = lj_ir_kptr(J, NULL);
 630  } else if (tref_isudata(sp)) {
 631    GCudata *ud = udataV(sval);
 632    if (ud->udtype == UDTYPE_IO_FILE || ud->udtype == UDTYPE_BUFFER) {
 633      TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
 634      emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, ud->udtype));
 635      sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp,
 636		  ud->udtype == UDTYPE_IO_FILE ? IRFL_UDATA_FILE :
 637						 IRFL_SBUF_R);
 638    } else {
 639      sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
 640    }
 641  } else if (tref_isstr(sp)) {
 642    if (ctype_isenum(d->info)) {  /* Match string against enum constant. */
 643      GCstr *str = strV(sval);
 644      CTSize ofs;
 645      CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
 646      /* Specialize to the name of the enum constant. */
 647      emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
 648      if (cct && ctype_isconstval(cct->info)) {
 649	lj_assertJ(ctype_child(cts, cct)->size == 4,
 650		   "only 32 bit const supported");  /* NYI */
 651	svisnz = (void *)(intptr_t)(ofs != 0);
 652	sp = lj_ir_kint(J, (int32_t)ofs);
 653	sid = ctype_cid(cct->info);
 654      }  /* else: interpreter will throw. */
 655    } else if (ctype_isrefarray(d->info)) {  /* Copy string to array. */
 656      lj_trace_err(J, LJ_TRERR_BADTYPE);  /* NYI */
 657    } else {  /* Otherwise pass the string data as a const char[]. */
 658      /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
 659      sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
 660      sid = CTID_A_CCHAR;
 661    }
 662  } else if (tref_islightud(sp)) {
 663#if LJ_64
 664    lj_trace_err(J, LJ_TRERR_NYICONV);
 665#endif
 666  } else {  /* NYI: tref_istab(sp). */
 667    IRType t;
 668    sid = argv2cdata(J, sp, sval)->ctypeid;
 669    s = ctype_raw(cts, sid);
 670    svisnz = cdataptr(cdataV(sval));
 671    if (ctype_isfunc(s->info)) {
 672      sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
 673      s = ctype_get(cts, sid);
 674      t = IRT_PTR;
 675    } else {
 676      t = crec_ct2irt(cts, s);
 677    }
 678    if (ctype_isptr(s->info)) {
 679      sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
 680      if (ctype_isref(s->info)) {
 681	svisnz = *(void **)svisnz;
 682	s = ctype_rawchild(cts, s);
 683	if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 684	t = crec_ct2irt(cts, s);
 685      } else {
 686	goto doconv;
 687      }
 688    } else if (t == IRT_I64 || t == IRT_U64) {
 689      sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
 690      lj_needsplit(J);
 691      goto doconv;
 692    } else if (t == IRT_INT || t == IRT_U32) {
 693      if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 694      sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
 695      goto doconv;
 696    } else {
 697      sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
 698    }
 699    if (ctype_isnum(s->info) && t != IRT_CDATA)
 700      sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Load number value. */
 701    goto doconv;
 702  }
 703  s = ctype_get(cts, sid);
 704doconv:
 705  if (ctype_isenum(d->info)) d = ctype_child(cts, d);
 706  return crec_ct_ct(J, d, s, dp, sp, svisnz);
 707}
 708
 709/* -- C data metamethods -------------------------------------------------- */
 710
 711/* This would be rather difficult in FOLD, so do it here:
 712** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
 713** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
 714*/
 715static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
 716{
 717  IRIns *ir = IR(tref_ref(tr));
 718  if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
 719      (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
 720    IRIns *irk = IR(ir->op2);
 721    ptrdiff_t k;
 722    if (LJ_64 && irk->o == IR_KINT64)
 723      k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
 724    else
 725      k = (ptrdiff_t)irk->i * sz;
 726    if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
 727    tr = ir->op1;  /* Not a TRef, but the caller doesn't care. */
 728  }
 729  return tr;
 730}
 731
 732/* Tailcall to function. */
 733static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
 734{
 735  TRef kfunc = lj_ir_kfunc(J, funcV(tv));
 736#if LJ_FR2
 737  J->base[-2] = kfunc;
 738  J->base[-1] = TREF_FRAME;
 739#else
 740  J->base[-1] = kfunc | TREF_FRAME;
 741#endif
 742  rd->nres = -1;  /* Pending tailcall. */
 743}
 744
 745/* Record ctype __index/__newindex metamethods. */
 746static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
 747			    RecordFFData *rd)
 748{
 749  CTypeID id = ctype_typeid(cts, ct);
 750  cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
 751  if (!tv)
 752    lj_trace_err(J, LJ_TRERR_BADTYPE);
 753  if (tvisfunc(tv)) {
 754    crec_tailcall(J, rd, tv);
 755  } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
 756    /* Specialize to result of __index lookup. */
 757    cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
 758    J->base[0] = lj_record_constify(J, o);
 759    if (!J->base[0])
 760      lj_trace_err(J, LJ_TRERR_BADTYPE);
 761    /* Always specialize to the key. */
 762    emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
 763  } else {
 764    /* NYI: resolving of non-function metamethods. */
 765    /* NYI: non-string keys for __index table. */
 766    /* NYI: stores to __newindex table. */
 767    lj_trace_err(J, LJ_TRERR_BADTYPE);
 768  }
 769}
 770
 771/* Record bitfield load/store. */
 772static void crec_index_bf(jit_State *J, RecordFFData *rd, TRef ptr, CTInfo info)
 773{
 774  IRType t = IRT_I8 + 2*lj_fls(ctype_bitcsz(info)) + ((info&CTF_UNSIGNED)?1:0);
 775  TRef tr = emitir(IRT(IR_XLOAD, t), ptr, 0);
 776  CTSize pos = ctype_bitpos(info), bsz = ctype_bitbsz(info), shift = 32 - bsz;
 777  lj_assertJ(t <= IRT_U32, "only 32 bit bitfields supported");  /* NYI */
 778  if (rd->data == 0) {  /* __index metamethod. */
 779    if ((info & CTF_BOOL)) {
 780      tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << pos))));
 781      /* Assume not equal to zero. Fixup and emit pending guard later. */
 782      lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
 783      J->postproc = LJ_POST_FIXGUARD;
 784      tr = TREF_TRUE;
 785    } else if (!(info & CTF_UNSIGNED)) {
 786      tr = emitir(IRTI(IR_BSHL), tr, lj_ir_kint(J, shift - pos));
 787      tr = emitir(IRTI(IR_BSAR), tr, lj_ir_kint(J, shift));
 788    } else {
 789      lj_assertJ(bsz < 32, "unexpected full bitfield index");
 790      tr = emitir(IRTI(IR_BSHR), tr, lj_ir_kint(J, pos));
 791      tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << bsz)-1)));
 792      /* We can omit the U32 to NUM conversion, since bsz < 32. */
 793    }
 794    J->base[0] = tr;
 795  } else {  /* __newindex metamethod. */
 796    CTState *cts = ctype_ctsG(J2G(J));
 797    CType *ct = ctype_get(cts,
 798			  (info & CTF_BOOL) ? CTID_BOOL :
 799			  (info & CTF_UNSIGNED) ? CTID_UINT32 : CTID_INT32);
 800    int32_t mask = (int32_t)(((1u << bsz)-1) << pos);
 801    TRef sp = crec_ct_tv(J, ct, 0, J->base[2], &rd->argv[2]);
 802    sp = emitir(IRTI(IR_BSHL), sp, lj_ir_kint(J, pos));
 803    /* Use of the target type avoids forwarding conversions. */
 804    sp = emitir(IRT(IR_BAND, t), sp, lj_ir_kint(J, mask));
 805    tr = emitir(IRT(IR_BAND, t), tr, lj_ir_kint(J, (int32_t)~mask));
 806    tr = emitir(IRT(IR_BOR, t), tr, sp);
 807    emitir(IRT(IR_XSTORE, t), ptr, tr);
 808    rd->nres = 0;
 809    J->needsnap = 1;
 810  }
 811}
 812
 813void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
 814{
 815  TRef idx, ptr = J->base[0];
 816  ptrdiff_t ofs = sizeof(GCcdata);
 817  GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
 818  CTState *cts = ctype_ctsG(J2G(J));
 819  CType *ct = ctype_raw(cts, cd->ctypeid);
 820  CTypeID sid = 0;
 821
 822  /* Resolve pointer or reference for cdata object. */
 823  if (ctype_isptr(ct->info)) {
 824    IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
 825    if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
 826    ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
 827    ofs = 0;
 828    ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
 829  }
 830
 831again:
 832  idx = J->base[1];
 833  if (tref_isnumber(idx)) {
 834    idx = lj_opt_narrow_cindex(J, idx);
 835    if (ctype_ispointer(ct->info)) {
 836      CTSize sz;
 837  integer_key:
 838      if ((ct->info & CTF_COMPLEX))
 839	idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
 840      sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
 841      idx = crec_reassoc_ofs(J, idx, &ofs, sz);
 842#if LJ_TARGET_ARM || LJ_TARGET_PPC
 843      /* Hoist base add to allow fusion of index/shift into operands. */
 844      if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
 845#if LJ_TARGET_ARM
 846	  && (sz == 1 || sz == 4)
 847#endif
 848	  ) {
 849	ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 850	ofs = 0;
 851      }
 852#endif
 853      idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
 854      ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
 855    }
 856  } else if (tref_iscdata(idx)) {
 857    GCcdata *cdk = cdataV(&rd->argv[1]);
 858    CType *ctk = ctype_raw(cts, cdk->ctypeid);
 859    IRType t = crec_ct2irt(cts, ctk);
 860    if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
 861      if (ctk->size == 8) {
 862	idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
 863      } else if (ctk->size == 4) {
 864	idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
 865      } else {
 866	idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
 867		     lj_ir_kintp(J, sizeof(GCcdata)));
 868	idx = emitir(IRT(IR_XLOAD, t), idx, 0);
 869      }
 870      if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
 871	idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
 872      if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
 873	idx = emitconv(idx, IRT_INTP, t, 0);
 874	lj_needsplit(J);
 875      }
 876      goto integer_key;
 877    }
 878  } else if (tref_isstr(idx)) {
 879    GCstr *name = strV(&rd->argv[1]);
 880    if (cd && cd->ctypeid == CTID_CTYPEID)
 881      ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
 882    if (ctype_isstruct(ct->info)) {
 883      CTSize fofs;
 884      CType *fct;
 885      fct = lj_ctype_getfield(cts, ct, name, &fofs);
 886      if (fct) {
 887	ofs += (ptrdiff_t)fofs;
 888	/* Always specialize to the field name. */
 889	emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 890	if (ctype_isconstval(fct->info)) {
 891	  if (fct->size >= 0x80000000u &&
 892	      (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
 893	    J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
 894	    return;
 895	  }
 896	  J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
 897	  return;  /* Interpreter will throw for newindex. */
 898	} else if (cd && cd->ctypeid == CTID_CTYPEID) {
 899	  /* Only resolve constants and metamethods for constructors. */
 900	} else if (ctype_isbitfield(fct->info)) {
 901	  if (ofs)
 902	    ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 903	  crec_index_bf(J, rd, ptr, fct->info);
 904	  return;
 905	} else {
 906	  lj_assertJ(ctype_isfield(fct->info), "field expected");
 907	  sid = ctype_cid(fct->info);
 908	}
 909      }
 910    } else if (ctype_iscomplex(ct->info)) {
 911      if (name->len == 2 &&
 912	  ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
 913	   (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
 914	/* Always specialize to the field name. */
 915	emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 916	if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
 917	sid = ctype_cid(ct->info);
 918      }
 919    }
 920  }
 921  if (!sid) {
 922    if (ctype_isptr(ct->info)) {  /* Automatically perform '->'. */
 923      CType *cct = ctype_rawchild(cts, ct);
 924      if (ctype_isstruct(cct->info)) {
 925	ct = cct;
 926	cd = NULL;
 927	if (tref_isstr(idx)) goto again;
 928      }
 929    }
 930    crec_index_meta(J, cts, ct, rd);
 931    return;
 932  }
 933
 934  if (ofs)
 935    ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 936
 937  /* Resolve reference for field. */
 938  ct = ctype_get(cts, sid);
 939  if (ctype_isref(ct->info)) {
 940    ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
 941    sid = ctype_cid(ct->info);
 942    ct = ctype_get(cts, sid);
 943  }
 944
 945  while (ctype_isattrib(ct->info))
 946    ct = ctype_child(cts, ct);  /* Skip attributes. */
 947
 948  if (rd->data == 0) {  /* __index metamethod. */
 949    J->base[0] = crec_tv_ct(J, ct, sid, ptr);
 950  } else {  /* __newindex metamethod. */
 951    rd->nres = 0;
 952    J->needsnap = 1;
 953    crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
 954  }
 955}
 956
 957/* Record setting a finalizer. */
 958static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
 959{
 960  if (tvisgcv(fin)) {
 961    if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
 962  } else if (tvisnil(fin)) {
 963    trfin = lj_ir_kptr(J, NULL);
 964  } else {
 965    lj_trace_err(J, LJ_TRERR_BADTYPE);
 966  }
 967  lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
 968	     trfin, lj_ir_kint(J, (int32_t)itype(fin)));
 969  J->needsnap = 1;
 970}
 971
 972/* Record cdata allocation. */
 973static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
 974{
 975  CTState *cts = ctype_ctsG(J2G(J));
 976  CTSize sz;
 977  CTInfo info = lj_ctype_info(cts, id, &sz);
 978  CType *d = ctype_raw(cts, id);
 979  TRef trcd, trid = lj_ir_kint(J, id);
 980  cTValue *fin;
 981  /* Use special instruction to box pointer or 32/64 bit integer. */
 982  if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
 983    TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
 984	      ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
 985	      sz == 4 ? lj_ir_kint(J, 0) :
 986	      (lj_needsplit(J), lj_ir_kint64(J, 0));
 987    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
 988    return;
 989  } else {
 990    TRef trsz = TREF_NIL;
 991    if ((info & CTF_VLA)) {  /* Calculate VLA/VLS size at runtime. */
 992      CTSize sz0, sz1;
 993      if (!J->base[1] || J->base[2])
 994	lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init VLA/VLS. */
 995      trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
 996			J->base[1], &rd->argv[1]);
 997      sz0 = lj_ctype_vlsize(cts, d, 0);
 998      sz1 = lj_ctype_vlsize(cts, d, 1);
 999      trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
1000      trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
1001      J->base[1] = 0;  /* Simplify logic below. */
1002    } else if (ctype_align(info) > CT_MEMALIGN) {
1003      trsz = lj_ir_kint(J, sz);
1004    }
1005    trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
1006    if (sz > 128 || (info & CTF_VLA)) {
1007      TRef dp;
1008      CTSize align;
1009    special:  /* Only handle bulk zero-fill for large/VLA/VLS types. */
1010      if (J->base[1])
1011	lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init large/VLA/VLS types. */
1012      dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
1013      if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
1014      align = ctype_align(info);
1015      if (align < CT_MEMALIGN) align = CT_MEMALIGN;
1016      crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
1017    } else if (J->base[1] && !J->base[2] &&
1018	!lj_cconv_multi_init(cts, d, &rd->argv[1])) {
1019      goto single_init;
1020    } else if (ctype_isarray(d->info)) {
1021      CType *dc = ctype_rawchild(cts, d);  /* Array element type. */
1022      CTSize ofs, esize = dc->size;
1023      TRef sp = 0;
1024      TValue tv;
1025      TValue *sval = &tv;
1026      MSize i;
1027      tv.u64 = 0;
1028      if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
1029	  esize * CREC_FILL_MAXUNROLL < sz)
1030	goto special;
1031      for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
1032	TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1033			 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
1034	if (J->base[i]) {
1035	  sp = J->base[i];
1036	  sval = &rd->argv[i];
1037	  i++;
1038	} else if (i != 2) {
1039	  sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
1040	}
1041	crec_ct_tv(J, dc, dp, sp, sval);
1042      }
1043    } else if (ctype_isstruct(d->info)) {
1044      CTypeID fid;
1045      MSize i = 1;
1046      if (!J->base[1]) {  /* Handle zero-fill of struct-of-NYI. */
1047	fid = d->sib;
1048	while (fid) {
1049	  CType *df = ctype_get(cts, fid);
1050	  fid = df->sib;
1051	  if (ctype_isfield(df->info)) {
1052	    CType *dc;
1053	    if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
1054	    dc = ctype_rawchild(cts, df);  /* Field type. */
1055	    if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1056		  ctype_isenum(dc->info)))
1057	      goto special;
1058	  } else if (!ctype_isconstval(df->info)) {
1059	    goto special;
1060	  }
1061	}
1062      }
1063      fid = d->sib;
1064      while (fid) {
1065	CType *df = ctype_get(cts, fid);
1066	fid = df->sib;
1067	if (ctype_isfield(df->info)) {
1068	  CType *dc;
1069	  TRef sp, dp;
1070	  TValue tv;
1071	  TValue *sval = &tv;
1072	  setintV(&tv, 0);
1073	  if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
1074	  dc = ctype_rawchild(cts, df);  /* Field type. */
1075	  if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1076		ctype_isenum(dc->info)))
1077	    lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init aggregates. */
1078	  if (J->base[i]) {
1079	    sp = J->base[i];
1080	    sval = &rd->argv[i];
1081	    i++;
1082	  } else {
1083	    sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
1084	  }
1085	  dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1086		      lj_ir_kintp(J, df->size + sizeof(GCcdata)));
1087	  crec_ct_tv(J, dc, dp, sp, sval);
1088	  if ((d->info & CTF_UNION)) {
1089	    if (d->size != dc->size)  /* NYI: partial init of union. */
1090	      lj_trace_err(J, LJ_TRERR_NYICONV);
1091	    break;
1092	  }
1093	} else if (!ctype_isconstval(df->info)) {
1094	  /* NYI: init bitfields and sub-structures. */
1095	  lj_trace_err(J, LJ_TRERR_NYICONV);
1096	}
1097      }
1098    } else {
1099      TRef dp;
1100    single_init:
1101      dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
1102      if (J->base[1]) {
1103	crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
1104      } else {
1105	TValue tv;
1106	tv.u64 = 0;
1107	crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
1108      }
1109    }
1110  }
1111  J->base[0] = trcd;
1112  /* Handle __gc metamethod. */
1113  fin = lj_ctype_meta(cts, id, MM_gc);
1114  if (fin)
1115    crec_finalizer(J, trcd, 0, fin);
1116}
1117
1118/* Record argument conversions.
1119** Note: may reallocate cts->tab and invalidate CType pointers.
1120*/
1121static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1122			   CTState *cts, CType *ct)
1123{
1124  TRef args[CCI_NARGS_MAX];
1125  CTypeID fid;
1126  CTInfo info = ct->info;  /* lj_ccall_ctid_vararg may invalidate ct pointer. */
1127  MSize i, n;
1128  TRef tr, *base;
1129  cTValue *o;
1130#if LJ_TARGET_X86
1131#if LJ_ABI_WIN
1132  TRef *arg0 = NULL, *arg1 = NULL;
1133#endif
1134  int ngpr = 0;
1135  if (ctype_cconv(info) == CTCC_THISCALL)
1136    ngpr = 1;
1137  else if (ctype_cconv(info) == CTCC_FASTCALL)
1138    ngpr = 2;
1139#elif LJ_TARGET_ARM64 && LJ_TARGET_OSX
1140  int ngpr = CCALL_NARG_GPR;
1141#endif
1142
1143  /* Skip initial attributes. */
1144  fid = ct->sib;
1145  while (fid) {
1146    CType *ctf = ctype_get(cts, fid);
1147    if (!ctype_isattrib(ctf->info)) break;
1148    fid = ctf->sib;
1149  }
1150  args[0] = TREF_NIL;
1151  for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1152    CTypeID did;
1153    CType *d;
1154
1155    if (n >= CCI_NARGS_MAX)
1156      lj_trace_err(J, LJ_TRERR_NYICALL);
1157
1158    if (fid) {  /* Get argument type from field. */
1159      CType *ctf = ctype_get(cts, fid);
1160      fid = ctf->sib;
1161      lj_assertJ(ctype_isfield(ctf->info), "field expected");
1162      did = ctype_cid(ctf->info);
1163    } else {
1164      if (!(info & CTF_VARARG))
1165	lj_trace_err(J, LJ_TRERR_NYICALL);  /* Too many arguments. */
1166#if LJ_TARGET_ARM64 && LJ_TARGET_OSX
1167      if (ngpr >= 0) {
1168	ngpr = -1;
1169	args[n++] = TREF_NIL;  /* Marker for start of varargs. */
1170	if (n >= CCI_NARGS_MAX)
1171	  lj_trace_err(J, LJ_TRERR_NYICALL);
1172      }
1173#endif
1174      did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
1175    }
1176    d = ctype_raw(cts, did);
1177    if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1178	  ctype_isenum(d->info)))
1179      lj_trace_err(J, LJ_TRERR_NYICALL);
1180    tr = crec_ct_tv(J, d, 0, *base, o);
1181    if (ctype_isinteger_or_bool(d->info)) {
1182#if LJ_TARGET_ARM64 && LJ_TARGET_OSX
1183      if (!ngpr) {
1184	/* Fixed args passed on the stack use their unpromoted size. */
1185	if (d->size != lj_ir_type_size[tref_type(tr)]) {
1186	  lj_assertJ(d->size == 1 || d->size==2, "unexpected size %d", d->size);
1187	  tr = emitconv(tr, d->size==1 ? IRT_U8 : IRT_U16, tref_type(tr), 0);
1188	}
1189      } else
1190#endif
1191      if (d->size < 4) {
1192	if ((d->info & CTF_UNSIGNED))
1193	  tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1194	else
1195	  tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1196      }
1197    } else if (LJ_SOFTFP32 && ctype_isfp(d->info) && d->size > 4) {
1198      lj_needsplit(J);
1199    }
1200#if LJ_TARGET_X86
1201    /* 64 bit args must not end up in registers for fastcall/thiscall. */
1202#if LJ_ABI_WIN
1203    if (!ctype_isfp(d->info)) {
1204      /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1205      if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1206	if (ngpr) {
1207	  arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1208	  if (ngpr) {
1209	    arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1210	  }
1211	}
1212      } else {
1213	if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1214	if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1215	if (ngpr) ngpr--;
1216      }
1217    }
1218#else
1219    if (!ctype_isfp(d->info) && ngpr) {
1220      if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1221	/* No reordering for other x86 ABIs. Simply add alignment args. */
1222	do { args[n++] = TREF_NIL; } while (--ngpr);
1223      } else {
1224	ngpr--;
1225      }
1226    }
1227#endif
1228#elif LJ_TARGET_ARM64 && LJ_TARGET_OSX
1229    if (!ctype_isfp(d->info) && ngpr) {
1230      ngpr--;
1231    }
1232#endif
1233    args[n] = tr;
1234  }
1235  tr = args[0];
1236  for (i = 1; i < n; i++)
1237    tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1238  return tr;
1239}
1240
1241/* Create a snapshot for the caller, simulating a 'false' return value. */
1242static void crec_snap_caller(jit_State *J)
1243{
1244  lua_State *L = J->L;
1245  TValue *base = L->base, *top = L->top;
1246  const BCIns *pc = J->pc;
1247  TRef ftr = J->base[-1-LJ_FR2];
1248  ptrdiff_t delta;
1249  if (!frame_islua(base-1) || J->framedepth <= 0)
1250    lj_trace_err(J, LJ_TRERR_NYICALL);
1251  J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
1252  L->top = base; L->base = base - delta;
1253  J->base[-1-LJ_FR2] = TREF_FALSE;
1254  J->base -= delta; J->baseslot -= (BCReg)delta;
1255  J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
1256  lj_snap_add(J);
1257  L->base = base; L->top = top;
1258  J->framedepth++; J->maxslot = 1;
1259  J->base += delta; J->baseslot += (BCReg)delta;
1260  J->base[-1-LJ_FR2] = ftr; J->pc = pc;
1261}
1262
1263/* Record function call. */
1264static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1265{
1266  CTState *cts = ctype_ctsG(J2G(J));
1267  CType *ct = ctype_raw(cts, cd->ctypeid);
1268  CTInfo info;
1269  IRType tp = IRT_PTR;
1270  if (ctype_isptr(ct->info)) {
1271    tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1272    ct = ctype_rawchild(cts, ct);
1273  }
1274  info = ct->info;  /* crec_call_args may invalidate ct pointer. */
1275  if (ctype_isfunc(info)) {
1276    TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1277    CType *ctr = ctype_rawchild(cts, ct);
1278    CTInfo ctr_info = ctr->info;  /* crec_call_args may invalidate ctr. */
1279    IRType t = crec_ct2irt(cts, ctr);
1280    TRef tr;
1281    TValue tv;
1282    /* Check for blacklisted C functions that might call a callback. */
1283    tv.u64 = ((uintptr_t)cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4) >> 2) | U64x(800000000, 00000000);
1284    if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1285      lj_trace_err(J, LJ_TRERR_BLACKL);
1286    if (ctype_isvoid(ctr_info)) {
1287      t = IRT_NIL;
1288      rd->nres = 0;
1289    } else if (!(ctype_isnum(ctr_info) || ctype_isptr(ctr_info) ||
1290		 ctype_isenum(ctr_info)) || t == IRT_CDATA) {
1291      lj_trace_err(J, LJ_TRERR_NYICALL);
1292    }
1293    if ((info & CTF_VARARG)
1294#if LJ_TARGET_X86
1295	|| ctype_cconv(info) != CTCC_CDECL
1296#endif
1297	)
1298      func = emitir(IRT(IR_CARG, IRT_NIL), func,
1299		    lj_ir_kint(J, ctype_typeid(cts, ct)));
1300    tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1301    if (ctype_isbool(ctr_info)) {
1302      if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1303	/* Don't check result if ignored. */
1304	tr = TREF_NIL;
1305      } else {
1306	crec_snap_caller(J);
1307#if LJ_TARGET_X86ORX64
1308	/* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1309	lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1310#else
1311	lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1312#endif
1313	J->postproc = LJ_POST_FIXGUARDSNAP;
1314	tr = TREF_TRUE;
1315      }
1316    } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1317	       t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr_info)) {
1318      TRef trid = lj_ir_kint(J, ctype_cid(info));
1319      tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1320      if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1321    } else if (t == IRT_FLOAT || t == IRT_U32) {
1322      tr = emitconv(tr, IRT_NUM, t, 0);
1323    } else if (t == IRT_I8 || t == IRT_I16) {
1324      tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1325    } else if (t == IRT_U8 || t == IRT_U16) {
1326      tr = emitconv(tr, IRT_INT, t, 0);
1327    }
1328    J->base[0] = tr;
1329    J->needsnap = 1;
1330    return 1;
1331  }
1332  return 0;
1333}
1334
1335void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1336{
1337  CTState *cts = ctype_ctsG(J2G(J));
1338  GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1339  CTypeID id = cd->ctypeid;
1340  CType *ct;
1341  cTValue *tv;
1342  MMS mm = MM_call;
1343  if (id == CTID_CTYPEID) {
1344    id = crec_constructor(J, cd, J->base[0]);
1345    mm = MM_new;
1346  } else if (crec_call(J, rd, cd)) {
1347    return;
1348  }
1349  /* Record ctype __call/__new metamethod. */
1350  ct = ctype_raw(cts, id);
1351  tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1352  if (tv) {
1353    if (tvisfunc(tv)) {
1354      crec_tailcall(J, rd, tv);
1355      return;
1356    }
1357  } else if (mm == MM_new) {
1358    crec_alloc(J, rd, id);
1359    return;
1360  }
1361  /* No metamethod or NYI: non-function metamethods. */
1362  lj_trace_err(J, LJ_TRERR_BADTYPE);
1363}
1364
1365static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1366{
1367  if (sp[0] && sp[1] && ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1368    IRType dt;
1369    CTypeID id;
1370    TRef tr;
1371    MSize i;
1372    IROp op;
1373    lj_needsplit(J);
1374    if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1375	((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1376      dt = IRT_U64; id = CTID_UINT64;
1377    } else {
1378      dt = IRT_I64; id = CTID_INT64;
1379      if (mm < MM_add &&
1380	  !((s[0]->info | s[1]->info) & CTF_FP) &&
1381	  s[0]->size == 4 && s[1]->size == 4) {  /* Try to narrow comparison. */
1382	if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1383	    (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1384	  dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1385	  goto comp;
1386	} else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1387	  dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1388	  goto comp;
1389	}
1390      }
1391    }
1392    for (i = 0; i < 2; i++) {
1393      IRType st = tref_type(sp[i]);
1394      if (st == IRT_NUM || st == IRT_FLOAT)
1395	sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
1396      else if (!(st == IRT_I64 || st == IRT_U64))
1397	sp[i] = emitconv(sp[i], dt, IRT_INT,
1398			 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1399    }
1400    if (mm < MM_add) {
1401    comp:
1402      /* Assume true comparison. Fixup and emit pending guard later. */
1403      if (mm == MM_eq) {
1404	op = IR_EQ;
1405      } else {
1406	op = mm == MM_lt ? IR_LT : IR_LE;
1407	if (dt == IRT_U32 || dt == IRT_U64)
1408	  op += (IR_ULT-IR_LT);
1409      }
1410      lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1411      J->postproc = LJ_POST_FIXGUARD;
1412      return TREF_TRUE;
1413    } else {
1414      tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1415    }
1416    return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1417  }
1418  return 0;
1419}
1420
1421static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1422{
1423  CTState *cts = ctype_ctsG(J2G(J));
1424  CType *ctp = s[0];
1425  if (!(sp[0] && sp[1])) return 0;
1426  if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1427    if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1428	(ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1429      if (mm == MM_sub) {  /* Pointer difference. */
1430	TRef tr;
1431	CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1432	if (sz == 0 || (sz & (sz-1)) != 0)
1433	  return 0;  /* NYI: integer division. */
1434	tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1435	tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1436#if LJ_64
1437	tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1438#endif
1439	return tr;
1440      } else {  /* Pointer comparison (unsigned). */
1441	/* Assume true comparison. Fixup and emit pending guard later. */
1442	IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1443	lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1444	J->postproc = LJ_POST_FIXGUARD;
1445	return TREF_TRUE;
1446      }
1447    }
1448    if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1449      return 0;
1450  } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1451	     (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1452    TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr;  /* Swap pointer and index. */
1453    ctp = s[1];
1454  } else {
1455    return 0;
1456  }
1457  {
1458    TRef tr = sp[1];
1459    IRType t = tref_type(tr);
1460    CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1461    CTypeID id;
1462#if LJ_64
1463    if (t == IRT_NUM || t == IRT_FLOAT)
1464      tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
1465    else if (!(t == IRT_I64 || t == IRT_U64))
1466      tr = emitconv(tr, IRT_INTP, IRT_INT,
1467		    ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1468#else
1469    if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1470      tr = emitconv(tr, IRT_INTP, t,
1471		    (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
1472    }
1473#endif
1474    tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1475    tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1476    id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1477			 CTSIZE_PTR);
1478    return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1479  }
1480}
1481
1482/* Record ctype arithmetic metamethods. */
1483static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1484			    RecordFFData *rd)
1485{
1486  cTValue *tv = NULL;
1487  if (J->base[0]) {
1488    if (tviscdata(&rd->argv[0])) {
1489      CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1490      CType *ct = ctype_raw(cts, id);
1491      if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1492      tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1493    }
1494    if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1495      CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1496      CType *ct = ctype_raw(cts, id);
1497      if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1498      tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1499    }
1500  }
1501  if (tv) {
1502    if (tvisfunc(tv)) {
1503      crec_tailcall(J, rd, tv);
1504      return 0;
1505    }  /* NYI: non-function metamethods. */
1506  } else if ((MMS)rd->data == MM_eq) {  /* Fallback cdata pointer comparison. */
1507    if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
1508      /* Assume true comparison. Fixup and emit pending guard later. */
1509      lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
1510      J->postproc = LJ_POST_FIXGUARD;
1511      return TREF_TRUE;
1512    } else {
1513      return TREF_FALSE;
1514    }
1515  }
1516  lj_trace_err(J, LJ_TRERR_BADTYPE);
1517  return 0;
1518}
1519
1520void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1521{
1522  CTState *cts = ctype_cts(J->L);
1523  MMS mm = (MMS)rd->data;
1524  TRef sp[2];
1525  CType *s[2];
1526  MSize i;
1527  for (i = 0; i < 2; i++) {
1528    TRef tr = J->base[i];
1529    CType *ct = ctype_get(cts, CTID_DOUBLE);
1530    if (!tr) {
1531      lj_trace_err(J, LJ_TRERR_BADTYPE);
1532    } else if (tref_iscdata(tr)) {
1533      CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1534      IRType t;
1535      ct = ctype_raw(cts, id);
1536      t = crec_ct2irt(cts, ct);
1537      if (ctype_isptr(ct->info)) {  /* Resolve pointer or reference. */
1538	tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1539	if (ctype_isref(ct->info)) {
1540	  ct = ctype_rawchild(cts, ct);
1541	  t = crec_ct2irt(cts, ct);
1542	}
1543      } else if (t == IRT_I64 || t == IRT_U64) {
1544	tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1545	lj_needsplit(J);
1546	goto ok;
1547      } else if (t == IRT_INT || t == IRT_U32) {
1548	tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1549	if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1550	goto ok;
1551      } else if (ctype_isfunc(ct->info)) {
1552	CTypeID id0 = i ? ctype_typeid(cts, s[0]) : 0;
1553	tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1554	ct = ctype_get(cts,
1555	  lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1556	if (i) {
1557	  s[0] = ctype_get(cts, id0);  /* cts->tab may have been reallocated. */
1558	}
1559	goto ok;
1560      } else {
1561	tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1562      }
1563      if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1564      if (ctype_isnum(ct->info)) {
1565	if (t == IRT_CDATA) {
1566	  tr = 0;
1567	} else {
1568	  if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1569	  tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1570	}
1571      }
1572    } else if (tref_isnil(tr)) {
1573      if (!(mm == MM_len || mm == MM_eq || mm == MM_lt || mm == MM_le))
1574	lj_trace_err(J, LJ_TRERR_BADTYPE);
1575      tr = lj_ir_kptr(J, NULL);
1576      ct = ctype_get(cts, CTID_P_VOID);
1577    } else if (tref_isinteger(tr)) {
1578      ct = ctype_get(cts, CTID_INT32);
1579    } else if (tref_isstr(tr)) {
1580      TRef tr2 = J->base[1-i];
1581      CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1582      ct = ctype_raw(cts, id);
1583      if (ctype_isenum(ct->info)) {  /* Match string against enum constant. */
1584	GCstr *str = strV(&rd->argv[i]);
1585	CTSize ofs;
1586	CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1587	if (cct && ctype_isconstval(cct->info)) {
1588	  /* Specialize to the name of the enum constant. */
1589	  emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1590	  ct = ctype_child(cts, cct);
1591	  tr = lj_ir_kint(J, (int32_t)ofs);
1592	} else {  /* Interpreter will throw or return false. */
1593	  lj_trace_err(J, LJ_TRERR_BADTYPE);
1594	}
1595      } else if (ctype_isptr(ct->info)) {
1596	tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
1597      } else {
1598	lj_trace_err(J, LJ_TRERR_BADTYPE);
1599      }
1600    } else if (!tref_isnum(tr)) {
1601      tr = 0;
1602      ct = ctype_get(cts, CTID_P_VOID);
1603    }
1604  ok:
1605    s[i] = ct;
1606    sp[i] = tr;
1607  }
1608  {
1609    TRef tr;
1610    if ((mm == MM_len || mm == MM_concat ||
1611	 (!(tr = crec_arith_int64(J, sp, s, mm)) &&
1612	  !(tr = crec_arith_ptr(J, sp, s, mm)))) &&
1613	!(tr = crec_arith_meta(J, sp, s, cts, rd)))
1614      return;
1615    J->base[0] = tr;
1616    /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1617    if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1618	!irt_isguard(J->guardemit)) {
1619      const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1620      if (bc_op(*pc) <= BC_ISNEP) {
1621	J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
1622	J->postproc = LJ_POST_FIXCOMP;
1623      }
1624    }
1625  }
1626}
1627
1628/* -- C library namespace metamethods ------------------------------------- */
1629
1630void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1631{
1632  CTState *cts = ctype_ctsG(J2G(J));
1633  if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1634      udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1635    CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1636    GCstr *name = strV(&rd->argv[1]);
1637    CType *ct;
1638    CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1639    cTValue *tv = lj_tab_getstr(cl->cache, name);
1640    rd->nres = rd->data;
1641    if (id && tv && !tvisnil(tv)) {
1642      /* Specialize to the symbol name and make the result a constant. */
1643      emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1644      if (ctype_isconstval(ct->info)) {
1645	if (ct->size >= 0x80000000u &&
1646	    (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1647	  J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1648	else
1649	  J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1650      } else if (ctype_isextern(ct->info)) {
1651	CTypeID sid = ctype_cid(ct->info);
1652	void *sp = *(void **)cdataptr(cdataV(tv));
1653	TRef ptr;
1654	ct = ctype_raw(cts, sid);
1655	if (LJ_64 && !checkptr32(sp))
1656	  ptr = lj_ir_kintp(J, (uintptr_t)sp);
1657	else
1658	  ptr = lj_ir_kptr(J, sp);
1659	if (rd->data) {
1660	  J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1661	} else {
1662	  J->needsnap = 1;
1663	  crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1664	}
1665      } else {
1666	J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1667      }
1668    } else {
1669      lj_trace_err(J, LJ_TRERR_NOCACHE);
1670    }
1671  }  /* else: interpreter will throw. */
1672}
1673
1674/* -- FFI library functions ----------------------------------------------- */
1675
1676static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1677{
1678  return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1679}
1680
1681void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1682{
1683  crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1684}
1685
1686void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1687{
1688  UNUSED(rd);
1689  if (J->base[0])
1690    lj_trace_err(J, LJ_TRERR_NYICALL);
1691  J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1692}
1693
1694void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1695{
1696  CTState *cts = ctype_ctsG(J2G(J));
1697  TRef tr = J->base[0];
1698  if (tr) {
1699    TRef trlen = J->base[1];
1700    if (!tref_isnil(trlen)) {
1701      trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1702      tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1703    } else {
1704      tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1705      trlen = lj_ir_call(J, IRCALL_strlen, tr);
1706    }
1707    J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1708  }  /* else: interpreter will throw. */
1709}
1710
1711void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1712{
1713  CTState *cts = ctype_ctsG(J2G(J));
1714  TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1715  if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1716    trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1717    trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1718    if (trlen) {
1719      trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1720    } else {
1721      trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1722      trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1723    }
1724    rd->nres = 0;
1725    crec_copy(J, trdst, trsrc, trlen, NULL);
1726  }  /* else: interpreter will throw. */
1727}
1728
1729void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1730{
1731  CTState *cts = ctype_ctsG(J2G(J));
1732  TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1733  if (trdst && trlen) {
1734    CTSize step = 1;
1735    if (tviscdata(&rd->argv[0])) {  /* Get alignment of original destination. */
1736      CTSize sz;
1737      CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1738      if (ctype_isptr(ct->info))
1739	ct = ctype_rawchild(cts, ct);
1740      step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1741    }
1742    trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1743    trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1744    if (trfill)
1745      trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1746    else
1747      trfill = lj_ir_kint(J, 0);
1748    rd->nres = 0;
1749    crec_fill(J, trdst, trlen, trfill, step);
1750  }  /* else: interpreter will throw. */
1751}
1752
1753void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1754{
1755  if (tref_iscdata(J->base[0])) {
1756    TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1757    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1758			lj_ir_kint(J, CTID_CTYPEID), trid);
1759  } else {
1760    setfuncV(J->L, &J->errinfo, J->fn);
1761    lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1762  }
1763}
1764
1765void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1766{
1767  argv2ctype(J, J->base[0], &rd->argv[0]);
1768  if (tref_iscdata(J->base[1])) {
1769    argv2ctype(J, J->base[1], &rd->argv[1]);
1770    J->postproc = LJ_POST_FIXBOOL;
1771    J->base[0] = TREF_TRUE;
1772  } else {
1773    J->base[0] = TREF_FALSE;
1774  }
1775}
1776
1777void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1778{
1779  if (tref_isstr(J->base[0])) {
1780    /* Specialize to the ABI string to make the boolean result a constant. */
1781    emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1782    J->postproc = LJ_POST_FIXBOOL;
1783    J->base[0] = TREF_TRUE;
1784  } else {
1785    lj_trace_err(J, LJ_TRERR_BADTYPE);
1786  }
1787}
1788
1789/* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1790void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1791{
1792  CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1793  if (rd->data == FF_ffi_sizeof) {
1794    CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1795    if (ctype_isvltype(ct->info))
1796      lj_trace_err(J, LJ_TRERR_BADTYPE);
1797  } else if (rd->data == FF_ffi_offsetof) {  /* Specialize to the field name. */
1798    if (!tref_isstr(J->base[1]))
1799      lj_trace_err(J, LJ_TRERR_BADTYPE);
1800    emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1801    rd->nres = 3;  /* Just in case. */
1802  }
1803  J->postproc = LJ_POST_FIXCONST;
1804  J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1805}
1806
1807void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1808{
1809  argv2cdata(J, J->base[0], &rd->argv[0]);
1810  if (!J->base[1])
1811    lj_trace_err(J, LJ_TRERR_BADTYPE);
1812  crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
1813}
1814
1815/* -- 64 bit bit.* library functions -------------------------------------- */
1816
1817/* Determine bit operation type from argument type. */
1818static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
1819{
1820  if (tviscdata(tv)) {
1821    CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
1822    if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1823    if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
1824	CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
1825      return CTID_UINT64;  /* Use uint64_t, since it has the highest rank. */
1826    return CTID_INT64;  /* Otherwise use int64_t. */
1827  }
1828  return 0;  /* Use regular 32 bit ops. */
1829}
1830
1831static TRef crec_bit64_arg(jit_State *J, CType *d, TRef sp, TValue *sval)
1832{
1833  if (LJ_UNLIKELY(tref_isstr(sp))) {
1834    if (lj_strscan_num(strV(sval), sval)) {
1835      sp = emitir(IRTG(IR_STRTO, IRT_NUM), sp, 0);
1836    }  /* else: interpreter will throw. */
1837  }
1838  return crec_ct_tv(J, d, 0, sp, sval);
1839}
1840
1841void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
1842{
1843  CTState *cts = ctype_ctsG(J2G(J));
1844  TRef tr = crec_bit64_arg(J, ctype_get(cts, CTID_INT64),
1845			   J->base[0], &rd->argv[0]);
1846  if (!tref_isinteger(tr))
1847    tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
1848  J->base[0] = tr;
1849}
1850
1851int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
1852{
1853  CTState *cts = ctype_ctsG(J2G(J));
1854  CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1855  if (id) {
1856    TRef tr = crec_bit64_arg(J, ctype_get(cts, id), J->base[0], &rd->argv[0]);
1857    tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
1858    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1859    return 1;
1860  }
1861  return 0;
1862}
1863
1864int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
1865{
1866  CTState *cts = ctype_ctsG(J2G(J));
1867  CTypeID id = 0;
1868  MSize i;
1869  for (i = 0; J->base[i] != 0; i++) {
1870    CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
1871    if (id < aid) id = aid;  /* Determine highest type rank of all arguments. */
1872  }
1873  if (id) {
1874    CType *ct = ctype_get(cts, id);
1875    uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
1876    TRef tr = crec_bit64_arg(J, ct, J->base[0], &rd->argv[0]);
1877    for (i = 1; J->base[i] != 0; i++) {
1878      TRef tr2 = crec_bit64_arg(J, ct, J->base[i], &rd->argv[i]);
1879      tr = emitir(ot, tr, tr2);
1880    }
1881    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1882    return 1;
1883  }
1884  return 0;
1885}
1886
1887int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
1888{
1889  CTState *cts = ctype_ctsG(J2G(J));
1890  CTypeID id;
1891  TRef tsh = 0;
1892  if (J->base[0] && tref_iscdata(J->base[1])) {
1893    tsh = crec_bit64_arg(J, ctype_get(cts, CTID_INT64),
1894			 J->base[1], &rd->argv[1]);
1895    if (LJ_32 && !tref_isinteger(tsh))
1896      tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
1897    J->base[1] = tsh;
1898  }
1899  id = crec_bit64_type(cts, &rd->argv[0]);
1900  if (id) {
1901    TRef tr = crec_bit64_arg(J, ctype_get(cts, id), J->base[0], &rd->argv[0]);
1902    uint32_t op = rd->data;
1903    IRType t;
1904    if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
1905    t = tref_isinteger(tsh) ? IRT_INT : tref_type(tsh);
1906    if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
1907	!tref_isk(tsh))
1908      tsh = emitir(IRT(IR_BAND, t), tsh, lj_ir_kint(J, 63));
1909#ifdef LJ_TARGET_UNIFYROT
1910    if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
1911      op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
1912      tsh = emitir(IRT(IR_NEG, t), tsh, tsh);
1913    }
1914#endif
1915    tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
1916    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1917    return 1;
1918  }
1919  return 0;
1920}
1921
1922TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
1923{
1924  CTState *cts = ctype_ctsG(J2G(J));
1925  CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1926  TRef tr, trsf = J->base[1];
1927  SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
1928  int32_t n;
1929  if (trsf) {
1930    CTypeID id2 = 0;
1931    n = (int32_t)lj_carith_check64(J->L, 2, &id2);
1932    if (id2)
1933      trsf = crec_bit64_arg(J, ctype_get(cts, CTID_INT32), trsf, &rd->argv[1]);
1934    else
1935      trsf = lj_opt_narrow_tobit(J, trsf);
1936    emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n));  /* Specialize to n. */
1937  } else {
1938    n = id ? 16 : 8;
1939  }
1940  if (n < 0) { n = (int32_t)(~n+1u); sf |= STRFMT_F_UPPER; }
1941  if ((uint32_t)n > 254) n = 254;
1942  sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
1943  if (id) {
1944    tr = crec_bit64_arg(J, ctype_get(cts, id), J->base[0], &rd->argv[0]);
1945    if (n < 16)
1946      tr = emitir(IRT(IR_BAND, IRT_U64), tr,
1947		  lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
1948  } else {
1949    tr = lj_opt_narrow_tobit(J, J->base[0]);
1950    if (n < 8)
1951      tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
1952    tr = emitconv(tr, IRT_U64, IRT_INT, 0);  /* No sign-extension. */
1953    lj_needsplit(J);
1954  }
1955  return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
1956}
1957
1958/* -- Miscellaneous library functions ------------------------------------- */
1959
1960void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1961{
1962  CTState *cts = ctype_ctsG(J2G(J));
1963  CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1964  if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1965  if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1966    if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1967	!(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1968      d = ctype_get(cts, CTID_INT32);
1969    else
1970      d = ctype_get(cts, CTID_DOUBLE);
1971    J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1972  } else {
1973    /* Specialize to the ctype that couldn't be converted. */
1974    argv2cdata(J, J->base[0], &rd->argv[0]);
1975    J->base[0] = TREF_NIL;
1976  }
1977}
1978
1979TRef lj_crecord_loadiu64(jit_State *J, TRef tr, cTValue *o)
1980{
1981  CTypeID id = argv2cdata(J, tr, o)->ctypeid;
1982  if (!(id == CTID_INT64 || id == CTID_UINT64))
1983    lj_trace_err(J, LJ_TRERR_BADTYPE);
1984  lj_needsplit(J);
1985  return emitir(IRT(IR_FLOAD, id == CTID_INT64 ? IRT_I64 : IRT_U64), tr,
1986		IRFL_CDATA_INT64);
1987}
1988
1989#if LJ_HASBUFFER
1990TRef lj_crecord_topcvoid(jit_State *J, TRef tr, cTValue *o)
1991{
1992  CTState *cts = ctype_ctsG(J2G(J));
1993  if (!tref_iscdata(tr)) lj_trace_err(J, LJ_TRERR_BADTYPE);
1994  return crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, o);
1995}
1996
1997TRef lj_crecord_topuint8(jit_State *J, TRef tr)
1998{
1999  return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, CTID_P_UINT8), tr);
2000}
2001#endif
2002
2003#undef IR
2004#undef emitir
2005#undef emitconv
2006
2007#endif