luna - luajit/src/lj

   1/*
   2** FFI C call handling.
   3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
   4*/
   5
   6#include "lj_obj.h"
   7
   8#if LJ_HASFFI
   9
  10#include "lj_gc.h"
  11#include "lj_err.h"
  12#include "lj_tab.h"
  13#include "lj_ctype.h"
  14#include "lj_cconv.h"
  15#include "lj_cdata.h"
  16#include "lj_ccall.h"
  17#include "lj_trace.h"
  18
  19/* Target-specific handling of register arguments. */
  20#if LJ_TARGET_X86
  21/* -- x86 calling conventions --------------------------------------------- */
  22
  23#define CCALL_PUSH(arg) \
  24  *(GPRArg *)((uint8_t *)cc->stack + nsp) = (GPRArg)(arg), nsp += CTSIZE_PTR
  25
  26#if LJ_ABI_WIN
  27
  28#define CCALL_HANDLE_STRUCTRET \
  29  /* Return structs bigger than 8 by reference (on stack only). */ \
  30  cc->retref = (sz > 8); \
  31  if (cc->retref) CCALL_PUSH(dp);
  32
  33#define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  34
  35#else
  36
  37#if LJ_TARGET_OSX
  38
  39#define CCALL_HANDLE_STRUCTRET \
  40  /* Return structs of size 1, 2, 4 or 8 in registers. */ \
  41  cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
  42  if (cc->retref) { \
  43    if (ngpr < maxgpr) \
  44      cc->gpr[ngpr++] = (GPRArg)dp; \
  45    else \
  46      CCALL_PUSH(dp); \
  47  } else {  /* Struct with single FP field ends up in FPR. */ \
  48    cc->resx87 = ccall_classify_struct(cts, ctr); \
  49  }
  50
  51#define CCALL_HANDLE_STRUCTRET2 \
  52  if (cc->resx87) sp = (uint8_t *)&cc->fpr[0]; \
  53  memcpy(dp, sp, ctr->size);
  54
  55#else
  56
  57#define CCALL_HANDLE_STRUCTRET \
  58  cc->retref = 1;  /* Return all structs by reference (in reg or on stack). */ \
  59  if (ngpr < maxgpr) \
  60    cc->gpr[ngpr++] = (GPRArg)dp; \
  61  else \
  62    CCALL_PUSH(dp);
  63
  64#endif
  65
  66#define CCALL_HANDLE_COMPLEXRET \
  67  /* Return complex float in GPRs and complex double by reference. */ \
  68  cc->retref = (sz > 8); \
  69  if (cc->retref) { \
  70    if (ngpr < maxgpr) \
  71      cc->gpr[ngpr++] = (GPRArg)dp; \
  72    else \
  73      CCALL_PUSH(dp); \
  74  }
  75
  76#endif
  77
  78#define CCALL_HANDLE_COMPLEXRET2 \
  79  if (!cc->retref) \
  80    *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  81
  82#define CCALL_HANDLE_STRUCTARG \
  83  ngpr = maxgpr;  /* Pass all structs by value on the stack. */
  84
  85#define CCALL_HANDLE_COMPLEXARG \
  86  isfp = 1;  /* Pass complex by value on stack. */
  87
  88#define CCALL_HANDLE_REGARG \
  89  if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
  90    if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
  91      if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
  92    } else if (ngpr + 1 <= maxgpr) { \
  93      dp = &cc->gpr[ngpr]; \
  94      ngpr += n; \
  95      goto done; \
  96    } \
  97  }
  98
  99#elif LJ_TARGET_X64 && LJ_ABI_WIN
 100/* -- Windows/x64 calling conventions ------------------------------------- */
 101
 102#define CCALL_HANDLE_STRUCTRET \
 103  /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
 104  cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
 105  if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 106
 107#define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
 108
 109#define CCALL_HANDLE_COMPLEXRET2 \
 110  if (!cc->retref) \
 111    *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
 112
 113#define CCALL_HANDLE_STRUCTARG \
 114  /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
 115  if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
 116    rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 117    sz = CTSIZE_PTR;  /* Pass all other structs by reference. */ \
 118  }
 119
 120#define CCALL_HANDLE_COMPLEXARG \
 121  /* Pass complex float in a GPR and complex double by reference. */ \
 122  if (sz != 2*sizeof(float)) { \
 123    rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 124    sz = CTSIZE_PTR; \
 125  }
 126
 127/* Windows/x64 argument registers are strictly positional (use ngpr). */
 128#define CCALL_HANDLE_REGARG \
 129  if (isfp) { \
 130    if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
 131  } else { \
 132    if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
 133  }
 134
 135#elif LJ_TARGET_X64
 136/* -- POSIX/x64 calling conventions --------------------------------------- */
 137
 138#define CCALL_HANDLE_STRUCTRET \
 139  int rcl[2]; rcl[0] = rcl[1] = 0; \
 140  if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
 141    cc->retref = 1;  /* Return struct by reference. */ \
 142    cc->gpr[ngpr++] = (GPRArg)dp; \
 143  } else { \
 144    cc->retref = 0;  /* Return small structs in registers. */ \
 145  }
 146
 147#define CCALL_HANDLE_STRUCTRET2 \
 148  int rcl[2]; rcl[0] = rcl[1] = 0; \
 149  ccall_classify_struct(cts, ctr, rcl, 0); \
 150  ccall_struct_ret(cc, rcl, dp, ctr->size);
 151
 152#define CCALL_HANDLE_COMPLEXRET \
 153  /* Complex values are returned in one or two FPRs. */ \
 154  cc->retref = 0;
 155
 156#define CCALL_HANDLE_COMPLEXRET2 \
 157  if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPR. */ \
 158    *(int64_t *)dp = cc->fpr[0].l[0]; \
 159  } else {  /* Copy non-contiguous complex double from FPRs. */ \
 160    ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
 161    ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
 162  }
 163
 164#define CCALL_HANDLE_STRUCTARG \
 165  int rcl[2]; rcl[0] = rcl[1] = 0; \
 166  if (!ccall_classify_struct(cts, d, rcl, 0)) { \
 167    cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
 168    if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
 169    nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
 170    continue; \
 171  }  /* Pass all other structs by value on stack. */
 172
 173#define CCALL_HANDLE_COMPLEXARG \
 174  isfp = 2;  /* Pass complex in FPRs or on stack. Needs postprocessing. */
 175
 176#define CCALL_HANDLE_REGARG \
 177  if (isfp) {  /* Try to pass argument in FPRs. */ \
 178    int n2 = ctype_isvector(d->info) ? 1 : n; \
 179    if (nfpr + n2 <= CCALL_NARG_FPR) { \
 180      dp = &cc->fpr[nfpr]; \
 181      nfpr += n2; \
 182      goto done; \
 183    } \
 184  } else {  /* Try to pass argument in GPRs. */ \
 185    /* Note that reordering is explicitly allowed in the x64 ABI. */ \
 186    if (n <= 2 && ngpr + n <= maxgpr) { \
 187      dp = &cc->gpr[ngpr]; \
 188      ngpr += n; \
 189      goto done; \
 190    } \
 191  }
 192
 193#elif LJ_TARGET_ARM
 194/* -- ARM calling conventions --------------------------------------------- */
 195
 196#if LJ_ABI_SOFTFP
 197
 198#define CCALL_HANDLE_STRUCTRET \
 199  /* Return structs of size <= 4 in a GPR. */ \
 200  cc->retref = !(sz <= 4); \
 201  if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 202
 203#define CCALL_HANDLE_COMPLEXRET \
 204  cc->retref = 1;  /* Return all complex values by reference. */ \
 205  cc->gpr[ngpr++] = (GPRArg)dp;
 206
 207#define CCALL_HANDLE_COMPLEXRET2 \
 208  UNUSED(dp); /* Nothing to do. */
 209
 210#define CCALL_HANDLE_STRUCTARG \
 211  /* Pass all structs by value in registers and/or on the stack. */
 212
 213#define CCALL_HANDLE_COMPLEXARG \
 214  /* Pass complex by value in 2 or 4 GPRs. */
 215
 216#define CCALL_HANDLE_REGARG_FP1
 217#define CCALL_HANDLE_REGARG_FP2
 218
 219#else
 220
 221#define CCALL_HANDLE_STRUCTRET \
 222  cc->retref = !ccall_classify_struct(cts, ctr, ct); \
 223  if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 224
 225#define CCALL_HANDLE_STRUCTRET2 \
 226  if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
 227  memcpy(dp, sp, ctr->size);
 228
 229#define CCALL_HANDLE_COMPLEXRET \
 230  if (!(ct->info & CTF_VARARG)) cc->retref = 0;  /* Return complex in FPRs. */
 231
 232#define CCALL_HANDLE_COMPLEXRET2 \
 233  if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
 234
 235#define CCALL_HANDLE_STRUCTARG \
 236  isfp = (ccall_classify_struct(cts, d, ct) > 1);
 237  /* Pass all structs by value in registers and/or on the stack. */
 238
 239#define CCALL_HANDLE_COMPLEXARG \
 240  isfp = 1;  /* Pass complex by value in FPRs or on stack. */
 241
 242#define CCALL_HANDLE_REGARG_FP1 \
 243  if (isfp && !(ct->info & CTF_VARARG)) { \
 244    if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 245      if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
 246	dp = &cc->fpr[nfpr]; \
 247	nfpr += (n >> 1); \
 248	goto done; \
 249      } \
 250    } else { \
 251      if (sz > 1 && fprodd != nfpr) fprodd = 0; \
 252      if (fprodd) { \
 253	if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
 254	  dp = (void *)&cc->fpr[fprodd-1].f[1]; \
 255	  nfpr += (n >> 1); \
 256	  if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
 257	  goto done; \
 258	} \
 259      } else { \
 260	if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
 261	  dp = (void *)&cc->fpr[nfpr]; \
 262	  nfpr += (n >> 1); \
 263	  if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
 264	  goto done; \
 265	} \
 266      } \
 267    } \
 268    fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 269  } else {
 270
 271#define CCALL_HANDLE_REGARG_FP2	}
 272
 273#endif
 274
 275#define CCALL_HANDLE_REGARG \
 276  CCALL_HANDLE_REGARG_FP1 \
 277  if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 278    if (ngpr < maxgpr) \
 279      ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 280  } \
 281  if (ngpr < maxgpr) { \
 282    dp = &cc->gpr[ngpr]; \
 283    if (ngpr + n > maxgpr) { \
 284      nsp += (ngpr + n - maxgpr) * CTSIZE_PTR;  /* Assumes contiguous gpr/stack fields. */ \
 285      if (nsp > CCALL_SIZE_STACK) goto err_nyi;  /* Too many arguments. */ \
 286      ngpr = maxgpr; \
 287    } else { \
 288      ngpr += n; \
 289    } \
 290    goto done; \
 291  } CCALL_HANDLE_REGARG_FP2
 292
 293#define CCALL_HANDLE_RET \
 294  if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
 295
 296#elif LJ_TARGET_ARM64
 297/* -- ARM64 calling conventions ------------------------------------------- */
 298
 299#define CCALL_HANDLE_STRUCTRET \
 300  cc->retref = !ccall_classify_struct(cts, ctr); \
 301  if (cc->retref) cc->retp = dp;
 302
 303#define CCALL_HANDLE_STRUCTRET2 \
 304  unsigned int cl = ccall_classify_struct(cts, ctr); \
 305  if ((cl & 4)) { /* Combine float HFA from separate registers. */ \
 306    CTSize i = (cl >> 8) - 1; \
 307    do { ((uint32_t *)dp)[i] = cc->fpr[i].lo; } while (i--); \
 308  } else { \
 309    if (cl > 1) sp = (uint8_t *)&cc->fpr[0]; \
 310    memcpy(dp, sp, ctr->size); \
 311  }
 312
 313#define CCALL_HANDLE_COMPLEXRET \
 314  /* Complex values are returned in one or two FPRs. */ \
 315  cc->retref = 0;
 316
 317#define CCALL_HANDLE_COMPLEXRET2 \
 318  if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 319    ((float *)dp)[0] = cc->fpr[0].f; \
 320    ((float *)dp)[1] = cc->fpr[1].f; \
 321  } else {  /* Copy complex double from FPRs. */ \
 322    ((double *)dp)[0] = cc->fpr[0].d; \
 323    ((double *)dp)[1] = cc->fpr[1].d; \
 324  }
 325
 326#define CCALL_HANDLE_STRUCTARG \
 327  unsigned int cl = ccall_classify_struct(cts, d); \
 328  if (cl == 0) {  /* Pass struct by reference. */ \
 329    rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 330    sz = CTSIZE_PTR; \
 331  } else if (cl > 1) {  /* Pass struct in FPRs or on stack. */ \
 332    isfp = (cl & 4) ? 2 : 1; \
 333  }  /* else: Pass struct in GPRs or on stack. */
 334
 335#define CCALL_HANDLE_COMPLEXARG \
 336  /* Pass complex by value in separate (!) FPRs or on stack. */ \
 337  isfp = sz == 2*sizeof(float) ? 2 : 1;
 338
 339#define CCALL_HANDLE_REGARG \
 340  if (LJ_TARGET_OSX && isva) { \
 341    /* IOS: All variadic arguments are on the stack. */ \
 342  } else if (isfp) {  /* Try to pass argument in FPRs. */ \
 343    int n2 = ctype_isvector(d->info) ? 1 : \
 344	     isfp == 1 ? n : (d->size >> (4-isfp)); \
 345    if (nfpr + n2 <= CCALL_NARG_FPR) { \
 346      dp = &cc->fpr[nfpr]; \
 347      nfpr += n2; \
 348      goto done; \
 349    } else { \
 350      nfpr = CCALL_NARG_FPR;  /* Prevent reordering. */ \
 351    } \
 352  } else {  /* Try to pass argument in GPRs. */ \
 353    if (!LJ_TARGET_OSX && (d->info & CTF_ALIGN) > CTALIGN_PTR) \
 354      ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 355    if (ngpr + n <= maxgpr) { \
 356      dp = &cc->gpr[ngpr]; \
 357      ngpr += n; \
 358      goto done; \
 359    } else { \
 360      ngpr = maxgpr;  /* Prevent reordering. */ \
 361    } \
 362  }
 363
 364#if LJ_BE
 365#define CCALL_HANDLE_RET \
 366  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 367    sp = (uint8_t *)&cc->fpr[0].f;
 368#endif
 369
 370
 371#elif LJ_TARGET_PPC
 372/* -- PPC calling conventions --------------------------------------------- */
 373
 374#define CCALL_HANDLE_STRUCTRET \
 375  cc->retref = 1;  /* Return all structs by reference. */ \
 376  cc->gpr[ngpr++] = (GPRArg)dp;
 377
 378#define CCALL_HANDLE_COMPLEXRET \
 379  /* Complex values are returned in 2 or 4 GPRs. */ \
 380  cc->retref = 0;
 381
 382#define CCALL_HANDLE_COMPLEXRET2 \
 383  memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 384
 385#define CCALL_HANDLE_STRUCTARG \
 386  rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 387  sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 388
 389#define CCALL_HANDLE_COMPLEXARG \
 390  /* Pass complex by value in 2 or 4 GPRs. */
 391
 392#define CCALL_HANDLE_GPR \
 393  /* Try to pass argument in GPRs. */ \
 394  if (n > 1) { \
 395    /* int64_t or complex (float). */ \
 396    lj_assertL(n == 2 || n == 4, "bad GPR size %d", n); \
 397    if (ctype_isinteger(d->info) || ctype_isfp(d->info)) \
 398      ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 399    else if (ngpr + n > maxgpr) \
 400      ngpr = maxgpr;  /* Prevent reordering. */ \
 401  } \
 402  if (ngpr + n <= maxgpr) { \
 403    dp = &cc->gpr[ngpr]; \
 404    ngpr += n; \
 405    goto done; \
 406  } \
 407
 408#if LJ_ABI_SOFTFP
 409#define CCALL_HANDLE_REGARG  CCALL_HANDLE_GPR
 410#else
 411#define CCALL_HANDLE_REGARG \
 412  if (isfp) {  /* Try to pass argument in FPRs. */ \
 413    if (nfpr + 1 <= CCALL_NARG_FPR) { \
 414      dp = &cc->fpr[nfpr]; \
 415      nfpr += 1; \
 416      d = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 417      goto done; \
 418    } \
 419  } else { \
 420    CCALL_HANDLE_GPR \
 421  }
 422#endif
 423
 424#if !LJ_ABI_SOFTFP
 425#define CCALL_HANDLE_RET \
 426  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 427    ctr = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */
 428#endif
 429
 430#elif LJ_TARGET_MIPS32
 431/* -- MIPS o32 calling conventions ---------------------------------------- */
 432
 433#define CCALL_HANDLE_STRUCTRET \
 434  cc->retref = 1;  /* Return all structs by reference. */ \
 435  cc->gpr[ngpr++] = (GPRArg)dp;
 436
 437#define CCALL_HANDLE_COMPLEXRET \
 438  /* Complex values are returned in 1 or 2 FPRs. */ \
 439  cc->retref = 0;
 440
 441#if LJ_ABI_SOFTFP
 442#define CCALL_HANDLE_COMPLEXRET2 \
 443  if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from GPRs. */ \
 444    ((intptr_t *)dp)[0] = cc->gpr[0]; \
 445    ((intptr_t *)dp)[1] = cc->gpr[1]; \
 446  } else {  /* Copy complex double from GPRs. */ \
 447    ((intptr_t *)dp)[0] = cc->gpr[0]; \
 448    ((intptr_t *)dp)[1] = cc->gpr[1]; \
 449    ((intptr_t *)dp)[2] = cc->gpr[2]; \
 450    ((intptr_t *)dp)[3] = cc->gpr[3]; \
 451  }
 452#else
 453#define CCALL_HANDLE_COMPLEXRET2 \
 454  if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 455    ((float *)dp)[0] = cc->fpr[0].f; \
 456    ((float *)dp)[1] = cc->fpr[1].f; \
 457  } else {  /* Copy complex double from FPRs. */ \
 458    ((double *)dp)[0] = cc->fpr[0].d; \
 459    ((double *)dp)[1] = cc->fpr[1].d; \
 460  }
 461#endif
 462
 463#define CCALL_HANDLE_STRUCTARG \
 464  /* Pass all structs by value in registers and/or on the stack. */
 465
 466#define CCALL_HANDLE_COMPLEXARG \
 467  /* Pass complex by value in 2 or 4 GPRs. */
 468
 469#define CCALL_HANDLE_GPR \
 470  if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
 471    ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 472  if (ngpr < maxgpr) { \
 473    dp = &cc->gpr[ngpr]; \
 474    if (ngpr + n > maxgpr) { \
 475     nsp += (ngpr + n - maxgpr) * CTSIZE_PTR;  /* Assumes contiguous gpr/stack fields. */ \
 476     if (nsp > CCALL_SIZE_STACK) goto err_nyi;  /* Too many arguments. */ \
 477     ngpr = maxgpr; \
 478    } else { \
 479     ngpr += n; \
 480    } \
 481    goto done; \
 482  }
 483
 484#if !LJ_ABI_SOFTFP	/* MIPS32 hard-float */
 485#define CCALL_HANDLE_REGARG \
 486  if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
 487    /* Try to pass argument in FPRs. */ \
 488    dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
 489    nfpr++; ngpr += n; \
 490    goto done; \
 491  } else {  /* Try to pass argument in GPRs. */ \
 492    nfpr = CCALL_NARG_FPR; \
 493    CCALL_HANDLE_GPR \
 494  }
 495#else			/* MIPS32 soft-float */
 496#define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
 497#endif
 498
 499#if !LJ_ABI_SOFTFP
 500/* On MIPS64 soft-float, position of float return values is endian-dependant. */
 501#define CCALL_HANDLE_RET \
 502  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 503    sp = (uint8_t *)&cc->fpr[0].f;
 504#endif
 505
 506#elif LJ_TARGET_MIPS64
 507/* -- MIPS n64 calling conventions ---------------------------------------- */
 508
 509#define CCALL_HANDLE_STRUCTRET \
 510  cc->retref = !(sz <= 16); \
 511  if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 512
 513#define CCALL_HANDLE_STRUCTRET2 \
 514  ccall_copy_struct(cc, ctr, dp, sp, ccall_classify_struct(cts, ctr, ct));
 515
 516#define CCALL_HANDLE_COMPLEXRET \
 517  /* Complex values are returned in 1 or 2 FPRs. */ \
 518  cc->retref = 0;
 519
 520#if LJ_ABI_SOFTFP	/* MIPS64 soft-float */
 521
 522#define CCALL_HANDLE_COMPLEXRET2 \
 523  if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from GPRs. */ \
 524    ((intptr_t *)dp)[0] = cc->gpr[0]; \
 525  } else {  /* Copy complex double from GPRs. */ \
 526    ((intptr_t *)dp)[0] = cc->gpr[0]; \
 527    ((intptr_t *)dp)[1] = cc->gpr[1]; \
 528  }
 529
 530#define CCALL_HANDLE_COMPLEXARG \
 531  /* Pass complex by value in 2 or 4 GPRs. */
 532
 533/* Position of soft-float 'float' return value depends on endianess.  */
 534#define CCALL_HANDLE_RET \
 535  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 536    sp = (uint8_t *)cc->gpr + LJ_ENDIAN_SELECT(0, 4);
 537
 538#else			/* MIPS64 hard-float */
 539
 540#define CCALL_HANDLE_COMPLEXRET2 \
 541  if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 542    ((float *)dp)[0] = cc->fpr[0].f; \
 543    ((float *)dp)[1] = cc->fpr[1].f; \
 544  } else {  /* Copy complex double from FPRs. */ \
 545    ((double *)dp)[0] = cc->fpr[0].d; \
 546    ((double *)dp)[1] = cc->fpr[1].d; \
 547  }
 548
 549#define CCALL_HANDLE_COMPLEXARG \
 550  if (sz == 2*sizeof(float)) { \
 551    isfp = 2; \
 552    if (ngpr < maxgpr) \
 553      sz *= 2; \
 554  }
 555
 556#define CCALL_HANDLE_RET \
 557  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 558    sp = (uint8_t *)&cc->fpr[0].f;
 559
 560#endif
 561
 562#define CCALL_HANDLE_STRUCTARG \
 563  /* Pass all structs by value in registers and/or on the stack. */
 564
 565#define CCALL_HANDLE_REGARG \
 566  if (ngpr < maxgpr) { \
 567    dp = &cc->gpr[ngpr]; \
 568    if (ngpr + n > maxgpr) { \
 569      nsp += (ngpr + n - maxgpr) * CTSIZE_PTR;  /* Assumes contiguous gpr/stack fields. */ \
 570      if (nsp > CCALL_SIZE_STACK) goto err_nyi;  /* Too many arguments. */ \
 571      ngpr = maxgpr; \
 572    } else { \
 573      ngpr += n; \
 574    } \
 575    goto done; \
 576  }
 577
 578#else
 579#error "Missing calling convention definitions for this architecture"
 580#endif
 581
 582#ifndef CCALL_HANDLE_STRUCTRET2
 583#define CCALL_HANDLE_STRUCTRET2 \
 584  memcpy(dp, sp, ctr->size);  /* Copy struct return value from GPRs. */
 585#endif
 586
 587/* -- x86 OSX ABI struct classification ----------------------------------- */
 588
 589#if LJ_TARGET_X86 && LJ_TARGET_OSX
 590
 591/* Check for struct with single FP field. */
 592static int ccall_classify_struct(CTState *cts, CType *ct)
 593{
 594  CTSize sz = ct->size;
 595  if (!(sz == sizeof(float) || sz == sizeof(double))) return 0;
 596  if ((ct->info & CTF_UNION)) return 0;
 597  while (ct->sib) {
 598    ct = ctype_get(cts, ct->sib);
 599    if (ctype_isfield(ct->info)) {
 600      CType *sct = ctype_rawchild(cts, ct);
 601      if (ctype_isfp(sct->info)) {
 602	if (sct->size == sz)
 603	  return (sz >> 2);  /* Return 1 for float or 2 for double. */
 604      } else if (ctype_isstruct(sct->info)) {
 605	if (sct->size)
 606	  return ccall_classify_struct(cts, sct);
 607      } else {
 608	break;
 609      }
 610    } else if (ctype_isbitfield(ct->info)) {
 611      break;
 612    } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 613      CType *sct = ctype_rawchild(cts, ct);
 614      if (sct->size)
 615	return ccall_classify_struct(cts, sct);
 616    }
 617  }
 618  return 0;
 619}
 620
 621#endif
 622
 623/* -- x64 struct classification ------------------------------------------- */
 624
 625#if LJ_TARGET_X64 && !LJ_ABI_WIN
 626
 627/* Register classes for x64 struct classification. */
 628#define CCALL_RCL_INT	1
 629#define CCALL_RCL_SSE	2
 630#define CCALL_RCL_MEM	4
 631/* NYI: classify vectors. */
 632
 633static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
 634
 635/* Classify a C type. */
 636static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 637{
 638  if (ctype_isarray(ct->info)) {
 639    CType *cct = ctype_rawchild(cts, ct);
 640    CTSize eofs, esz = cct->size, asz = ct->size;
 641    for (eofs = 0; eofs < asz; eofs += esz)
 642      ccall_classify_ct(cts, cct, rcl, ofs+eofs);
 643  } else if (ctype_isstruct(ct->info)) {
 644    ccall_classify_struct(cts, ct, rcl, ofs);
 645  } else {
 646    int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
 647    lj_assertCTS(ctype_hassize(ct->info),
 648		 "classify ctype %08x without size", ct->info);
 649    if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM;  /* Unaligned. */
 650    rcl[(ofs >= 8)] |= cl;
 651  }
 652}
 653
 654/* Recursively classify a struct based on its fields. */
 655static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 656{
 657  if (ct->size > 16) return CCALL_RCL_MEM;  /* Too big, gets memory class. */
 658  while (ct->sib) {
 659    CTSize fofs;
 660    ct = ctype_get(cts, ct->sib);
 661    fofs = ofs+ct->size;
 662    if (ctype_isfield(ct->info))
 663      ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 664    else if (ctype_isbitfield(ct->info))
 665      rcl[(fofs >= 8)] |= CCALL_RCL_INT;  /* NYI: unaligned bitfields? */
 666    else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
 667      ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 668  }
 669  return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM);  /* Memory class? */
 670}
 671
 672/* Try to split up a small struct into registers. */
 673static int ccall_struct_reg(CCallState *cc, CTState *cts, GPRArg *dp, int *rcl)
 674{
 675  MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
 676  uint32_t i;
 677  UNUSED(cts);
 678  for (i = 0; i < 2; i++) {
 679    lj_assertCTS(!(rcl[i] & CCALL_RCL_MEM), "pass mem struct in reg");
 680    if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 681      if (ngpr >= CCALL_NARG_GPR) return 1;  /* Register overflow. */
 682      cc->gpr[ngpr++] = dp[i];
 683    } else if ((rcl[i] & CCALL_RCL_SSE)) {
 684      if (nfpr >= CCALL_NARG_FPR) return 1;  /* Register overflow. */
 685      cc->fpr[nfpr++].l[0] = dp[i];
 686    }
 687  }
 688  cc->ngpr = ngpr; cc->nfpr = nfpr;
 689  return 0;  /* Ok. */
 690}
 691
 692/* Pass a small struct argument. */
 693static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
 694			    TValue *o, int narg)
 695{
 696  GPRArg dp[2];
 697  dp[0] = dp[1] = 0;
 698  /* Convert to temp. struct. */
 699  lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 700  if (ccall_struct_reg(cc, cts, dp, rcl)) {
 701    /* Register overflow? Pass on stack. */
 702    MSize nsp = cc->nsp, sz = rcl[1] ? 2*CTSIZE_PTR : CTSIZE_PTR;
 703    if (nsp + sz > CCALL_SIZE_STACK)
 704      return 1;  /* Too many arguments. */
 705    cc->nsp = nsp + sz;
 706    memcpy((uint8_t *)cc->stack + nsp, dp, sz);
 707  }
 708  return 0;  /* Ok. */
 709}
 710
 711/* Combine returned small struct. */
 712static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
 713{
 714  GPRArg sp[2];
 715  MSize ngpr = 0, nfpr = 0;
 716  uint32_t i;
 717  for (i = 0; i < 2; i++) {
 718    if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 719      sp[i] = cc->gpr[ngpr++];
 720    } else if ((rcl[i] & CCALL_RCL_SSE)) {
 721      sp[i] = cc->fpr[nfpr++].l[0];
 722    }
 723  }
 724  memcpy(dp, sp, sz);
 725}
 726#endif
 727
 728/* -- ARM hard-float ABI struct classification ---------------------------- */
 729
 730#if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
 731
 732/* Classify a struct based on its fields. */
 733static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
 734{
 735  CTSize sz = ct->size;
 736  unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
 737  if ((ctf->info & CTF_VARARG)) goto noth;
 738  while (ct->sib) {
 739    CType *sct;
 740    ct = ctype_get(cts, ct->sib);
 741    if (ctype_isfield(ct->info)) {
 742      sct = ctype_rawchild(cts, ct);
 743      if (ctype_isfp(sct->info)) {
 744	r |= sct->size;
 745	if (!isu) n++; else if (n == 0) n = 1;
 746      } else if (ctype_iscomplex(sct->info)) {
 747	r |= (sct->size >> 1);
 748	if (!isu) n += 2; else if (n < 2) n = 2;
 749      } else if (ctype_isstruct(sct->info)) {
 750	goto substruct;
 751      } else {
 752	goto noth;
 753      }
 754    } else if (ctype_isbitfield(ct->info)) {
 755      goto noth;
 756    } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 757      sct = ctype_rawchild(cts, ct);
 758    substruct:
 759      if (sct->size > 0) {
 760	unsigned int s = ccall_classify_struct(cts, sct, ctf);
 761	if (s <= 1) goto noth;
 762	r |= (s & 255);
 763	if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
 764      }
 765    }
 766  }
 767  if ((r == 4 || r == 8) && n <= 4)
 768    return r + (n << 8);
 769noth:  /* Not a homogeneous float/double aggregate. */
 770  return (sz <= 4);  /* Return structs of size <= 4 in a GPR. */
 771}
 772
 773#endif
 774
 775/* -- ARM64 ABI struct classification ------------------------------------- */
 776
 777#if LJ_TARGET_ARM64
 778
 779/* Classify a struct based on its fields. */
 780static unsigned int ccall_classify_struct(CTState *cts, CType *ct)
 781{
 782  CTSize sz = ct->size;
 783  unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
 784  while (ct->sib && n <= 4) {
 785    unsigned int m = 1;
 786    CType *sct;
 787    ct = ctype_get(cts, ct->sib);
 788    if (ctype_isfield(ct->info)) {
 789      sct = ctype_rawchild(cts, ct);
 790      if (ctype_isarray(sct->info)) {
 791	CType *cct = ctype_rawchild(cts, sct);
 792	if (!cct->size) continue;
 793	m = sct->size / cct->size;
 794	sct = cct;
 795      }
 796      if (ctype_isfp(sct->info)) {
 797	r |= sct->size;
 798	if (!isu) n += m; else if (n < m) n = m;
 799      } else if (ctype_iscomplex(sct->info)) {
 800	r |= (sct->size >> 1);
 801	if (!isu) n += 2*m; else if (n < 2*m) n = 2*m;
 802      } else if (ctype_isstruct(sct->info)) {
 803	goto substruct;
 804      } else {
 805	goto noth;
 806      }
 807    } else if (ctype_isbitfield(ct->info)) {
 808      goto noth;
 809    } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 810      sct = ctype_rawchild(cts, ct);
 811    substruct:
 812      if (sct->size > 0) {
 813	unsigned int s = ccall_classify_struct(cts, sct), sn;
 814	if (s <= 1) goto noth;
 815	r |= (s & 255);
 816	sn = (s >> 8) * m;
 817	if (!isu) n += sn; else if (n < sn) n = sn;
 818      }
 819    }
 820  }
 821  if ((r == 4 || r == 8) && n <= 4)
 822    return r + (n << 8);
 823noth:  /* Not a homogeneous float/double aggregate. */
 824  return (sz <= 16);  /* Return structs of size <= 16 in GPRs. */
 825}
 826
 827#endif
 828
 829/* -- MIPS64 ABI struct classification ---------------------------- */
 830
 831#if LJ_TARGET_MIPS64
 832
 833#define FTYPE_FLOAT	1
 834#define FTYPE_DOUBLE	2
 835
 836/* Classify FP fields (max. 2) and their types. */
 837static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
 838{
 839  int n = 0, ft = 0;
 840  if ((ctf->info & CTF_VARARG) || (ct->info & CTF_UNION))
 841    goto noth;
 842  while (ct->sib) {
 843    CType *sct;
 844    ct = ctype_get(cts, ct->sib);
 845    if (n == 2) {
 846      goto noth;
 847    } else if (ctype_isfield(ct->info)) {
 848      sct = ctype_rawchild(cts, ct);
 849      if (ctype_isfp(sct->info)) {
 850	ft |= (sct->size == 4 ? FTYPE_FLOAT : FTYPE_DOUBLE) << 2*n;
 851	n++;
 852      } else {
 853	goto noth;
 854      }
 855    } else if (ctype_isbitfield(ct->info) ||
 856	       ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 857      goto noth;
 858    }
 859  }
 860  if (n <= 2)
 861    return ft;
 862noth:  /* Not a homogeneous float/double aggregate. */
 863  return 0;  /* Struct is in GPRs. */
 864}
 865
 866static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp,
 867			      int ft)
 868{
 869  if (LJ_ABI_SOFTFP ? ft :
 870      ((ft & 3) == FTYPE_FLOAT || (ft >> 2) == FTYPE_FLOAT)) {
 871    int i, ofs = 0;
 872    for (i = 0; ft != 0; i++, ft >>= 2) {
 873      if ((ft & 3) == FTYPE_FLOAT) {
 874#if LJ_ABI_SOFTFP
 875	/* The 2nd FP struct result is in CARG1 (gpr[2]) and not CRET2. */
 876	memcpy((uint8_t *)dp + ofs,
 877	       (uint8_t *)&cc->gpr[2*i] + LJ_ENDIAN_SELECT(0, 4), 4);
 878#else
 879	*(float *)((uint8_t *)dp + ofs) = cc->fpr[i].f;
 880#endif
 881	ofs += 4;
 882      } else {
 883	ofs = (ofs + 7) & ~7;  /* 64 bit alignment. */
 884#if LJ_ABI_SOFTFP
 885	*(intptr_t *)((uint8_t *)dp + ofs) = cc->gpr[2*i];
 886#else
 887	*(double *)((uint8_t *)dp + ofs) = cc->fpr[i].d;
 888#endif
 889	ofs += 8;
 890      }
 891    }
 892  } else {
 893#if !LJ_ABI_SOFTFP
 894    if (ft) sp = (uint8_t *)&cc->fpr[0];
 895#endif
 896    memcpy(dp, sp, ctr->size);
 897  }
 898}
 899
 900#endif
 901
 902/* -- Common C call handling ---------------------------------------------- */
 903
 904/* Infer the destination CTypeID for a vararg argument.
 905** Note: may reallocate cts->tab and invalidate CType pointers.
 906*/
 907CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
 908{
 909  if (tvisnumber(o)) {
 910    return CTID_DOUBLE;
 911  } else if (tviscdata(o)) {
 912    CTypeID id = cdataV(o)->ctypeid;
 913    CType *s = ctype_get(cts, id);
 914    if (ctype_isrefarray(s->info)) {
 915      return lj_ctype_intern(cts,
 916	       CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
 917    } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
 918      /* NYI: how to pass a struct by value in a vararg argument? */
 919      return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
 920    } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
 921      return CTID_DOUBLE;
 922    } else {
 923      return id;
 924    }
 925  } else if (tvisstr(o)) {
 926    return CTID_P_CCHAR;
 927  } else if (tvisbool(o)) {
 928    return CTID_BOOL;
 929  } else {
 930    return CTID_P_VOID;
 931  }
 932}
 933
 934/* Setup arguments for C call.
 935** Note: may reallocate cts->tab and invalidate CType pointers.
 936*/
 937static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
 938			  CCallState *cc)
 939{
 940  int gcsteps = 0;
 941  TValue *o, *top = L->top;
 942  CTypeID fid;
 943  CTInfo info = ct->info;  /* lj_ccall_ctid_vararg may invalidate ct pointer. */
 944  CType *ctr;
 945  MSize maxgpr, ngpr = 0, nsp = 0, narg;
 946#if CCALL_NARG_FPR
 947  MSize nfpr = 0;
 948#if LJ_TARGET_ARM
 949  MSize fprodd = 0;
 950#endif
 951#endif
 952
 953  /* Clear unused regs to get some determinism in case of misdeclaration. */
 954  memset(cc->gpr, 0, sizeof(cc->gpr));
 955#if CCALL_NUM_FPR
 956  memset(cc->fpr, 0, sizeof(cc->fpr));
 957#endif
 958
 959#if LJ_TARGET_X86
 960  /* x86 has several different calling conventions. */
 961  cc->resx87 = 0;
 962  switch (ctype_cconv(info)) {
 963  case CTCC_FASTCALL: maxgpr = 2; break;
 964  case CTCC_THISCALL: maxgpr = 1; break;
 965  default: maxgpr = 0; break;
 966  }
 967#else
 968  maxgpr = CCALL_NARG_GPR;
 969#endif
 970
 971  /* Perform required setup for some result types. */
 972  ctr = ctype_rawchild(cts, ct);
 973  if (ctype_isvector(ctr->info)) {
 974    if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
 975      goto err_nyi;
 976  } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
 977    /* Preallocate cdata object and anchor it after arguments. */
 978    CTSize sz = ctr->size;
 979    GCcdata *cd = lj_cdata_new(cts, ctype_cid(info), sz);
 980    void *dp = cdataptr(cd);
 981    setcdataV(L, L->top++, cd);
 982    if (ctype_isstruct(ctr->info)) {
 983      CCALL_HANDLE_STRUCTRET
 984    } else {
 985      CCALL_HANDLE_COMPLEXRET
 986    }
 987#if LJ_TARGET_X86
 988  } else if (ctype_isfp(ctr->info)) {
 989    cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
 990#endif
 991  }
 992
 993  /* Skip initial attributes. */
 994  fid = ct->sib;
 995  while (fid) {
 996    CType *ctf = ctype_get(cts, fid);
 997    if (!ctype_isattrib(ctf->info)) break;
 998    fid = ctf->sib;
 999  }
1000
1001#if LJ_TARGET_ARM64 && LJ_ABI_WIN
1002  if ((info & CTF_VARARG)) {
1003    nsp -= maxgpr * CTSIZE_PTR;  /* May end up with negative nsp. */
1004    ngpr = maxgpr;
1005    nfpr = CCALL_NARG_FPR;
1006  }
1007#endif
1008
1009  /* Walk through all passed arguments. */
1010  for (o = L->base+1, narg = 1; o < top; o++, narg++) {
1011    CTypeID did;
1012    CType *d;
1013    CTSize sz;
1014    MSize n, isfp = 0, isva = 0;
1015    void *dp, *rp = NULL;
1016
1017    if (fid) {  /* Get argument type from field. */
1018      CType *ctf = ctype_get(cts, fid);
1019      fid = ctf->sib;
1020      lj_assertL(ctype_isfield(ctf->info), "field expected");
1021      did = ctype_cid(ctf->info);
1022    } else {
1023      if (!(info & CTF_VARARG))
1024	lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too many arguments. */
1025      did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
1026      isva = 1;
1027    }
1028    d = ctype_raw(cts, did);
1029    sz = d->size;
1030
1031    /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
1032    if (ctype_isnum(d->info)) {
1033      if (sz > 8) goto err_nyi;
1034      if ((d->info & CTF_FP))
1035	isfp = 1;
1036    } else if (ctype_isvector(d->info)) {
1037      if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
1038	isfp = 1;
1039      else
1040	goto err_nyi;
1041    } else if (ctype_isstruct(d->info)) {
1042      CCALL_HANDLE_STRUCTARG
1043    } else if (ctype_iscomplex(d->info)) {
1044      CCALL_HANDLE_COMPLEXARG
1045    } else if (!(CCALL_PACK_STACKARG && ctype_isenum(d->info))) {
1046      sz = CTSIZE_PTR;
1047    }
1048    n = (sz + CTSIZE_PTR-1) / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
1049
1050    CCALL_HANDLE_REGARG  /* Handle register arguments. */
1051
1052    /* Otherwise pass argument on stack. */
1053    if (CCALL_ALIGN_STACKARG) {  /* Align argument on stack. */
1054      MSize align = (1u << ctype_align(d->info)) - 1;
1055      if (rp || (CCALL_PACK_STACKARG && isva && align < CTSIZE_PTR-1))
1056	align = CTSIZE_PTR-1;
1057      nsp = (nsp + align) & ~align;
1058    }
1059#if LJ_TARGET_ARM64 && LJ_ABI_WIN
1060    /* A negative nsp points into cc->gpr. Blame MS for their messy ABI. */
1061    dp = ((uint8_t *)cc->stack) + (int32_t)nsp;
1062#else
1063    dp = ((uint8_t *)cc->stack) + nsp;
1064#endif
1065    nsp += CCALL_PACK_STACKARG ? sz : n * CTSIZE_PTR;
1066    if ((int32_t)nsp > CCALL_SIZE_STACK) {  /* Too many arguments. */
1067    err_nyi:
1068      lj_err_caller(L, LJ_ERR_FFI_NYICALL);
1069    }
1070    isva = 0;
1071
1072  done:
1073    if (rp) {  /* Pass by reference. */
1074      gcsteps++;
1075      *(void **)dp = rp;
1076      dp = rp;
1077    }
1078    lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
1079    /* Extend passed integers to 32 bits at least. */
1080    if (ctype_isinteger_or_bool(d->info) && d->size < 4 &&
1081	(!CCALL_PACK_STACKARG || !((uintptr_t)dp & 3))) {  /* Assumes LJ_LE. */
1082      if (d->info & CTF_UNSIGNED)
1083	*(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
1084					 (uint32_t)*(uint16_t *)dp;
1085      else
1086	*(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
1087					(int32_t)*(int16_t *)dp;
1088    }
1089#if LJ_TARGET_ARM64 && LJ_BE
1090    if (isfp && d->size == sizeof(float))
1091      ((float *)dp)[1] = ((float *)dp)[0];  /* Floats occupy high slot. */
1092#endif
1093#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
1094    if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info)
1095#if LJ_TARGET_MIPS64
1096	 || (isfp && nsp == 0)
1097#endif
1098	 ) && d->size <= 4) {
1099      *(int64_t *)dp = (int64_t)*(int32_t *)dp;  /* Sign-extend to 64 bit. */
1100    }
1101#endif
1102#if LJ_TARGET_X64 && LJ_ABI_WIN
1103    if (isva) {  /* Windows/x64 mirrors varargs in both register sets. */
1104      if (nfpr == ngpr)
1105	cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
1106      else
1107	cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
1108    }
1109#else
1110    UNUSED(isva);
1111#endif
1112#if LJ_TARGET_X64 && !LJ_ABI_WIN
1113    if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
1114      cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1];  /* Split complex double. */
1115      cc->fpr[nfpr-2].d[1] = 0;
1116    }
1117#elif LJ_TARGET_ARM64 || (LJ_TARGET_MIPS64 && !LJ_ABI_SOFTFP)
1118    if (isfp == 2 && (uint8_t *)dp < (uint8_t *)cc->stack) {
1119      /* Split float HFA or complex float into separate registers. */
1120      CTSize i = (sz >> 2) - 1;
1121      do { ((uint64_t *)dp)[i] = ((uint32_t *)dp)[i]; } while (i--);
1122    }
1123#else
1124    UNUSED(isfp);
1125#endif
1126  }
1127  if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too few arguments. */
1128#if LJ_TARGET_ARM64 && LJ_ABI_WIN
1129  if ((int32_t)nsp < 0) nsp = 0;
1130#endif
1131
1132#if LJ_TARGET_X64 || (LJ_TARGET_PPC && !LJ_ABI_SOFTFP)
1133  cc->nfpr = nfpr;  /* Required for vararg functions. */
1134#endif
1135  cc->nsp = (nsp + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
1136  cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA) * CTSIZE_PTR;
1137  if (cc->nsp > CCALL_SPS_FREE * CTSIZE_PTR)
1138    cc->spadj += (((cc->nsp - CCALL_SPS_FREE * CTSIZE_PTR) + 15u) & ~15u);
1139  return gcsteps;
1140}
1141
1142/* Get results from C call. */
1143static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
1144			     CCallState *cc, int *ret)
1145{
1146  CType *ctr = ctype_rawchild(cts, ct);
1147  uint8_t *sp = (uint8_t *)&cc->gpr[0];
1148  if (ctype_isvoid(ctr->info)) {
1149    *ret = 0;  /* Zero results. */
1150    return 0;  /* No additional GC step. */
1151  }
1152  *ret = 1;  /* One result. */
1153  if (ctype_isstruct(ctr->info)) {
1154    /* Return cdata object which is already on top of stack. */
1155    if (!cc->retref) {
1156      void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
1157      CCALL_HANDLE_STRUCTRET2
1158    }
1159    return 1;  /* One GC step. */
1160  }
1161  if (ctype_iscomplex(ctr->info)) {
1162    /* Return cdata object which is already on top of stack. */
1163    void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
1164    CCALL_HANDLE_COMPLEXRET2
1165    return 1;  /* One GC step. */
1166  }
1167  if (LJ_BE && ctr->size < CTSIZE_PTR &&
1168      (ctype_isinteger_or_bool(ctr->info) || ctype_isenum(ctr->info)))
1169    sp += (CTSIZE_PTR - ctr->size);
1170#if CCALL_NUM_FPR
1171  if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
1172    sp = (uint8_t *)&cc->fpr[0];
1173#endif
1174#ifdef CCALL_HANDLE_RET
1175  CCALL_HANDLE_RET
1176#endif
1177  /* No reference types end up here, so there's no need for the CTypeID. */
1178  lj_assertL(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)),
1179	     "unexpected reference ctype");
1180  return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
1181}
1182
1183/* Call C function. */
1184int lj_ccall_func(lua_State *L, GCcdata *cd)
1185{
1186  CTState *cts = ctype_cts(L);
1187  CType *ct = ctype_raw(cts, cd->ctypeid);
1188  CTSize sz = CTSIZE_PTR;
1189  if (ctype_isptr(ct->info)) {
1190    sz = ct->size;
1191    ct = ctype_rawchild(cts, ct);
1192  }
1193  if (ctype_isfunc(ct->info)) {
1194    CTypeID id = ctype_typeid(cts, ct);
1195    CCallState cc;
1196    int gcsteps, ret;
1197    cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
1198    gcsteps = ccall_set_args(L, cts, ct, &cc);
1199    cts->cb.slot = ~0u;
1200    lj_vm_ffi_call(&cc);
1201    if (cts->cb.slot != ~0u) {  /* Blacklist function that called a callback. */
1202      TValue tv;
1203      tv.u64 = ((uintptr_t)(void *)cc.func >> 2) | U64x(800000000, 00000000);
1204      setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
1205    }
1206    ct = ctype_get(cts, id);  /* Table may have been reallocated. */
1207    gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
1208#if LJ_TARGET_X86 && LJ_ABI_WIN
1209    /* Automatically detect __stdcall and fix up C function declaration. */
1210    if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
1211      CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
1212      lj_trace_abort(G(L));
1213    }
1214#endif
1215    while (gcsteps-- > 0)
1216      lj_gc_check(L);
1217    return ret;
1218  }
1219  return -1;  /* Not a function. */
1220}
1221
1222#endif