/* vim: ts=4 sw=4 sts=4 et tw=78
* Copyright (c) 2011 James R. McKaskill. See license in ffi.h
*/
|.if X64
|.arch x64
|.else
|.arch x86
|.endif
|.actionlist build_actionlist
|.globalnames globnames
|.externnames extnames
|.if not X64
|.define RET_H, edx // for int64_t returns
|.define RET_L, eax
|.endif
|.if X64WIN
|
|.macro call_rrrp, func, arg0, arg1, arg2, arg3
| mov64 r9, arg3
| mov r8, arg2
| mov rdx, arg1
| mov rcx, arg0
| call func
|.endmacro
|.macro call_rrrr, func, arg0, arg1, arg2, arg3
| mov r9, arg3
| mov r8, arg2
| mov rdx, arg1
| mov rcx, arg0
| call func
|.endmacro
|
|.macro call_rrp, func, arg0, arg1, arg2
| mov64 r8, arg2
| mov rdx, arg1
| mov rcx, arg0
| call func
|.endmacro
|.macro call_rrr, func, arg0, arg1, arg2
| mov r8, arg2
| mov rdx, arg1
| mov rcx, arg0
| call func
|.endmacro
|
|.macro call_rp, func, arg0, arg1
| mov64 rdx, arg1
| mov rcx, arg0
| call func
|.endmacro
|.macro call_rr, func, arg0, arg1
| mov rdx, arg1
| mov rcx, arg0
| call func
|.endmacro
|
|.macro call_r, func, arg0
| mov rcx, arg0
| call func
|.endmacro
|
|.elif X64
|
| // the 5 and 6 arg forms are only used on posix x64
|.macro call_rrrrrr, func, arg0, arg1, arg2, arg3, arg4, arg5
| mov r9, arg5
| mov r8, arg4
| mov rcx, arg3
| mov rdx, arg2
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|.macro call_rrrrr, func, arg0, arg1, arg2, arg3, arg4
| mov r8, arg4
| mov rcx, arg3
| mov rdx, arg2
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|
|.macro call_rrrp, func, arg0, arg1, arg2, arg3
| mov64 rcx, arg3
| mov rdx, arg2
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|.macro call_rrrr, func, arg0, arg1, arg2, arg3
| mov rcx, arg3
| mov rdx, arg2
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|
|.macro call_rrp, func, arg0, arg1, arg2
| mov64 rdx, arg2
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|.macro call_rrr, func, arg0, arg1, arg2
| mov rdx, arg2
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|
|.macro call_rp, func, arg0, arg1
| mov64 rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|.macro call_rr, func, arg0, arg1
| mov rsi, arg1
| mov rdi, arg0
| call func
|.endmacro
|
|.macro call_r, func, arg0
| mov rdi, arg0
| call func
|.endmacro
|
|.else
| // define the 64bit registers to the 32 bit counterparts, so the common
| // code can use r*x for all pointers
|.define rax, eax
|.define rcx, ecx
|.define rdx, edx
|.define rsp, esp
|.define rbp, ebp
|.define rdi, edi
|.define rsi, esi
|.define mov64, mov
|
|.macro call_rrrr, func, arg0, arg1, arg2, arg3
| mov dword [rsp+12], arg3
| mov dword [rsp+8], arg2
| mov dword [rsp+4], arg1
| mov dword [rsp], arg0
| call func
|.endmacro
|.macro call_rrr, func, arg0, arg1, arg2
| mov dword [rsp+8], arg2
| mov dword [rsp+4], arg1
| mov dword [rsp], arg0
| call func
|.endmacro
|.macro call_rr, func, arg0, arg1
| mov dword [rsp+4], arg1
| mov dword [rsp], arg0
| call func
|.endmacro
|.macro call_r, func, arg0
| mov dword [rsp], arg0
| call func
|.endmacro
|
|.define call_rrrp, call_rrrr
|.define call_rrp, call_rrr
|.define call_rp, call_rr
|
|.endif
#if defined _WIN64 || defined __amd64__
#define JUMP_SIZE 14
#else
#define JUMP_SIZE 4
#endif
#define MIN_BRANCH INT32_MIN
#define MAX_BRANCH INT32_MAX
#define BRANCH_OFF 4
static void compile_extern_jump(struct jit* jit, lua_State* L, cfunction func, uint8_t* code)
{
/* The jump code is the function pointer followed by a stub to call the
* function pointer. The stub exists in 64 bit so we can jump to functions
* with an offset greater than 2 GB.
*
* Note we have to manually set this up since there are commands buffered
* in the jit state and dynasm doesn't support rip relative addressing.
*
* eg on 64 bit:
* 0-8: function ptr
* 8-14: jmp aword [rip-14]
*
* for 32 bit we only set the function ptr as it can always fit in a 32
* bit displacement
*/
#if defined _WIN64 || defined __amd64__
*(cfunction*) code = func;
code[8] = 0xFF; /* FF /4 operand for jmp */
code[9] = 0x25; /* RIP displacement */
*(int32_t*) &code[10] = -14;
#else
*(cfunction*) code = func;
#endif
}
void compile_globals(struct jit* jit, lua_State* L)
{
struct jit* Dst = jit;
int* perr = &jit->last_errno;
dasm_setup(Dst, build_actionlist);
/* Note: since the return code uses EBP to reset the stack pointer, we
* don't have to track the amount of stack space used. It also means we
* can handle stdcall and cdecl with the same code.
*/
/* Note the various call_* functions want 32 bytes of 16 byte aligned
* stack
*/
|.if X64
|.define L_ARG, r12
|.define TOP, r13
|.else
|.define L_ARG, rdi
|.define TOP, rsi
|.endif
|.macro epilog
|.if X64
| mov TOP, [rbp-16]
| mov L_ARG, [rbp-8]
|.else
| mov TOP, [rbp-8]
| mov L_ARG, [rbp-4]
|.endif
| mov rsp, rbp
| pop rbp
| ret
|.endmacro
|.macro get_errno // note trashes registers
| call extern GetLastError
| mov64 rcx, perr
| mov dword [rcx], eax
|.endmacro
/* the general idea for the return functions is:
* 1) Save return value on stack
* 2) Call get_errno (this trashes the registers hence #1)
* 3) Unpack return value from stack
* 4) Call lua push function
* 5) Set eax to number of returned args (0 or 1)
* 6) Call return which pops our stack frame
*/
|->lua_return_arg:
| mov eax, 1
| epilog
|->lua_return_void:
| get_errno
| mov eax, 0
| epilog
|->lua_return_double:
|.if X64
| movq qword [rsp+32], xmm0
|.else
| fstp qword [rsp+4] // note get_errno doesn't require any stack on x86
|.endif
|
| get_errno
|
|.if X64WIN
| movq xmm1, qword [rsp+32]
| mov rcx, L_ARG
|.elif X64
| movq xmm0, qword [rsp+32]
| mov rdi, L_ARG
|.else
| mov [rsp], L_ARG
|.endif
| call extern lua_pushnumber
| jmp ->lua_return_arg
|->lua_return_bool:
| movzx eax, al
| mov [rsp+32], eax
| get_errno
| mov eax, [rsp+32]
| call_rr extern lua_pushboolean, L_ARG, rax
| jmp ->lua_return_arg
|->lua_return_int:
| mov [rsp+32], eax
| get_errno
| mov eax, [rsp+32]
| call_rr extern push_int, L_ARG, rax
| jmp ->lua_return_arg
|->lua_return_uint:
| mov [rsp+32], eax
| get_errno
| mov eax, [rsp+32]
| call_rr extern push_uint, L_ARG, rax
| jmp ->lua_return_arg
|->too_few_arguments:
| mov ax, 0
| call_rp extern luaL_error, L_ARG, &"too few arguments"
|->too_many_arguments:
| mov ax, 0
| call_rp extern luaL_error, L_ARG, &"too many arguments"
|->save_registers:
| // use rbp relative so we store values in the outer stack frame
|.if X64WIN
| // use the provided shadow space for int registers above prev rbp and
| // return address
| mov [rbp+16], rcx
| mov [rbp+24], rdx
| mov [rbp+32], r8
| mov [rbp+40], r9
| // use the extra space we added for float registers
| // -16 to store underneath previous value of L_ARG
| movq qword [rbp-16], xmm0
| movq qword [rbp-24], xmm1
| movq qword [rbp-32], xmm2
| movq qword [rbp-40], xmm3
|.elif X64
| movq qword [rbp-16], xmm0
| movq qword [rbp-24], xmm1
| movq qword [rbp-32], xmm2
| movq qword [rbp-40], xmm3
| movq qword [rbp-48], xmm4
| movq qword [rbp-56], xmm5
| movq qword [rbp-64], xmm6
| movq qword [rbp-72], xmm7
| mov [rbp-80], rdi
| mov [rbp-88], rsi
| mov [rbp-96], rdx
| mov [rbp-104], rcx
| mov [rbp-112], r8
| mov [rbp-120], r9
|.else
| // fastcall, -8 to store underneath previous value of L_ARG
| mov [rbp-8], ecx
| mov [rbp-12], edx
|.endif
| ret
compile(Dst, L, NULL, LUA_NOREF);
}
int x86_return_size(lua_State* L, int usr, const struct ctype* ct)
{
int ret = 0;
const struct ctype* mt;
if (ct->calling_convention != C_CALL) {
size_t i;
size_t argn = lua_rawlen(L, usr);
for (i = 1; i <= argn; i++) {
lua_rawgeti(L, usr, (int) i);
mt = (const struct ctype*) lua_touserdata(L, -1);
if (mt->pointers) {
ret += sizeof(void*);
} else {
switch (mt->type) {
case DOUBLE_TYPE:
case COMPLEX_FLOAT_TYPE:
case INT64_TYPE:
ret += 8;
break;
case COMPLEX_DOUBLE_TYPE:
ret += 16;
break;
case INTPTR_TYPE:
ret += sizeof(intptr_t);
break;
case FUNCTION_PTR_TYPE:
ret += sizeof(cfunction);
break;
case BOOL_TYPE:
case FLOAT_TYPE:
case INT8_TYPE:
case INT16_TYPE:
case INT32_TYPE:
case ENUM_TYPE:
ret += 4;
break;
default:
return luaL_error(L, "NYI - argument type");
}
}
lua_pop(L, 1);
}
}
#if !defined _WIN64 && !defined __amd64__
lua_rawgeti(L, usr, 0);
mt = (const struct ctype*) lua_touserdata(L, -1);
if (!mt->pointers && mt->type == COMPLEX_DOUBLE_TYPE) {
ret += sizeof(void*);
}
lua_pop(L, 1);
#endif
return ret;
}
#ifdef _WIN64
#define MAX_REGISTERS(ct) 4 /* rcx, rdx, r8, r9 */
#elif defined __amd64__
#define MAX_INT_REGISTERS(ct) 6 /* rdi, rsi, rdx, rcx, r8, r9 */
#define MAX_FLOAT_REGISTERS(ct) 8 /* xmm0-7 */
#else
#define MAX_INT_REGISTERS(ct) ((ct)->calling_convention == FAST_CALL ? 2 /* ecx, edx */ : 0)
#define MAX_FLOAT_REGISTERS(ct) 0
#endif
struct reg_alloc {
#ifdef _WIN64
int regs;
int is_float[4];
int is_int[4];
#else
int floats;
int ints;
#endif
int off;
};
#ifdef _WIN64
#define REGISTER_STACK_SPACE(ct) (4*8)
#elif defined __amd64__
#define REGISTER_STACK_SPACE(ct) (14*8)
#else
#define REGISTER_STACK_SPACE(ct) ALIGN_UP(((ct)->calling_convention == FAST_CALL ? 2*4 : 0), 15)
#endif
/* Fastcall:
* Uses ecx, edx as first two int registers
* Everything else on stack (include 64bit ints)
* No overflow stack space
* Pops the stack before returning
* Returns int in eax, float in ST0
* We use the same register allocation logic as posix x64 with 2 int regs and 0 float regs
*/
static void get_int(Dst_DECL, const struct ctype* ct, struct reg_alloc* reg, int is_int64)
{
/* grab the register from the shadow space */
#ifdef _WIN64
if (reg->regs < MAX_REGISTERS(ct)) {
| mov rcx, [rbp + 16 + 8*reg->regs]
reg->regs++;
}
#elif __amd64__
if (reg->ints < MAX_INT_REGISTERS(ct)) {
| mov rcx, [rbp - 80 - 8*reg->ints]
reg->ints++;
}
#else
if (!is_int64 && reg->ints < MAX_INT_REGISTERS(ct)) {
| mov ecx, [rbp - 8 - 4*reg->ints]
reg->ints++;
}
#endif
else if (is_int64) {
|.if X64
| mov rcx, [rbp + reg->off]
|.else
| mov rcx, [rbp + reg->off]
| mov rdx, [rbp + reg->off + 4]
|.endif
reg->off += 8;
} else {
| mov ecx, [rbp + reg->off]
reg->off += 4;
}
}
static void add_int(Dst_DECL, const struct ctype* ct, struct reg_alloc* reg, int is_int64)
{
#ifdef _WIN64
if (reg->regs < MAX_REGISTERS(ct)) {
| mov [rsp + 32 + 8*(reg->regs)], rax
reg->is_int[reg->regs++] = 1;
}
#elif __amd64__
if (reg->ints < MAX_INT_REGISTERS(ct)) {
| mov [rsp + 32 + 8*reg->ints], rax
reg->ints++;
}
#else
if (!is_int64 && reg->ints < MAX_INT_REGISTERS(ct)) {
| mov [rsp + 32 + 4*reg->ints], rax
reg->ints++;
}
#endif
else if (is_int64) {
|.if X64
| mov [rsp + reg->off], rax
|.else
| mov [rsp + reg->off], RET_L
| mov [rsp + reg->off + 4], RET_H
|.endif
reg->off += 8;
} else {
| mov [rsp+reg->off], eax
reg->off += 4;
}
}
static void get_float(Dst_DECL, const struct ctype* ct, struct reg_alloc* reg, int is_double)
{
#if !defined _WIN64 && !defined __amd64__
assert(MAX_FLOAT_REGISTERS(ct) == 0);
if (is_double) {
| fld qword [rbp + reg->off]
reg->off += 8;
} else {
| fld dword [rbp + reg->off]
reg->off += 4;
}
#else
int off;
#ifdef _WIN64
if (reg->regs < MAX_REGISTERS(ct)) {
off = -16 - 8*reg->regs;
reg->regs++;
}
#else
if (reg->floats < MAX_FLOAT_REGISTERS(ct)) {
off = -16 - 8*reg->floats;
reg->floats++;
}
#endif
else {
off = reg->off;
reg->off += is_double ? 8 : 4;
}
if (is_double) {
| movq xmm0, qword [rbp + off]
} else {
| cvtss2sd xmm0, dword [rbp + off]
}
#endif
}
static void add_float(Dst_DECL, const struct ctype* ct, struct reg_alloc* reg, int is_double)
{
#if !defined _WIN64 && !defined __amd64__
assert(MAX_FLOAT_REGISTERS(ct) == 0);
if (is_double) {
| fstp qword [rsp + reg->off]
reg->off += 8;
} else {
| fstp dword [rsp + reg->off]
reg->off += 4;
}
#else
#ifdef _WIN64
if (reg->regs < MAX_REGISTERS(ct)) {
if (is_double) {
| movq qword [rsp + 32 + 8*(reg->regs)], xmm0
} else {
| cvtsd2ss xmm0, xmm0
| movq qword [rsp + 32 + 8*(reg->regs)], xmm0
}
reg->is_float[reg->regs++] = 1;
}
#else
if (reg->floats < MAX_FLOAT_REGISTERS(ct)) {
if (is_double) {
| movq qword [rsp + 32 + 8*(MAX_INT_REGISTERS(ct) + reg->floats)], xmm0
} else {
| cvtsd2ss xmm0, xmm0
| movq qword [rsp + 32 + 8*(MAX_INT_REGISTERS(ct) + reg->floats)], xmm0
}
reg->floats++;
}
#endif
else if (is_double) {
| movq qword [rsp + reg->off], xmm0
reg->off += 8;
} else {
| cvtsd2ss xmm0, xmm0
| movd dword [rsp + reg->off], xmm0
reg->off += 4;
}
#endif
}
#if defined _WIN64 || defined __amd64__
#define add_pointer(jit, ct, reg) add_int(jit, ct, reg, 1)
#define get_pointer(jit, ct, reg) get_int(jit, ct, reg, 1)
#else
#define add_pointer(jit, ct, reg) add_int(jit, ct, reg, 0)
#define get_pointer(jit, ct, reg) get_int(jit, ct, reg, 0)
#endif
cfunction compile_callback(lua_State* L, int fidx, int ct_usr, const struct ctype* ct)
{
int i, nargs;
cfunction* pf;
struct ctype ct2 = *ct;
const struct ctype* mt;
struct reg_alloc reg;
int num_upvals = 0;
int top = lua_gettop(L);
struct jit* Dst = get_jit(L);
int ref;
int hidden_arg_off = 0;
ct_usr = lua_absindex(L, ct_usr);
fidx = lua_absindex(L, fidx);
assert(lua_isnil(L, fidx) || lua_isfunction(L, fidx));
memset(®, 0, sizeof(reg));
#ifdef _WIN64
reg.off = 16 + REGISTER_STACK_SPACE(ct); /* stack registers are above the shadow space */
#elif __amd64__
reg.off = 16;
#else
reg.off = 8;
#endif
dasm_setup(Dst, build_actionlist);
// add a table to store ctype and function upvalues
// callback_set assumes the first value is the lua function
nargs = (int) lua_rawlen(L, ct_usr);
lua_newtable(L);
lua_pushvalue(L, -1);
ref = luaL_ref(L, LUA_REGISTRYINDEX);
if (ct->has_var_arg) {
luaL_error(L, "can't create callbacks with varargs");
}
// setup a stack frame to hold args for the call into lua_call
| push rbp
| mov rbp, rsp
| push L_ARG
| // stack is 4 or 8 (mod 16) (L_ARG, rbp, rip)
|.if X64
| // 8 to realign, 16 for return vars, 32 for local calls, rest to save registers
| sub rsp, 8 + 16 + 32 + REGISTER_STACK_SPACE(ct)
| call ->save_registers
|.else
| // 4 to realign, 16 for return vars, 32 for local calls, rest to save registers
| sub rsp, 4 + 16 + 32 + REGISTER_STACK_SPACE(ct)
if (ct->calling_convention == FAST_CALL) {
| call ->save_registers
}
|.endif
// hardcode the lua_State* value into the assembly
| mov64 L_ARG, L
/* get the upval table */
| call_rrr extern lua_rawgeti, L_ARG, LUA_REGISTRYINDEX, ref
/* get the lua function */
lua_pushvalue(L, fidx);
lua_rawseti(L, -2, ++num_upvals);
assert(num_upvals == CALLBACK_FUNC_USR_IDX);
| call_rrr extern lua_rawgeti, L_ARG, -1, num_upvals
#if !defined _WIN64 && !defined __amd64__
lua_rawgeti(L, ct_usr, 0);
mt = (const struct ctype*) lua_touserdata(L, -1);
if (!mt->pointers && mt->type == COMPLEX_DOUBLE_TYPE) {
hidden_arg_off = reg.off;
reg.off += sizeof(void*);
}
lua_pop(L, 1);
#else
(void) hidden_arg_off;
#endif
for (i = 1; i <= nargs; i++) {
lua_rawgeti(L, ct_usr, i);
mt = (const struct ctype*) lua_touserdata(L, -1);
if (mt->pointers) {
lua_getuservalue(L, -1);
lua_rawseti(L, -3, ++num_upvals); /* usr value */
lua_rawseti(L, -2, ++num_upvals); /* mt */
/* on the lua stack in the callback:
* upval tbl, lua func, i-1 args
*/
| call_rrr extern lua_rawgeti, L_ARG, -i-1, num_upvals-1
| call_rrp extern push_cdata, L_ARG, -1, mt
get_pointer(Dst, ct, ®);
| mov [rax], rcx
| call_rr, extern lua_remove, L_ARG, -2
} else {
switch (mt->type) {
case INT64_TYPE:
lua_getuservalue(L, -1);
lua_rawseti(L, -3, ++num_upvals); /* mt */
lua_pop(L, 1);
| call_rrp extern push_cdata, L_ARG, 0, mt
get_int(Dst, ct, ®, 1);
|.if X64
| mov [rax], rcx
|.else
| mov [rax], ecx
| mov [rax+4], edx
|.endif
break;
case INTPTR_TYPE:
lua_getuservalue(L, -1);
lua_rawseti(L, -3, ++num_upvals); /* mt */
lua_pop(L, 1);
| call_rrp extern push_cdata, L_ARG, 0, mt
get_pointer(Dst, ct, ®);
| mov [rax], rcx
break;
case COMPLEX_FLOAT_TYPE:
lua_pop(L, 1);
#if defined _WIN64 || defined __amd64__
/* complex floats are two floats packed into a double */
| call_rrp extern push_cdata, L_ARG, 0, mt
get_float(Dst, ct, ®, 1);
| movq qword [rax], xmm0
#else
/* complex floats are real followed by imag on the stack */
| call_rrp extern push_cdata, L_ARG, 0, mt
get_float(Dst, ct, ®, 0);
| fstp dword [rax]
get_float(Dst, ct, ®, 0);
| fstp dword [rax+4]
#endif
break;
case COMPLEX_DOUBLE_TYPE:
lua_pop(L, 1);
| call_rrp extern push_cdata, L_ARG, 0, mt
/* real */
get_float(Dst, ct, ®, 1);
|.if X64
| movq qword [rax], xmm0
|.else
| fstp qword [rax]
|.endif
/* imag */
get_float(Dst, ct, ®, 1);
|.if X64
| movq qword [rax+8], xmm0
|.else
| fstp qword [rax+8]
|.endif
break;
case FLOAT_TYPE:
case DOUBLE_TYPE:
lua_pop(L, 1);
get_float(Dst, ct, ®, mt->type == DOUBLE_TYPE);
|.if X64WIN
| movq xmm1, xmm0
| mov rcx, L_ARG
|.elif X64
| // for 64bit xmm0 is already set
| mov rdi, L_ARG
|.else
| fstp qword [rsp+4]
| mov [rsp], L_ARG
|.endif
| call extern lua_pushnumber
break;
case BOOL_TYPE:
lua_pop(L, 1);
get_int(Dst, ct, ®, 0);
| movzx ecx, cl
| call_rr extern lua_pushboolean, L_ARG, rcx
break;
case INT8_TYPE:
lua_pop(L, 1);
get_int(Dst, ct, ®, 0);
if (mt->is_unsigned) {
| movzx ecx, cl
} else {
| movsx ecx, cl
}
| call_rr extern push_int, L_ARG, rcx
break;
case INT16_TYPE:
lua_pop(L, 1);
get_int(Dst, ct, ®, 0);
if (mt->is_unsigned) {
| movzx ecx, cx
} else {
| movsx ecx, cx
}
| call_rr extern push_int, L_ARG, rcx
break;
case ENUM_TYPE:
case INT32_TYPE:
lua_pop(L, 1);
get_int(Dst, ct, ®, 0);
if (mt->is_unsigned) {
| call_rr extern push_uint, L_ARG, rcx
} else {
| call_rr extern push_int, L_ARG, rcx
}
break;
default:
luaL_error(L, "NYI: callback arg type");
}
}
}
lua_rawgeti(L, ct_usr, 0);
mt = (const struct ctype*) lua_touserdata(L, -1);
| call_rrrp extern lua_callk, L_ARG, nargs, (mt->pointers || mt->type != VOID_TYPE) ? 1 : 0, 0
// Unpack the return argument if not "void", also clean-up the lua stack
// to remove the return argument and bind table. Use lua_settop rather
// than lua_pop as lua_pop is implemented as a macro.
if (mt->pointers) {
lua_getuservalue(L, -1);
lua_rawseti(L, -3, ++num_upvals); /* usr value */
lua_rawseti(L, -2, ++num_upvals); /* mt */
| call_rrr extern lua_rawgeti, L_ARG, -2, num_upvals-1
| call_rrrp extern check_typed_pointer, L_ARG, -2, -1, mt
| mov [rsp+32], rax
| call_rr extern lua_settop, L_ARG, -4
| mov rax, [rsp+32]
} else {
switch (mt->type) {
case ENUM_TYPE:
lua_getuservalue(L, -1);
lua_rawseti(L, -3, ++num_upvals); /* usr value */
lua_rawseti(L, -2, ++num_upvals); /* mt */
| call_rrr extern lua_rawgeti, L_ARG, -2, num_upvals-1
| call_rrrp, extern check_enum, L_ARG, -2, -1, mt
| mov [rsp+32], eax
| call_rr extern lua_settop, L_ARG, -4
| mov eax, [rsp+32]
break;
case VOID_TYPE:
lua_pop(L, 1);
| call_rr extern lua_settop, L_ARG, -2
break;
case BOOL_TYPE:
case INT8_TYPE:
case INT16_TYPE:
case INT32_TYPE:
lua_pop(L, 1);
if (mt->is_unsigned) {
| call_rr extern check_uint32, L_ARG, -1
} else {
| call_rr extern check_int32, L_ARG, -1
}
| mov [rsp+32], eax
| call_rr extern lua_settop, L_ARG, -3
| mov eax, [rsp+32]
break;
case INT64_TYPE:
lua_pop(L, 1);
if (mt->is_unsigned) {
| call_rr extern check_uint64, L_ARG, -1
} else {
| call_rr extern check_int64, L_ARG, -1
}
|.if X64
| mov [rsp+32], rax
|.else
| mov [rsp+32], RET_L
| mov [rsp+36], RET_H
|.endif
| call_rr extern lua_settop, L_ARG, -3
|.if X64
| mov rax, [rsp+32]
|.else
| mov RET_L, [rsp+32]
| mov RET_H, [rsp+36]
|.endif
break;
case INTPTR_TYPE:
lua_pop(L, 1);
| call_rr extern check_uintptr, L_ARG, -1
| mov [rsp+32], rax
| call_rr extern lua_settop, L_ARG, -3
| mov rax, [rsp+32]
break;
case FLOAT_TYPE:
case DOUBLE_TYPE:
lua_pop(L, 1);
| call_rr extern check_double, L_ARG, -1
|.if X64
| movq qword [rsp+32], xmm0
| call_rr extern lua_settop, L_ARG, -3
if (mt->type == FLOAT_TYPE) {
| cvtsd2ss xmm0, qword [rsp+32]
} else {
| movq xmm0, qword [rsp+32]
}
|.else
| fstp qword [rsp+32]
| call_rr extern lua_settop, L_ARG, -3
| fld qword [rsp+32]
|.endif
break;
case COMPLEX_FLOAT_TYPE:
lua_pop(L, 1);
#if !defined HAVE_COMPLEX
luaL_error(L, "ffi lib compiled without complex number support");
#endif
/* on 64 bit complex floats are two floats packed into a double,
* on 32 bit returned complex floats use eax and edx */
| call_rr extern check_complex_float, L_ARG, -1
|
|.if X64
| movq qword [rsp+32], xmm0
|.else
| mov [rsp+32], eax
| mov [rsp+36], edx
|.endif
|
| call_rr extern lua_settop, L_ARG, -3
|
|.if X64
| movq xmm0, qword [rsp+32]
|.else
| mov eax, [rsp+32]
| mov edx, [rsp+36]
|.endif
break;
case COMPLEX_DOUBLE_TYPE:
lua_pop(L, 1);
#if !defined HAVE_COMPLEX
luaL_error(L, "ffi lib compiled without complex number support");
#endif
/* on 64 bit, returned complex doubles use xmm0, xmm1, on 32 bit
* there is a hidden first parameter that points to 16 bytes where
* the returned arg is stored which is popped by the called
* function */
#if defined _WIN64 || defined __amd64__
| call_rr extern check_complex_double, L_ARG, -1
| movq qword [rsp+32], xmm0
| movq qword [rsp+40], xmm1
| call_rr extern lua_settop, L_ARG, -3
| movq xmm0, qword [rsp+32]
| movq xmm1, qword [rsp+40]
#else
| mov rcx, [rbp + hidden_arg_off]
| call_rrr extern check_complex_double, rcx, L_ARG, -1
| sub rsp, 4 // to realign from popped hidden arg
| call_rr extern lua_settop, L_ARG, -3
#endif
break;
default:
luaL_error(L, "NYI: callback return type");
}
}
|.if X64
| mov L_ARG, [rbp-8]
|.else
| mov L_ARG, [rbp-4]
|.endif
| mov rsp, rbp
| pop rbp
| ret x86_return_size(L, ct_usr, ct)
lua_pop(L, 1); /* upval table - already in registry */
assert(lua_gettop(L) == top);
ct2.is_jitted = 1;
pf = (cfunction*) push_cdata(L, ct_usr, &ct2);
*pf = compile(Dst, L, NULL, ref);
assert(lua_gettop(L) == top + 1);
return *pf;
}
void compile_function(lua_State* L, cfunction func, int ct_usr, const struct ctype* ct)
{
size_t i, nargs;
int num_upvals;
const struct ctype* mbr_ct;
struct jit* Dst = get_jit(L);
struct reg_alloc reg;
void* p;
int top = lua_gettop(L);
int* perr = &Dst->last_errno;
ct_usr = lua_absindex(L, ct_usr);
memset(®, 0, sizeof(reg));
reg.off = 32 + REGISTER_STACK_SPACE(ct);
dasm_setup(Dst, build_actionlist);
p = push_cdata(L, ct_usr, ct);
*(cfunction*) p = func;
num_upvals = 1;
nargs = lua_rawlen(L, ct_usr);
if (ct->calling_convention != C_CALL && ct->has_var_arg) {
luaL_error(L, "vararg is only allowed with the c calling convention");
}
| push rbp
| mov rbp, rsp
| push L_ARG
| push TOP
| // stack is 0 (mod 16) (TOP, L_ARG, rbp, rip)
|
| // Get L from our arguments and allocate some stack for lua_gettop
|.if X64WIN
| mov L_ARG, rcx
| sub rsp, 32 // shadow space
|.elif X64
| mov L_ARG, rdi
|.else
| mov L_ARG, [rbp + 8]
| sub rsp, 16
|.endif
|
| call_r extern lua_gettop, L_ARG
| mov TOP, rax // no need for movzxd rax, eax - high word guarenteed to be zero by x86-64
| cmp rax, nargs
| jl ->too_few_arguments
if (!ct->has_var_arg) {
| jg ->too_many_arguments
}
/* no need to zero extend eax returned by lua_gettop to rax as x86-64
* preguarentees that the upper 32 bits will be zero */
| shl rax, 4 // reserve 16 bytes per argument - this maintains the alignment mod 16
| sub rsp, rax
| sub rsp, 32 + REGISTER_STACK_SPACE(ct) // reserve an extra 32 to call local functions
#if !defined _WIN64 && !defined __amd64__
/* Returned complex doubles require a hidden first parameter where the
* data is stored, which is popped by the calling code. */
lua_rawgeti(L, ct_usr, 0);
mbr_ct = (const struct ctype*) lua_touserdata(L, -1);
if (!mbr_ct->pointers && mbr_ct->type == COMPLEX_DOUBLE_TYPE) {
/* we can allocate more space for arguments as long as no add_*
* function has been called yet, mbr_ct will be added as an upvalue in
* the return processing later */
| call_rrp extern push_cdata, L_ARG, 0, mbr_ct
| sub rsp, 16
add_pointer(Dst, ct, ®);
}
lua_pop(L, 1);
#endif
for (i = 1; i <= nargs; i++) {
lua_rawgeti(L, ct_usr, (int) i);
mbr_ct = (const struct ctype*) lua_touserdata(L, -1);
if (mbr_ct->pointers) {
lua_getuservalue(L, -1);
num_upvals += 2;
| call_rrrp extern check_typed_pointer, L_ARG, i, lua_upvalueindex(num_upvals), mbr_ct
add_pointer(Dst, ct, ®);
} else {
switch (mbr_ct->type) {
case FUNCTION_PTR_TYPE:
lua_getuservalue(L, -1);
num_upvals += 2;
| call_rrrp extern check_typed_cfunction, L_ARG, i, lua_upvalueindex(num_upvals), mbr_ct
add_pointer(Dst, ct, ®);
break;
case ENUM_TYPE:
lua_getuservalue(L, -1);
num_upvals += 2;
| call_rrrp, extern check_enum, L_ARG, i, lua_upvalueindex(num_upvals), mbr_ct
add_int(Dst, ct, ®, 0);
break;
case INT8_TYPE:
| call_rr extern check_int32, L_ARG, i
if (mbr_ct->is_unsigned) {
| movzx eax, al
} else {
| movsx eax, al
}
add_int(Dst, ct, ®, 0);
lua_pop(L, 1);
break;
case INT16_TYPE:
| call_rr extern check_int32, L_ARG, i
if (mbr_ct->is_unsigned) {
| movzx eax, ax
} else {
| movsx eax, ax
}
add_int(Dst, ct, ®, 0);
lua_pop(L, 1);
break;
case BOOL_TYPE:
| call_rr extern check_int32, L_ARG, i
| cmp eax, 0
| setne al
| movzx eax, al
add_int(Dst, ct, ®, 0);
lua_pop(L, 1);
break;
case INT32_TYPE:
if (mbr_ct->is_unsigned) {
| call_rr extern check_uint32, L_ARG, i
} else {
| call_rr extern check_int32, L_ARG, i
}
add_int(Dst, ct, ®, 0);
lua_pop(L, 1);
break;
case INTPTR_TYPE:
| call_rr extern check_uintptr, L_ARG, i
add_pointer(Dst, ct, ®);
lua_pop(L, 1);
break;
case INT64_TYPE:
if (mbr_ct->is_unsigned) {
| call_rr extern check_uint64, L_ARG, i
} else {
| call_rr extern check_int64, L_ARG, i
}
add_int(Dst, ct, ®, 1);
lua_pop(L, 1);
break;
case DOUBLE_TYPE:
| call_rr extern check_double, L_ARG, i
add_float(Dst, ct, ®, 1);
lua_pop(L, 1);
break;
case COMPLEX_DOUBLE_TYPE:
/* on 64 bit, returned complex doubles use xmm0, xmm1, on 32 bit
* there is a hidden first parameter that points to 16 bytes where
* the returned arg is stored (this is popped by the called
* function) */
#if defined _WIN64 || defined __amd64__
| call_rr extern check_complex_double, L_ARG, i
add_float(Dst, ct, ®, 1);
| movq xmm0, xmm1
add_float(Dst, ct, ®, 1);
#else
| lea rax, [rsp+reg.off]
| sub rsp, 4
| call_rrr extern check_complex_double, rax, L_ARG, i
reg.off += 16;
#endif
lua_pop(L, 1);
break;
case FLOAT_TYPE:
| call_rr extern check_double, L_ARG, i
add_float(Dst, ct, ®, 0);
lua_pop(L, 1);
break;
case COMPLEX_FLOAT_TYPE:
#if defined _WIN64 || defined __amd64__
| call_rr extern check_complex_float, L_ARG, i
/* complex floats are two floats packed into a double */
add_float(Dst, ct, ®, 1);
#else
/* returned complex floats use eax and edx */
| call_rr extern check_complex_float, L_ARG, i
| mov [rsp], eax
| fld dword [rsp]
add_float(Dst, ct, ®, 0);
| mov [rsp], edx
| fld dword [rsp]
add_float(Dst, ct, ®, 0);
#endif
lua_pop(L, 1);
break;
default:
luaL_error(L, "NYI: call arg type");
}
}
}
if (ct->has_var_arg) {
#ifdef _WIN64
|.if X64WIN
if (reg.regs < MAX_REGISTERS(ct)) {
assert(reg.regs == nargs);
| cmp TOP, MAX_REGISTERS(ct)
| jle >1
| // unpack onto stack
| mov rax, rsp
| add rax, 32 + 8*MAX_REGISTERS(ct)
| call_rrrr extern unpack_varargs_stack, L_ARG, MAX_REGISTERS(ct)+1, TOP, rax
| // unpack to registers
| mov rax, rsp
| add rax, 32 + 8*(reg.regs)
| call_rrrr extern unpack_varargs_reg, L_ARG, nargs+1, MAX_REGISTERS(ct), rax
| jmp >2
|1:
| // unpack just to registers
| mov rax, rsp
| add rax, 32 + 8*(reg.regs)
| call_rrrr extern unpack_varargs_reg, L_ARG, nargs+1, TOP, rax
|2:
} else {
| // unpack just to stack
| mov rax, rsp
| add rax, reg.off
| call_rrrr extern unpack_varargs_stack, L_ARG, nargs+1, TOP, rax
}
for (i = nargs; i < MAX_REGISTERS(ct); i++) {
reg.is_int[i] = reg.is_float[i] = 1;
}
reg.regs = MAX_REGISTERS(ct);
#elif defined __amd64__
|.elif X64
if (reg.floats < MAX_FLOAT_REGISTERS(ct)) {
| mov rax, rsp
| add rax, 32 + 8*(MAX_INT_REGISTERS(ct) + reg.floats)
| call_rrrrr extern unpack_varargs_float, L_ARG, nargs+1, TOP, MAX_FLOAT_REGISTERS(ct) - reg.floats, rax
}
if (reg.ints < MAX_INT_REGISTERS(ct)) {
| mov rax, rsp
| add rax, 32 + 8*(reg.ints)
| call_rrrrr extern unpack_varargs_int, L_ARG, nargs+1, TOP, MAX_INT_REGISTERS(ct) - reg.ints, rax
}
| mov rax, rsp
| add rax, reg.off
| call_rrrrrr extern unpack_varargs_stack_skip, L_ARG, nargs+1, TOP, MAX_INT_REGISTERS(ct) - reg.ints, MAX_FLOAT_REGISTERS(ct) - reg.floats, rax
reg.floats = MAX_FLOAT_REGISTERS(ct);
reg.ints = MAX_INT_REGISTERS(ct);
#else
|.else
| mov rax, rsp
| add rax, reg.off
| call_rrrr extern unpack_varargs_stack, L_ARG, nargs+1, TOP, rax
|.endif
#endif
}
| mov64 rcx, perr
| mov eax, dword [rcx]
| call_r extern SetLastError, rax
/* remove the stack space to call local functions */
|.if X32WIN
| add rsp, 28 // SetLastError will have already popped 4
|.else
| add rsp, 32
|.endif
#ifdef _WIN64
|.if X64WIN
switch (reg.regs) {
case 4:
if (reg.is_float[3]) {
| movq xmm3, qword [rsp + 8*3]
}
if (reg.is_int[3]) {
| mov r9, [rsp + 8*3]
}
case 3:
if (reg.is_float[2]) {
| movq xmm2, qword [rsp + 8*2]
}
if (reg.is_int[2]) {
| mov r8, [rsp + 8*2]
}
case 2:
if (reg.is_float[1]) {
| movq xmm1, qword [rsp + 8*1]
}
if (reg.is_int[1]) {
| mov rdx, [rsp + 8*1]
}
case 1:
if (reg.is_float[0]) {
| movq xmm0, qword [rsp]
}
if (reg.is_int[0]) {
| mov rcx, [rsp]
}
case 0:
break;
}
/* don't remove the space for the registers as we need 32 bytes of register overflow space */
assert(REGISTER_STACK_SPACE(ct) == 32);
#elif defined __amd64__
|.elif X64
switch (reg.floats) {
case 8:
| movq xmm7, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+7)]
case 7:
| movq xmm6, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+6)]
case 6:
| movq xmm5, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+5)]
case 5:
| movq xmm4, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+4)]
case 4:
| movq xmm3, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+3)]
case 3:
| movq xmm2, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+2)]
case 2:
| movq xmm1, qword [rsp + 8*(MAX_INT_REGISTERS(ct)+1)]
case 1:
| movq xmm0, qword [rsp + 8*(MAX_INT_REGISTERS(ct))]
case 0:
break;
}
switch (reg.ints) {
case 6:
| mov r9, [rsp + 8*5]
case 5:
| mov r8, [rsp + 8*4]
case 4:
| mov rcx, [rsp + 8*3]
case 3:
| mov rdx, [rsp + 8*2]
case 2:
| mov rsi, [rsp + 8*1]
case 1:
| mov rdi, [rsp]
case 0:
break;
}
| add rsp, REGISTER_STACK_SPACE(ct)
#else
|.else
if (ct->calling_convention == FAST_CALL) {
switch (reg.ints) {
case 2:
| mov edx, [rsp + 4]
case 1:
| mov ecx, [rsp]
case 0:
break;
}
| add rsp, REGISTER_STACK_SPACE(ct)
}
|.endif
#endif
#ifdef __amd64__
if (ct->has_var_arg) {
/* al stores an upper limit on the number of float register, note that
* its allowed to be more than the actual number of float registers used as
* long as its 0-8 */
|.if X64 and not X64WIN
| mov al, 8
|.endif
}
#endif
| call extern FUNCTION
| sub rsp, 48 // 32 to be able to call local functions, 16 so we can store some local variables
/* note on windows X86 the stack may be only aligned to 4 (stdcall will
* have popped a multiple of 4 bytes), but we don't need 16 byte alignment on
* that platform
*/
lua_rawgeti(L, ct_usr, 0);
mbr_ct = (const struct ctype*) lua_touserdata(L, -1);
if (mbr_ct->pointers || mbr_ct->type == INTPTR_TYPE) {
lua_getuservalue(L, -1);
num_upvals += 2;
| mov [rsp+32], rax // save the pointer
| get_errno
| call_rrp extern push_cdata, L_ARG, lua_upvalueindex(num_upvals), mbr_ct
| mov rcx, [rsp+32]
| mov [rax], rcx // *(void**) cdata = val
| jmp ->lua_return_arg
} else {
switch (mbr_ct->type) {
case FUNCTION_PTR_TYPE:
lua_getuservalue(L, -1);
num_upvals += 2;
| mov [rsp+32], rax // save the function pointer
| get_errno
| call_rrp extern push_cdata, L_ARG, lua_upvalueindex(num_upvals), mbr_ct
| mov rcx, [rsp+32]
| mov [rax], rcx // *(cfunction**) cdata = val
| jmp ->lua_return_arg
break;
case INT64_TYPE:
num_upvals++;
| // save the return value
|.if X64
| mov [rsp+32], rax
|.else
| mov [rsp+36], edx // high
| mov [rsp+32], eax // low
|.endif
|
| get_errno
| call_rrp extern push_cdata, L_ARG, 0, mbr_ct
|
| // *(int64_t*) cdata = val
|.if X64
| mov rcx, [rsp+32]
| mov [rax], rcx
|.else
| mov rcx, [rsp+36]
| mov rdx, [rsp+32]
| mov [rax+4], rcx
| mov [rax], rdx
|.endif
|
| jmp ->lua_return_arg
break;
case COMPLEX_FLOAT_TYPE:
num_upvals++;
|.if X64
| // complex floats are returned as two floats packed into xmm0
| movq qword [rsp+32], xmm0
|.else
| // complex floats are returned as floats in eax and edx
| mov [rsp+32], eax
| mov [rsp+36], edx
|.endif
|
| get_errno
| call_rrp extern push_cdata, L_ARG, 0, mbr_ct
|
| // ((complex_float*) cdata) = val
|.if X64
| mov rcx, [rsp+32]
| mov [rax], rcx
|.else
| mov ecx, [rsp+32]
| mov [rax], ecx
| mov ecx, [rsp+36]
| mov [rax+4], ecx
|.endif
|
| jmp ->lua_return_arg
break;
case COMPLEX_DOUBLE_TYPE:
num_upvals++;
|.if X64
| // complex doubles are returned as xmm0 and xmm1
| movq qword [rsp+40], xmm1
| movq qword [rsp+32], xmm0
|
| get_errno
| call_rrp extern push_cdata, L_ARG, 0, mbr_ct
|
| // ((complex_double*) cdata)->real = val0
| // ((complex_double*) cdata)->imag = val1
| mov rcx, [rsp+40]
| mov [rax+8], rcx
| mov rcx, [rsp+32]
| mov [rax], rcx
|
|.else
| // On 32 bit we have already handled this by pushing a new cdata
| // and handing the cdata ptr in as the hidden first param, but
| // still need to add mbr_ct as an upval as its used earlier.
| // Hidden param was popped by called function, we need to realign.
| sub rsp, 4
| get_errno
|.endif
|
| jmp ->lua_return_arg
break;
case VOID_TYPE:
lua_pop(L, 1);
| jmp ->lua_return_void
break;
case BOOL_TYPE:
lua_pop(L, 1);
| jmp ->lua_return_bool
break;
case INT8_TYPE:
lua_pop(L, 1);
if (mbr_ct->is_unsigned) {
| movzx eax, al
} else {
| movsx eax, al
}
| jmp ->lua_return_int
break;
case INT16_TYPE:
lua_pop(L, 1);
if (mbr_ct->is_unsigned) {
| movzx eax, ax
} else {
| movsx eax, ax
}
| jmp ->lua_return_int
break;
case INT32_TYPE:
case ENUM_TYPE:
lua_pop(L, 1);
if (mbr_ct->is_unsigned) {
| jmp ->lua_return_uint
} else {
| jmp ->lua_return_int
}
break;
case FLOAT_TYPE:
lua_pop(L, 1);
|.if X64
| cvtss2sd xmm0, xmm0
|.endif
| jmp ->lua_return_double
break;
case DOUBLE_TYPE:
lua_pop(L, 1);
| jmp ->lua_return_double
break;
default:
luaL_error(L, "NYI: call return type");
}
}
assert(lua_gettop(L) == top + num_upvals);
{
cfunction f = compile(Dst, L, func, LUA_NOREF);
/* add a callback as an upval so that the jitted code gets cleaned up when
* the function gets gc'd */
push_callback(L, f);
lua_pushcclosure(L, (lua_CFunction) f, num_upvals+1);
}
}