Fix PGXP support in 64bit dynarec on OSX/Linux

3 files changed, 143 insertions, 89 deletions

diff --git a/libpcsxcore/ix86_64/iPGXP-64.h b/libpcsxcore/ix86_64/iPGXP-64.h
index be69c650..c61190f0 100644
--- a/libpcsxcore/ix86_64/iPGXP-64.h
+++ b/libpcsxcore/ix86_64/iPGXP-64.h
@@ -1,6 +1,24 @@
 #ifndef _I_PGXP_H_
 #define _I_PGXP_H_
 
+// Microsoft Windows uses a different x86_64 calling convention than everyone
+// else. I have not yet bothered implementing it here because;
+//
+// 1. Nobody cares about a Windows 64-bit build of PCSXR since that would mean
+//    dropping popular closed source 32-bit plugins like Pete's OpenGL2
+// 2. The Windows convention is annoying (only 4 register params, caller must
+//    reserve stack space for register spilling) and would require more
+//    extensive code changes (e.g. the PGXP_DBG_OP_E() macro would have to
+//    handle cases where the arg needs to go on the stack instead of in a
+//    register, and cleanup afterwards).
+//
+// See https://msdn.microsoft.com/en-us/library/ms235286.aspx
+// and https://en.wikipedia.org/wiki/X86_calling_conventions#x86-64_calling_conventions
+//
+// MrLavender
+#ifdef _MSC_VER
+#error PGXP dynarec support is not implemented for Windows 64-bit
+#endif
 
 /////////////////////////////////////////////
 // PGXP wrapper functions
@@ -8,13 +26,32 @@
 
 void pgxpRecNULL() {}
 
+// Debug wrappers for x86_64 (because eOp will be last)
+#ifdef PGXP_CPU_DEBUG
+static void PGXP64_psxTraceOp(u32 code, u32 eOp) {
+	PGXP_psxTraceOp(eOp, code);
+}
+static void PGXP64_psxTraceOp1(u32 code, u32 op1, u32 eOp) {
+	PGXP_psxTraceOp1(eOp, code, op1);
+}
+static void PGXP64_psxTraceOp2(u32 code, u32 op1, u32 op2, u32 eOp) {
+	PGXP_psxTraceOp2(eOp, code, op1, op2);
+}
+static void PGXP64_psxTraceOp3(u32 code, u32 op1, u32 op2, u32 op3, u32 eOp) {
+	PGXP_psxTraceOp3(eOp, code, op1, op2, op3);
+}
+static void PGXP64_psxTraceOp4(u32 code, u32 op1, u32 op2, u32 op3, u32 op4, u32 eOp) {
+	PGXP_psxTraceOp4(eOp, code, op1, op2, op3, op4);
+}
+#endif
+
 // Choose between debug and direct function
 #ifdef PGXP_CPU_DEBUG
-#define PGXP_REC_FUNC_OP(pu, op, nReg) PGXP_psxTraceOp##nReg
-#define PGXP_DBG_OP_E(op) PUSH32I(DBG_E_##op); resp+=4;
+#define PGXP_REC_FUNC_OP(pu, op, nReg) PGXP64_psxTraceOp##nReg
+#define PGXP_DBG_OP_E(op, arg) MOV32ItoR(arg, DBG_E_##op);
 #else
 #define PGXP_REC_FUNC_OP(pu, op, nReg) PGXP_##pu##_##op
-#define PGXP_DBG_OP_E(op)
+#define PGXP_DBG_OP_E(op, arg)
 #endif
 
 #define PGXP_REC_FUNC_PASS(pu, op) \
@@ -24,45 +61,45 @@ static void pgxpRec##op() { \
 
 #define PGXP_REC_FUNC(pu, op) \
 static void pgxpRec##op() { \
-	PUSH32I(psxRegs.code); \
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, )); \
-	resp += 4; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	PGXP_DBG_OP_E(op, X86ARG2) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, )); \
 	rec##op();\
 }
 
 #define PGXP_REC_FUNC_1(pu, op, reg1) \
 static void pgxpRec##op() { \
 	reg1;\
-	PUSH32I(psxRegs.code); \
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 1)); \
-	resp += 8; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	POP64R(X86ARG2); \
+	PGXP_DBG_OP_E(op, X86ARG3) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 1)); \
 	rec##op();\
 }
 
-#define PGXP_REC_FUNC_2_2(pu, op, test, nReg, reg1, reg2, reg3, reg4) \
-static void pgxpRec##op() { \
-	if(test) { rec##op(); return; }\
-	reg1;\
-	reg2;\
-	rec##op();\
-	reg3;\
-	reg4;\
-	PUSH32I(psxRegs.code); \
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, nReg)); \
-	resp += (4 * nReg) + 4; \
-}
+//#define PGXP_REC_FUNC_2_2(pu, op, test, nReg, reg1, reg2, reg3, reg4) \
+//static void pgxpRec##op() { \
+//	if(test) { rec##op(); return; }\
+//	reg1;\
+//	reg2;\
+//	rec##op();\
+//	reg3;\
+//	reg4;\
+//	PUSH32I(psxRegs.code); \
+//	PGXP_DBG_OP_E(op) \
+//	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, nReg)); \
+//	resp += (4 * nReg) + 4; \
+//}
 
 #define PGXP_REC_FUNC_2(pu, op, reg1, reg2) \
 static void pgxpRec##op() { \
 	reg1;\
 	reg2;\
-	PUSH32I(psxRegs.code); \
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 2)); \
-	resp += 12; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	POP64R(X86ARG2); \
+	POP64R(X86ARG3); \
+	PGXP_DBG_OP_E(op, X86ARG4) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 2)); \
 	rec##op();\
 }
 
@@ -76,34 +113,34 @@ static void pgxpRec##op()	\
 	}	\
 	else\
 	{\
-		MOV32MtoR(EAX, (u32)&psxRegs.GPR.r[_Rs_]);\
+		MOV32MtoR(EAX, (uptr)&psxRegs.GPR.r[_Rs_]);\
 		if (_Imm_)\
 		{\
 			ADD32ItoR(EAX, _Imm_);\
 		}\
 	}\
-	MOV32RtoM((u32)&gTempAddr, EAX);\
+	MOV32RtoM((uptr)&gTempAddr, EAX);\
 	rec##op();\
-	PUSH64M((u32)&gTempAddr);\
 	reg1;\
-	PUSH32I(psxRegs.code);	\
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 2)); \
-	resp += 12; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	POP64R(X86ARG2); \
+	MOV32MtoR(X86ARG3, (uptr)&gTempAddr); \
+	PGXP_DBG_OP_E(op, X86ARG4) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 2)); \
 }
 
 
-#define CPU_REG_NC(idx) MOV32MtoR(EAX,(u32)&psxRegs.GPR.r[idx])
+#define CPU_REG_NC(idx) MOV32MtoR(EAX,(uptr)&psxRegs.GPR.r[idx])
 
 #define CPU_REG(idx) \
 	if (IsConst(idx))	\
 		MOV32ItoR(EAX, iRegs[idx].k);	\
 	else\
-		MOV32MtoR(EAX, (u32)&psxRegs.GPR.r[idx]);
+		MOV32MtoR(EAX, (uptr)&psxRegs.GPR.r[idx]);
 
-#define CP0_REG(idx) MOV32MtoR(EAX,(u32)&psxRegs.CP0.r[idx])
-#define GTE_DATA_REG(idx) MOV32MtoR(EAX,(u32)&psxRegs.CP2D.r[idx])
-#define GTE_CTRL_REG(idx) MOV32MtoR(EAX,(u32)&psxRegs.CP2C.r[idx])
+#define CP0_REG(idx) MOV32MtoR(EAX,(uptr)&psxRegs.CP0.r[idx])
+#define GTE_DATA_REG(idx) MOV32MtoR(EAX,(uptr)&psxRegs.CP2D.r[idx])
+#define GTE_CTRL_REG(idx) MOV32MtoR(EAX,(uptr)&psxRegs.CP2C.r[idx])
 
 static u32 gTempInstr = 0;
 static u32 gTempReg1 = 0;
@@ -113,14 +150,14 @@ static void pgxpRec##op()	\
 {	\
 	if(test) { rec##op(); return; }\
 	reg1;\
-	MOV32RtoM((u32)&gTempReg1, EAX);\
+	MOV32RtoM((uptr)&gTempReg1, EAX);\
 	rec##op();\
-	PUSH64M((u32)&gTempReg1);\
 	reg2;\
-	PUSH32I(psxRegs.code);	\
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 2)); \
-	resp += 12; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	POP64R(X86ARG2); \
+	MOV32MtoR(X86ARG3, (uptr)&gTempReg1); \
+	PGXP_DBG_OP_E(op, X86ARG4) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 2)); \
 }
 
 #define PGXP_REC_FUNC_R2_1(pu, op, test, reg1, reg2, reg3) \
@@ -128,17 +165,17 @@ static void pgxpRec##op()	\
 {	\
 	if(test) { rec##op(); return; }\
 	reg1;\
-	MOV32RtoM((u32)&gTempReg1, EAX);\
+	MOV32RtoM((uptr)&gTempReg1, EAX);\
 	reg2;\
-	MOV32RtoM((u32)&gTempReg2, EAX);\
+	MOV32RtoM((uptr)&gTempReg2, EAX);\
 	rec##op();\
-	PUSH64M((u32)&gTempReg1);\
-	PUSH64M((u32)&gTempReg2);\
 	reg3;\
-	PUSH32I(psxRegs.code);	\
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 3)); \
-	resp += 16; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	POP64R(X86ARG2); \
+	MOV32MtoR(X86ARG3, (uptr)&gTempReg2); \
+	MOV32MtoR(X86ARG4, (uptr)&gTempReg1); \
+	PGXP_DBG_OP_E(op, X86ARG5) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 3)); \
 }
 
 #define PGXP_REC_FUNC_R2_2(pu, op, test, reg1, reg2, reg3, reg4) \
@@ -146,18 +183,19 @@ static void pgxpRec##op()	\
 {	\
 	if(test) { rec##op(); return; }\
 	reg1;\
-	MOV32RtoM((u32)&gTempReg1, EAX);\
+	MOV32RtoM((uptr)&gTempReg1, EAX);\
 	reg2;\
-	MOV32RtoM((u32)&gTempReg2, EAX);\
+	MOV32RtoM((uptr)&gTempReg2, EAX);\
 	rec##op();\
-	PUSH64M((u32)&gTempReg1);\
-	PUSH64M((u32)&gTempReg2);\
 	reg3;\
 	reg4;\
-	PUSH32I(psxRegs.code);	\
-	PGXP_DBG_OP_E(op) \
-	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 4)); \
-	resp += 20; \
+	MOV32ItoR(X86ARG1, psxRegs.code); \
+	POP64R(X86ARG2); \
+	POP64R(X86ARG3); \
+	MOV32MtoR(X86ARG4, (uptr)&gTempReg2); \
+	MOV32MtoR(X86ARG5, (uptr)&gTempReg1); \
+	PGXP_DBG_OP_E(op, X86ARG6) \
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 4)); \
 }
 
 //#define PGXP_REC_FUNC_R1i_1(pu, op, test, reg1, reg2) \
@@ -167,16 +205,29 @@ static void pgxpRec##op()	\
 //	if (IsConst(reg1))	\
 //		MOV32ItoR(EAX, iRegs[reg1].k);	\
 //	else\
-//		MOV32MtoR(EAX, (u32)&psxRegs.GPR.r[reg1]);\
-//	MOV32RtoM((u32)&gTempReg, EAX);\
+//		MOV32MtoR(EAX, (uptr)&psxRegs.GPR.r[reg1]);\
+//	MOV32RtoM((uptr)&gTempReg, EAX);\
 //	rec##op();\
-//	PUSH64M((u32)&gTempReg);\
+//	PUSH64M((uptr)&gTempReg);\
 //	reg2;\
 //	PUSH32I(psxRegs.code);	\
-//	CALLFunc((u32)PGXP_REC_FUNC_OP(pu, op, 2)); \
+//	CALLFunc((uptr)PGXP_REC_FUNC_OP(pu, op, 2)); \
 //	resp += 12; \
 //}
 
+// Push m32
+#define PUSH32M(from) MOV32MtoR(EAX, from); PUSH64R(RAX);
+
+static void iPushReg(int reg)
+{
+	if (IsConst(reg)) {
+		PUSH32I(iRegs[reg].k);
+	}
+	else {
+		PUSH32M((uptr)&psxRegs.GPR.r[reg]);
+	}
+}
+
 // Rt = Rs op imm 
 PGXP_REC_FUNC_R1_1(CPU, ADDI,	!_Rt_, CPU_REG(_Rs_), iPushReg(_Rt_))
 PGXP_REC_FUNC_R1_1(CPU, ADDIU,	!_Rt_, CPU_REG(_Rs_), iPushReg(_Rt_))
@@ -187,7 +238,18 @@ PGXP_REC_FUNC_R1_1(CPU, SLTI,	!_Rt_, CPU_REG(_Rs_), iPushReg(_Rt_))
 PGXP_REC_FUNC_R1_1(CPU, SLTIU,	!_Rt_, CPU_REG(_Rs_), iPushReg(_Rt_))
 
 // Rt = imm
-PGXP_REC_FUNC_2_2(CPU, LUI, !_Rt_, 1, , , iPushReg(_Rt_), )
+//PGXP_REC_FUNC_2_2(CPU, LUI, !_Rt_, 1, , , iPushReg(_Rt_), )
+//This macro is harder to implement for x86_64, and only used once, so... :) MrL
+static void pgxpRecLUI()
+{
+	if (!_Rt_) { recLUI(); return; }
+	recLUI();
+	iPushReg(_Rt_);
+	MOV32ItoR(X86ARG1, psxRegs.code);
+	POP64R(X86ARG2);
+	PGXP_DBG_OP_E(LUI, X86ARG3)
+	CALLFunc((uptr)PGXP_REC_FUNC_OP(CPU, LUI, 1));
+}
 
 // Rd = Rs op Rt 
 PGXP_REC_FUNC_R2_1(CPU, ADD,	!_Rd_, CPU_REG(_Rt_), CPU_REG(_Rs_), iPushReg(_Rd_))
@@ -202,10 +264,10 @@ PGXP_REC_FUNC_R2_1(CPU, SLT,	!_Rd_, CPU_REG(_Rt_), CPU_REG(_Rs_), iPushReg(_Rd_)
 PGXP_REC_FUNC_R2_1(CPU, SLTU,	!_Rd_, CPU_REG(_Rt_), CPU_REG(_Rs_), iPushReg(_Rd_))
 
 // Hi/Lo = Rs op Rt
-PGXP_REC_FUNC_R2_2(CPU, MULT, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH64M((u32)&psxRegs.GPR.n.lo), PUSH64M((u32)&psxRegs.GPR.n.hi))
-PGXP_REC_FUNC_R2_2(CPU, MULTU, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH64M((u32)&psxRegs.GPR.n.lo), PUSH64M((u32)&psxRegs.GPR.n.hi))
-PGXP_REC_FUNC_R2_2(CPU, DIV, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH64M((u32)&psxRegs.GPR.n.lo), PUSH64M((u32)&psxRegs.GPR.n.hi))
-PGXP_REC_FUNC_R2_2(CPU, DIVU, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH64M((u32)&psxRegs.GPR.n.lo), PUSH64M((u32)&psxRegs.GPR.n.hi))
+PGXP_REC_FUNC_R2_2(CPU, MULT, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH32M((uptr)&psxRegs.GPR.n.lo), PUSH32M((uptr)&psxRegs.GPR.n.hi))
+PGXP_REC_FUNC_R2_2(CPU, MULTU, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH32M((uptr)&psxRegs.GPR.n.lo), PUSH32M((uptr)&psxRegs.GPR.n.hi))
+PGXP_REC_FUNC_R2_2(CPU, DIV, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH32M((uptr)&psxRegs.GPR.n.lo), PUSH32M((uptr)&psxRegs.GPR.n.hi))
+PGXP_REC_FUNC_R2_2(CPU, DIVU, 0, CPU_REG(_Rt_), CPU_REG(_Rs_), PUSH32M((uptr)&psxRegs.GPR.n.lo), PUSH32M((uptr)&psxRegs.GPR.n.hi))
 
 PGXP_REC_FUNC_ADDR_1(CPU, SB, iPushReg(_Rt_))
 PGXP_REC_FUNC_ADDR_1(CPU, SH, iPushReg(_Rt_))
@@ -232,24 +294,24 @@ PGXP_REC_FUNC_R2_1(CPU, SRLV, !_Rd_, CPU_REG(_Rs_), CPU_REG(_Rt_), iPushReg(_Rd_
 PGXP_REC_FUNC_R2_1(CPU, SRAV, !_Rd_, CPU_REG(_Rs_), CPU_REG(_Rt_), iPushReg(_Rd_))
 
 PGXP_REC_FUNC_R1_1(CPU, MFHI, !_Rd_,	CPU_REG_NC(33), iPushReg(_Rd_))
-PGXP_REC_FUNC_R1_1(CPU, MTHI, 0,		CPU_REG(_Rd_),	PUSH64M((u32)&psxRegs.GPR.n.hi))
+PGXP_REC_FUNC_R1_1(CPU, MTHI, 0,		CPU_REG(_Rd_),  PUSH32M((uptr)&psxRegs.GPR.n.hi))
 PGXP_REC_FUNC_R1_1(CPU, MFLO, !_Rd_,	CPU_REG_NC(32), iPushReg(_Rd_))
-PGXP_REC_FUNC_R1_1(CPU, MTLO, 0,		CPU_REG(_Rd_),	PUSH64M((u32)&psxRegs.GPR.n.lo))
+PGXP_REC_FUNC_R1_1(CPU, MTLO, 0,		CPU_REG(_Rd_),  PUSH32M((uptr)&psxRegs.GPR.n.lo))
 
 // COP2 (GTE)
 PGXP_REC_FUNC_R1_1(GTE, MFC2, !_Rt_, GTE_DATA_REG(_Rd_), iPushReg(_Rt_))
 PGXP_REC_FUNC_R1_1(GTE, CFC2, !_Rt_, GTE_CTRL_REG(_Rd_), iPushReg(_Rt_))
-PGXP_REC_FUNC_R1_1(GTE, MTC2, 0, CPU_REG(_Rt_), PUSH64M((u32)&psxRegs.CP2D.r[_Rd_]))
-PGXP_REC_FUNC_R1_1(GTE, CTC2, 0, CPU_REG(_Rt_), PUSH64M((u32)&psxRegs.CP2C.r[_Rd_]))
+PGXP_REC_FUNC_R1_1(GTE, MTC2, 0, CPU_REG(_Rt_), PUSH32M((uptr)&psxRegs.CP2D.r[_Rd_]))
+PGXP_REC_FUNC_R1_1(GTE, CTC2, 0, CPU_REG(_Rt_), PUSH32M((uptr)&psxRegs.CP2C.r[_Rd_]))
 
-PGXP_REC_FUNC_ADDR_1(GTE, LWC2, PUSH64M((u32)&psxRegs.CP2D.r[_Rt_]))
-PGXP_REC_FUNC_ADDR_1(GTE, SWC2, PUSH64M((u32)&psxRegs.CP2D.r[_Rt_]))
+PGXP_REC_FUNC_ADDR_1(GTE, LWC2, PUSH32M((uptr)&psxRegs.CP2D.r[_Rt_]))
+PGXP_REC_FUNC_ADDR_1(GTE, SWC2, PUSH32M((uptr)&psxRegs.CP2D.r[_Rt_]))
 
 // COP0
 PGXP_REC_FUNC_R1_1(CP0, MFC0, !_Rd_, CP0_REG(_Rd_), iPushReg(_Rt_))
 PGXP_REC_FUNC_R1_1(CP0, CFC0, !_Rd_, CP0_REG(_Rd_), iPushReg(_Rt_))
-PGXP_REC_FUNC_R1_1(CP0, MTC0, !_Rt_, CPU_REG(_Rt_), PUSH64M((u32)&psxRegs.CP0.r[_Rd_]))
-PGXP_REC_FUNC_R1_1(CP0, CTC0, !_Rt_, CPU_REG(_Rt_), PUSH64M((u32)&psxRegs.CP0.r[_Rd_]))
+PGXP_REC_FUNC_R1_1(CP0, MTC0, !_Rt_, CPU_REG(_Rt_), PUSH32M((uptr)&psxRegs.CP0.r[_Rd_]))
+PGXP_REC_FUNC_R1_1(CP0, CTC0, !_Rt_, CPU_REG(_Rt_), PUSH32M((uptr)&psxRegs.CP0.r[_Rd_]))
 PGXP_REC_FUNC(CP0, RFE)
 
 #endif//_I_PGXP_H_
diff --git a/libpcsxcore/ix86_64/iR3000A-64.c b/libpcsxcore/ix86_64/iR3000A-64.c
index e1245987..64d9c1d5 100755
--- a/libpcsxcore/ix86_64/iR3000A-64.c
+++ b/libpcsxcore/ix86_64/iR3000A-64.c
@@ -191,16 +191,6 @@ static void iFlushRegs() {
 	}
 }
 
-static void iPushReg(int reg)
-{
-	if (IsConst(reg)) {
-		PUSH32I(iRegs[reg].k);
-	}
-	else {
-		PUSH64M((uptr)&psxRegs.GPR.r[reg]);
-	}
-}
-
 static void iStoreCycle() {
 	count = ((pc - pcold) / 4) * BIAS;
 	ADD32ItoM((uptr)&psxRegs.cycle, count);
diff --git a/libpcsxcore/ix86_64/ix86-64.h b/libpcsxcore/ix86_64/ix86-64.h
index 009fa5a1..f0c84023 100755
--- a/libpcsxcore/ix86_64/ix86-64.h
+++ b/libpcsxcore/ix86_64/ix86-64.h
@@ -90,6 +90,8 @@ extern x86IntRegType g_x86non8bitregs[3];
 #define X86ARG2 RSI
 #define X86ARG3 RDX
 #define X86ARG4 RCX
+#define X86ARG5 R8
+#define X86ARG6 R9
 #endif
 
 #else