/***************************************************************************
 *
 * All modifications in this file to the original code are
 * (C) Copyright 1992, ..., 2014 the "DOSEMU-Development-Team".
 *
 * for details see file COPYING in the DOSEMU distribution
 *
 *
 *  SIMX86 a Intel 80x86 cpu emulator
 *  Copyright (C) 1997,2001 Alberto Vignani, FIAT Research Center
 *				a.vignani@crf.it
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * Additional copyright notes:
 *
 * 1. The kernel-level vm86 handling was taken out of the Linux kernel
 *  (linux/arch/i386/kernel/vm86.c). This code originally was written by
 *  Linus Torvalds with later enhancements by Lutz Molgedey and Hans Lermen.
 *
 ***************************************************************************/

/*
 * BACK-END for the cpuemu interpreter.
 *
 * It translates the intermediate ops (defined in codegen.h) to their
 * final binary form and stores the generated code into a temporary
 * buffer (CodeBuf).
 * These intermediate ops are still being reworked and grow in an
 * incremental way; I hope they will converge to some better defined
 * set as soon as I'll start coding for some other processor.
 *
 * There should be other similar modules, one for each target. So you
 * can have codegen-ppc.c or codegen-emulated.c or whatever else.
 *
 * This module generates x86 code. Hey, wait... x86 from x86?
 * Actually the generated code runs always in 32-bit mode, so in a way
 * the 16-bit V86 mode is "emulated".
 *
 * All instructions operate on a virtual CPU image in memory ("TheCPU"),
 * and are completely self-contained. They read from TheCPU registers,
 * operate, and store back to the same registers. There's an exception -
 * FLAGS, which are not stored back until the end of a code block.
 * In fact, you will note that there's NO flag handling here, because on
 * x86 we use the real hardware to calculate them, and this speeds up
 * things a lot compared to a full interpreter. Flags will be a nightmare
 * for non-x86 host CPUs.
 *
 * This only applies to the condition code flags though, OF, SF, ZF, AF,
 * PF and CF (0x8d5). All other flags are stored in EFLAGS=TheCPU.eflags,
 * including DF. Normally the real DF is clear for compatibility with C
 * code; it is only temporarily set during string instructions.
 *
 * There is NO optimization for the produced code. It is a very pipeline-
 * unconscious code full of register dependencies and reloadings.
 * Clearly we hope that the 1st level cache of the host CPU works as
 * advertised ;-)
 *
 */

/***************************************************************************
 *
 * Registers on enter:
 *	ebx/rbx		pointer to SynCPU (must not be changed)
 *	ebp/rbp		memory base address (mem_base)
 *	[esp/rsp]	cpu->eflags
 *
 * Registers used by the 32/64-bit machine:
 *	eax		scratch, data
 *	ebx/rbx		pointer to SynCPU (must not be changed)
 *	ecx		scratch, address/count
 *	edx		scratch, data
 *	esi		scratch, address
 *	edi		memory/register address
 *	ebp/rbp		memory base address (mem_base)
 *	esp/rsp		not modified
 *	flags		modified
 *
 * Registers on exit:
 *	eax		PC for the next instruction
 *	edx		flags
 *	edi		last memory address
 *
 ***************************************************************************/
//

#include <stddef.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include "utilities.h"
#include "emu86.h"
#include "cpatch.h"
#include "codegen-x86.h"
#include "misc/dis8086.h"

static unsigned char *CodeGen_x86(unsigned char *CodePtr, unsigned char *BaseGenBuf, const IGen *IG);
static unsigned Exec_x86(void *SeqStart);

/////////////////////////////////////////////////////////////////////////////

/* This code is appended at the end of every instruction sequence. It
 * passes back the IP of the next instruction after the sequence.
 * (the one where we switch back to interpreted code).
 *
 *		movl #return_PC,eax
 *		movl eax, ecx (signal no link)
 *		popl edx (flags)
 *		ret
 */

static unsigned char TailCode[TAILSIZE+1] =
	{ 0xb8,0,0,0,0,0x89,0x43,Ofs_KEY,0x5a,0xc3,0xf4 };

/*
 * This function is only here for looking at the generated binary code
 * with objdump.
 */
static void _test_(void) __attribute__((used));

static void _test_(void)
{
	__asm__ __volatile__ (" \
		nop \
		" : : : "memory" );
}

/////////////////////////////////////////////////////////////////////////////

void InitGen_x86(void)
{
	CodeGen = CodeGen_x86;
	Exec = Exec_x86;
	Cpatch_init();
}

/////////////////////////////////////////////////////////////////////////////


/* NOTE: parameters IG->px must be the last argument in a Gn() macro
 * because of the OR operator, which would cause trouble if the parameter
 * is negative */

static unsigned char *CodeGen_x86(unsigned char *CodePtr, unsigned char *BaseGenBuf, const IGen *IG)
{
	/* evil hack, keeping state from MOVS_SavA to MOVS_SetA in
	   a static variable */
	static unsigned char * rep_retry_ptr = (unsigned char*)0xdeadbeef;
	unsigned char *Cp = CodePtr;
	unsigned char * CpTemp;
	int mode = IG->mode;
	int rcod;
#if PROFILE >= 2
	hitimer_t t0 = 0;
	if (debug_level('e')) t0 = GETTSC();
#endif

	switch(IG->op) {
	case A_DI_0:			// base(32), imm
		// movl $imm,%%edi
		G1(0xbf,Cp); G4(IG->p1,Cp);
		if (!(mode & MLEA)) {
			// addl offs(%%ebx),%%edi (seg reg offset)
			G3M(0x03,0x7b,IG->p0,Cp);
		}
		break;
	case A_DI_1: {			// base(32), {imm}, reg
		int idsp=IG->p1;

		if (mode & ADDR16) {
			// movzwl offs(%%ebx),%%edi
			G4M(0x0f,0xb7,0x7b,IG->p2,Cp);
			if ((mode&IMMED) && (idsp!=0)) {
				// addw $immed,%%di
				G3(0xc78166,Cp); G2(idsp,Cp);
			}
		}
		else {
			// movl offs(%%ebx),%%edi
			G3M(0x8b,0x7b,IG->p2,Cp);
			if (idsp!=0) {
				GenLeaEDI(idsp);
			}
		}
		if (!(mode & MLEA)) {
			// addl offs(%%ebx),%%edi (seg reg offset)
			G3M(0x03,0x7b,IG->p0,Cp);
		} }
		break;
	case A_DI_2: {			// base(32), {imm}, reg, reg, {shift}
		int idsp=IG->p1;

		if (mode & ADDR16) {
			// movzwl offs(%%ebx),%%edi
			G4M(0x0f,0xb7,0x7b,IG->p3,Cp);
			// addw offs(%%ebx),%%di
			G4M(0x66,0x03,0x7b,IG->p2,Cp);
			if (idsp!=0) {
				// addw $immed,%%di
				G3(0xc78166,Cp); G2(idsp,Cp);
			}
		}
		else {
			unsigned char sh = IG->p4;
			// movl offs(%%ebx),%%edi
			G3M(0x8b,0x7b,IG->p3,Cp);
			if (sh) {
				// shll $1,%%edi
				if (sh==1) { G2(0xe7d1,Cp); }
				// shll $count,%%edi
				else { G2(0xe7c1,Cp); G1(sh,Cp); }
			}
			// addl offs(%%ebx),%%edi
			G3M(0x03,0x7b,IG->p2,Cp);
			if (idsp!=0) {
			    GenLeaEDI(idsp);
			}
		}
		if (!(mode & MLEA)) {
			// addl offs(%%ebx),%%edi (seg reg offset)
			G3M(0x03,0x7b,IG->p0,Cp);
		} }
		break;
	case A_DI_2D: {			// modrm_sibd, 32-bit mode
		int idsp = IG->p1;
		unsigned char sh = IG->p3;
		// movl offs(%%ebx),%%edi
		G3M(0x8b,0x7b,IG->p2,Cp);
		// shll $count,%%ecx
		if (sh)	{
			// shll $1,%%edi
			if (sh==1) { G2(0xe7d1,Cp); }
			// shll $count,%%edi
			else { G2(0xe7c1,Cp); G1(sh,Cp); }
		}
		if (idsp!=0) {
			GenLeaEDI(idsp);
		}
		if (!(mode & MLEA)) {
			// addl offs(%%ebx),%%edi (seg reg offset)
			G3M(0x03,0x7b,IG->p0,Cp);
		} }
		break;
	case A_SR_SH4: {	// real mode make base addr from seg
		// movw %%ax,offs(%%ebx)
		G4M(0x66,0x89,0x43,IG->p0,Cp);
		// movzwl %%ax,%%eax
		G3(0xc0b70f,Cp);
		// shll $4,%%eax
		G3M(0xc1,0xe0,0x04,Cp);
		// movl %%eax,ofs(%%ebx)
		G3M(0x89,0x43,IG->p1,Cp);
		// addl $0xffff,%eax
		G1(0x05,Cp); G4(0x0000ffff,Cp);
		// movl %%eax,ofs(%%ebx)
		G3M(0x89,0x43,IG->p1+4,Cp);
		}
		break;
	case A_SR_PROT: {	// prot mode make base addr from seg
		// movl Ofs,%%edx
		G1(0xba,Cp); G4(IG->p0,Cp);
		// call Ofs_SetSegProt(%%ebx)
		G3M(0xff,0x53,Ofs_SetSegProt(),Cp);
		// cmpl $0x0,Ofs_ERR(%rbx)
		G4M(0x83,0x7b,Ofs_ERR,0x00,Cp);
		// jz skip
		G2M(JE_JZ,TAILSIZE,Cp);
		// movl {exit_addr},%%eax; movl %%eax, %%Ofs_KEY(%%ebx)
		G1(0xb8,Cp); G4(IG->p1,Cp); G3M(0x89,0x43,Ofs_KEY,Cp);
		// pop %%edx; ret
		G2M(0x5a,0xc3,Cp);
		}
		break;
	case L_NOP:
		G1(0x90,Cp);
		break;
	// Special case: CR0&0x3f
	case L_CR0:
		// movl Ofs_CR0(%%ebx),%%eax
		G2M(0x8b,0x83,Cp); G4(Ofs_CR0,Cp);
		// andl $0x3f,%%eax
		G3(0x3fe083,Cp);
		break;
	case O_FOP:
		Cp = Fp87_op_x86(Cp, IG->p0, IG->p1);
		break;

	case L_REG: {
		if (mode&(MBYTE|MBYTX))	{
			// movb offs(%%ebx),%%al
			G3M(0x8a,0x43,IG->p0,Cp);
		}
		else {
			// mov{wl} offs(%%ebx),%%{e}ax
			Gen66(mode,Cp); G3M(0x8b,0x43,IG->p0,Cp);
		} }
		break;
	case S_REG: {
		if (mode&MBYTE)	{
			// movb %%al,offs(%%ebx)
			G3M(0x88,0x43,IG->p0,Cp);
		}
		else {
			// mov{wl} %%{e}ax,offs(%%ebx)
			Gen66(mode,Cp); G3M(0x89,0x43,IG->p0,Cp);
		} }
		break;
	case L_REG2REG: {
		if (mode&MBYTE) {
			G3M(0x8a,0x43,IG->p0,Cp);	// rsrc
			G3M(0x88,0x43,IG->p1,Cp);	// rdest
		} else {
			Gen66(mode,Cp);	G3M(0x8b,0x43,IG->p0,Cp);
			Gen66(mode,Cp);	G3M(0x89,0x43,IG->p1,Cp);
		} }
		break;
	case S_DI_R: {
		// mov{wl} %%{e}di,offs(%%ebx)
		Gen66(mode,Cp);	G3M(0x89,0x7b,IG->p0,Cp);
		}
		break;
	case S_DI_IMM: {
		if (mode&MBYTE) {
			// movb $xx,(%%edi)
			G1(0xb0,Cp); G1(IG->p0,Cp);
			STD_WRITE_B;
		} else {
			// mov{wl} $xx,(%%edi)
			G1(0xb8,Cp); G4(IG->p0,Cp);
			STD_WRITE_WL(mode);
		} }
		break;

	case L_IMM: {
		if (mode&MBYTE)	{
			// movb $immed,offs(%%ebx)
			G3M(0xc6,0x43,IG->p0,Cp); G1(IG->p1,Cp);
		}
		else {
			// mov{wl} $immed,offs(%%ebx)
			Gen66(mode,Cp);
			G3M(0xc7,0x43,IG->p0,Cp); G2_4(mode,IG->p1,Cp);
		} }
		break;
	case L_IMM_R1: {
		if (mode&MBYTE)	{
			// movb $immed,%%al
			G1(0xb0,Cp); G1(IG->p0,Cp);
		}
		else {
			// mov{wl} $immed,%%{e}ax
			Gen66(mode,Cp);	G1(0xb8,Cp); G2_4(mode,IG->p0,Cp);
		} }
		break;
	case L_MOVZS:
		if (mode & MBYTX) {
			if (!(mode & DATA16)) {
				// mov{sz}bw %%al,%%eax
				G3M(0x0f,(0xb6|(IG->p0<<3)),0xc0,Cp);
			}
			else if (IG->p0) {
				// movsbw %%al,%%ax = cbw
				G2M(0x66,0x98,Cp);
			}
			else {
				// movzbw %%al,%%ax = movb $0, %%ah
				G2M(0xb4,0x00,Cp);
			}
		}
		else {
			if (mode & DATA16) {
				// mov{sz}ww %%ax,%%ax
				G4M(0x66,0x0f,(0xb7|(IG->p0<<3)),0xc0,Cp);
			}
			else if (IG->p0) {
				// movswl %%ax,%%eax = cwde
				G1(0x98,Cp);
			}
			else {
				// movzwl %%ax,%%eax
				G3M(0x0f,0xb7,0xc0,Cp);
			}
		}
		// mov{wl} %%{e}ax,offs(%%ebx)
		Gen66(mode,Cp); G3M(0x89,0x43,IG->p1,Cp);
		break;

	case L_LXS1: {
		// mov{wl} (%%edi,%%ebp,1),%%{e}ax
		Gen66(mode,Cp); G3M(0x8b,0x04,0x2f,Cp);
		// mov{wl} %%{e}ax,offs(%%ebx)
		Gen66(mode,Cp);	G3M(0x89,0x43,IG->p0,Cp);
		}
		break;
	case L_LXS2: {	/* load segment from memory + 2/4 */
		// leal {2|4}(%%edi),%%edi
		G2M(0x8d,0x7f,Cp); G1((mode&DATA16? 2:4),Cp);
		// movzwl (%%edi,%%ebp,1),%%eax
		G4M(0x0f,0xb7,0x04,0x2f,Cp);
		// leal {-2|-4}(%%edi),%%edi
		G2M(0x8d,0x7f,Cp); G1((mode&DATA16? -2:-4),Cp);
		}
		break;

	case L_DI_R1:
		if (mode&(MBYTE|MBYTX)) {
		    G3(0x2f048a,Cp);
		}
		else {
		    Gen66(mode,Cp);
		    G3(0x2f048b,Cp);
		}
		break;
	case S_DI:
		if (mode&MBYTE) {
		    STD_WRITE_B;
		}
		else {
		    STD_WRITE_WL(mode);
		}
		break;

	case O_ADD_R:				// acc = acc op	reg
		rcod = ADDbtrm; goto arith0;
	case O_OR_R:
		rcod = ORbtrm; goto arith0;
	case O_ADC_R:
		rcod = ADCbtrm; goto arith0;
	case O_SBB_R:
		rcod = SBBbtrm; goto arith0;
	case O_AND_R:
		rcod = ANDbtrm; goto arith0;
	case O_SUB_R:
		rcod = SUBbtrm; goto arith0;
	case O_XOR_R:
		rcod = XORbtrm; goto arith0;
	case O_CMP_R:
		rcod = CMPbtrm; goto arith0;
	case O_INC_R:
		rcod = GRP2brm; goto arith0;
	case O_DEC_R:
		rcod = 0x08fe;
arith0:		{
		G1(POPdx,Cp);	// get flags from stack into %%edx
		switch (IG->op) {
		case O_ADC_R: // tests carry
		case O_SBB_R: // tests carry
		case O_INC_R: // preserves carry
		case O_DEC_R: // preserves carry
			// shr $1,%%edx to get carry flag from stack
			G2M(0xd1,0xea,Cp);
		}
		if (mode & MBYTE) {
			if (mode & IMMED) {
				// OPb $immed,%%al
				G1(rcod+2,Cp); G1(IG->p0,Cp);
			}
			else {
				// OPb offs(%%ebx),%%al
				G2(0x4300|rcod,Cp); G1(IG->p0,Cp);
			}
		}
		else {
			if (mode & IMMED) {
				// OP{lw} $immed,%%{e}ax
				Gen66(mode,Cp);
				G1(rcod+3,Cp); G2_4(mode,IG->p0,Cp);
			}
			else {
				// OP{wl} offs(%%ebx),%%{e}ax
				Gen66(mode,Cp);
				G2(0x4301|rcod,Cp); G1(IG->p0,Cp);
			}
		}
		G1(PUSHF,Cp);	// flags back on stack
		}
		break;
	case O_CLEAR:
		G3M(POPdx,0x31,0xc0,Cp);	//ignore flags; xorl %%eax,%%eax
		if (mode & MBYTE) {
			// movb %%al,offs(%%ebx)
			G3M(0x88,0x43,IG->p0,Cp);
		}
		else {
			// mov{wl} %%{e}ax,offs(%%ebx)
			Gen66(mode,Cp); G3M(0x89,0x43,IG->p0,Cp);
		}
		G1(PUSHF,Cp);	// new flags on stack
		break;
	case O_TEST:
		G1(POPdx,Cp);			// ignore flags
		if (mode & MBYTE) {
			// testb $0xff,offs(%%ebx)
			G4M(0xf6,0x43,IG->p0,0xffu,Cp);
		}
		else if (mode&DATA16) {
			// testw $0xffff,offs(%%ebx)
			G4M(0x66,0xf7,0x43,IG->p0,Cp); G2(0xffff,Cp);
		}
		else {
			// test $0xffffffff,offs(%%ebx)
			G3M(0xf7,0x43,IG->p0,Cp); G4(0xffffffff,Cp);
		}
		G1(PUSHF,Cp);	// new flags on stack
		break;
	case O_SBSELF:
		// if CY=0 -> reg=0,  flag=xx46
		// if CY=1 -> reg=-1, flag=xx97
		// pop %%edx; shr $1,%%edx to get carry flag from stack
		G3M(POPdx,0xd1,0xea,Cp);
		// sbbl %%eax,%%eax
		G2M(0x19,0xc0,Cp);
		if (mode & MBYTE) {
			// movb %%al,offs(%%ebx)
			G3M(0x88,0x43,IG->p0,Cp);
		}
		else {
			// mov{wl} %%{e}ax,offs(%%ebx)
			Gen66(mode,Cp); G3M(0x89,0x43,IG->p0,Cp);
		}
		G1(PUSHF,Cp);	// flags back on stack
		break;
	case O_ADD_FR:
		rcod = ADDbfrm; /* 0x00 */ goto arith1;
	case O_OR_FR:
		rcod = ORbfrm;  /* 0x08 */ goto arith1;
	case O_ADC_FR:
		rcod = ADCbfrm; /* 0x10 */ goto arith1;
	case O_SBB_FR:
		rcod = SBBbfrm; /* 0x18 */ goto arith1;
	case O_AND_FR:
		rcod = ANDbfrm; /* 0x20 */ goto arith1;
	case O_SUB_FR:
		rcod = SUBbfrm; /* 0x28 */ goto arith1;
	case O_XOR_FR:
		rcod = XORbfrm; /* 0x30 */ goto arith1;
	case O_CMP_FR:
		rcod = CMPbfrm; /* 0x38 */
arith1:
		G1(POPdx,Cp);	// get flags from stack into %%edx
		if (IG->op == O_ADC_FR || IG->op == O_SBB_FR) {
			// shr $1,%%edx to get carry flag from stack
			G2M(0xd1,0xea,Cp);
		}
		if (mode & MBYTE) {
			if (mode & IMMED) {
				// OPb $immed,offs(%%ebx)
				G4M(0x80,0x43|rcod,IG->p0,IG->p1,Cp);
			}
			else {
				// OPb %%al,offs(%%ebx)
				G2(0x4300|rcod,Cp); G1(IG->p0,Cp);
			}
		}
		else {
			if (mode & IMMED) {
				// OP{wl} $immed,offs(%%ebx)
				Gen66(mode,Cp);
				G3M(0x81,0x43|rcod,IG->p0,Cp);
				G2_4(mode,IG->p1,Cp);
			}
			else {
				// OP{wl} %%eax,offs(%%ebx)
				Gen66(mode,Cp);
				G2(0x4301|rcod,Cp); G1(IG->p0,Cp);
			}
		}
		G1(PUSHF,Cp);	// flags back on stack
		break;
	case O_NOT:
		if (mode & MBYTE) {
			// notb %%al
			G2M(0xf6,0xd0,Cp);
		}
		else {
			// NOT{wl} %%(e)ax
			Gen66(mode,Cp);
			G2M(0xf7,0xd0,Cp);
		}
		break;
	case O_NEG:
		G1(POPdx,Cp);	// ignore flags from stack
		if (mode & MBYTE) {
			// negb %%al
			G2M(0xf6,0xd8,Cp);
		}
		else {
			// neg{wl} (%%edi)
			Gen66(mode,Cp);
			G2M(0xf7,0xd8,Cp);
		}
		G1(PUSHF,Cp);	// new flags on stack
		break;
	case O_INC:
		G1(POPdx,Cp);	// get flags from stack into %%edx
		// shr $1,%%edx to get preserved carry flag from stack
		G2M(0xd1,0xea,Cp);
		if (mode & MBYTE) {
			// incb %%al
			G2M(0xfe,0xc0,Cp);
		}
		else if (mode & DATA16) {
			// inc %%ax
#ifdef __x86_64__ // 0x40 is a REX byte, not inc
			G3M(0x66,0xff,0xc0,Cp);
#else
			G2M(0x66,0x40,Cp);
#endif
		}
		else {
			// inc %%eax
#ifdef __x86_64__
			G2M(0xff,0xc0,Cp);
#else
			G1(0x40,Cp);
#endif
		}
		G1(PUSHF,Cp);	// flags back on stack before writing
		break;
	case O_DEC:
		G1(POPdx,Cp);	// get flags from stack into %%edx
		// shr $1,%%edx to get preserved carry flag from stack
		G2M(0xd1,0xea,Cp);
		if (mode & MBYTE) {
			// decb %%al
			G2M(0xfe,0xc8,Cp);
		}
		else if (mode & DATA16) {
			// dec %%ax
#ifdef __x86_64__ // 0x48 is a REX byte, not dec
			G3M(0x66,0xff,0xc8,Cp);
#else
			G2M(0x66,0x48,Cp);
#endif
		}
		else {
			// dec %%eax
#ifdef __x86_64__
			G2M(0xff,0xc8,Cp);
#else
			G1(0x48,Cp);
#endif
		}
		G1(PUSHF,Cp);	// flags back on stack
		break;
	case O_CMPXCHG: {
		G1(POPdx,Cp);	// ignore flags from stack
		if (mode & MBYTE) {
			// movb offs1(%%ebx),%%dl
			G3M(0x8a,0x53,IG->p0,Cp);
			// movb %%al,%%cl
			G2M(0x88,0xc1,Cp);
			// movb Ofs_AL(%%ebx),%%al
			G3M(0x8a,0x43,Ofs_AL,Cp);
			// cmpxchgb %%dl,%%cl
			G3M(0x0f,0xb0,0xd1,Cp);
			// movb %%al,Ofs_AL(%%ebx)
			G3M(0x88,0x43,Ofs_AL,Cp);
			// movb %%cl,%%al,
			G2M(0x88,0xc8,Cp);
		}
		else {
			// mov{wl} offs1(%%ebx),%%{e}dx
			Gen66(mode,Cp);	G3M(0x8b,0x53,IG->p0,Cp);
			// mov{wl} %%{e}ax,%%{e}cx
			Gen66(mode,Cp); G2M(0x89,0xc1,Cp);
			// mov{wl} Ofs_EAX(%%ebx),%%{e}ax
			Gen66(mode,Cp); G3M(0x8b,0x43,Ofs_EAX,Cp);
			// cmpxchg{wl} %%{e}dx,%%{e}cx
			Gen66(mode,Cp);	G3M(0x0f,0xb1,0xd1,Cp);
			// mov{wl} %%{e}ax,Ofs_EAX(%%ebx)
			Gen66(mode,Cp);	G3M(0x89,0x43,Ofs_EAX,Cp);
			// mov{wl} %%{e}cx,%%{e}ax
			Gen66(mode,Cp); G2M(0x89,0xc8,Cp);
		} }
		G1(PUSHF,Cp);	// flags back on stack
		break;
	case O_XCHG: {
		if (mode & MBYTE) {
			// xchgb offs(%%ebx),%%al
			G3M(0x86,0x43,IG->p0,Cp);
		}
		else {
			// xchg{wl} offs(%%ebx),%%{e}ax
			Gen66(mode,Cp);	G3M(0x87,0x43,IG->p0,Cp);
		} }
		break;
	case O_XCHG_R: {
		// mov{wl} offs1(%%ebx),%%{e}ax
		Gen66(mode,Cp);	G3M(0x8b,0x43,IG->p0,Cp);
		// xchg{wl} offs2(%%ebx),%%{e}ax
		Gen66(mode,Cp); G3M(0x87,0x43,IG->p1,Cp);
		// mov{wl} %%{e}ax,offs1(%%ebx)
		Gen66(mode,Cp);	G3M(0x89,0x43,IG->p0,Cp);
		}
		break;
	case O_MUL:
		G1(POPF,Cp);	// get flags from stack
		if (mode & MBYTE) {
			// mulb Ofs_AL(%%ebx),%%al
			G3M(0xf6,0x63,Ofs_AL,Cp);
			// movw %%ax,Ofs_AX(%%ebx)
			G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
		}
		else if (mode&DATA16) {
			// mulw Ofs_AX(%%ebx),%%ax
			G4M(OPERoverride,0xf7,0x63,Ofs_AX,Cp);
			// movw %%ax,Ofs_AX(%%ebx)
			G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
			// movw %%dx,Ofs_DX(%%ebx)
			G4M(OPERoverride,0x89,0x53,Ofs_DX,Cp);
		}
		else {
			// mull Ofs_EAX(%%ebx),%%eax
			G3M(0xf7,0x63,Ofs_EAX,Cp);
			// movl %%eax,Ofs_EAX(%%ebx)
			G3M(0x89,0x43,Ofs_EAX,Cp);
			// movl %%edx,Ofs_EDX(%%ebx)
			G3M(0x89,0x53,Ofs_EDX,Cp);
		}
		G1(PUSHF,Cp);	// flags back on stack
		break;
	case O_IMUL:
		G1(POPF,Cp);	// get flags from stack
		if (mode & MBYTE) {
			if ((mode&(IMMED|DATA16))==(IMMED|DATA16)) {
				// imul $immed,%%ax,%%ax
				G3M(OPERoverride,0x6b,0xc0,Cp); G1(IG->p0,Cp);
				// movw %%ax,offs(%%ebx)
				G4M(OPERoverride,0x89,0x43,IG->p1,Cp);
			}
			else if ((mode&(IMMED|DATA16))==IMMED) {
				// imul $immed,%%eax,%%eax
				G2M(0x6b,0xc0,Cp); G1(IG->p0,Cp);
				// movl %%eax,offs(%%ebx)
				G3M(0x89,0x43,IG->p1,Cp);
			}
			else {
				// imul Ofs_AL(%%ebx),%%al
				G3M(0xf6,0x6b,Ofs_AL,Cp);
				// movw %%ax,Ofs_AX(%%ebx)
				G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
			}
		}
		else if (mode&DATA16) {
			if (mode&IMMED) {
				// imul $immed,%%ax,%%ax
				G3M(OPERoverride,0x69,0xc0,Cp); G2(IG->p0,Cp);
				// movw %%ax,offs(%%ebx)
				G4M(OPERoverride,0x89,0x43,IG->p1,Cp);
			}
			else if (mode&MEMADR) {
				// imul offs(%%ebx),%%ax
				G4M(OPERoverride,0x0f,0xaf,0x43,Cp);
				G1(IG->p0,Cp);
				// movw %%ax,offs(%%ebx)
				G4M(OPERoverride,0x89,0x43,IG->p0,Cp);
			}
			else {
				// imul Ofs_AX(%%ebx),%%ax
				G4M(OPERoverride,0xf7,0x6b,Ofs_AX,Cp);
				// movw %%ax,Ofs_AX(%%ebx)
				G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
				// movw %%dx,Ofs_DX(%%ebx)
				G4M(OPERoverride,0x89,0x53,Ofs_DX,Cp);
			}
		}
		else {
			if (mode&IMMED) {
				// imul $immed,%%eax,%%eax
				G2M(0x69,0xc0,Cp); G4(IG->p0,Cp);
				// movl %%eax,offs(%%ebx)
				G3M(0x89,0x43,IG->p1,Cp);
			}
			else if (mode&MEMADR) {
				// imul offs(%%ebx),%%eax
				G4M(0x0f,0xaf,0x43,IG->p0,Cp);
				// movl %%eax,offs(%%ebx)
				G3M(0x89,0x43,IG->p0,Cp);
			}
			else {
				// imul Ofs_EAX(%%ebx),%%eax
				G3M(0xf7,0x6b,Ofs_EAX,Cp);
				// movl %%eax,Ofs_EAX(%%ebx)
				G3M(0x89,0x43,Ofs_EAX,Cp);
				// movl %%edx,Ofs_EDX(%%ebx)
				G3M(0x89,0x53,Ofs_EDX,Cp);
			}
		}
		G1(PUSHF,Cp);	// flags back on stack
		break;

	case O_DIV: {
		G1(POPF,Cp);	// get flags from stack
		G2(0xc189,Cp);  // movl %%eax,%%ecx
		if (mode & MBYTE) {
			// movw Ofs_AX(%%ebx),%%ax
			G4M(OPERoverride,0x8b,0x43,Ofs_AX,Cp);
			// div %%cl,%%al
			G2M(0xf6,0xf1,Cp);
			// movw %%ax,Ofs_AX(%%ebx)
			G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
		}
		else if (mode&DATA16) {
			// movw Ofs_AX(%%ebx),%%ax
			G4M(OPERoverride,0x8b,0x43,Ofs_AX,Cp);
			// movw Ofs_DX(%%ebx),%%dx
			G4M(OPERoverride,0x8b,0x53,Ofs_DX,Cp);
			// div %%cx,%%ax
			G3M(OPERoverride,0xf7,0xf1,Cp);
			// movw %%ax,Ofs_AX(%%ebx)
			G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
			// movw %%dx,Ofs_DX(%%ebx)
			G4M(OPERoverride,0x89,0x53,Ofs_DX,Cp);
		}
		else {
			// movl Ofs_EAX(%%ebx),%%eax
			G3M(0x8b,0x43,Ofs_EAX,Cp);
			// movl Ofs_EDX(%%ebx),%%edx
			G3M(0x8b,0x53,Ofs_EDX,Cp);
			// div %%ecx,%%eax
			G2M(0xf7,0xf1,Cp);
			// movl %%eax,Ofs_EAX(%%ebx)
			G3M(0x89,0x43,Ofs_EAX,Cp);
			// movl %%edx,Ofs_EDX(%%ebx)
			G3M(0x89,0x53,Ofs_EDX,Cp);
		}
		G1(PUSHF,Cp);	// flags back on stack
		}
		break;
	case O_IDIV: {
		G1(POPF,Cp);	// get flags from stack
		G2(0xc189,Cp);  // movw %%eax,%%ecx
		if (mode & MBYTE) {
			// movw Ofs_AX(%%ebx),%%ax
			G4M(OPERoverride,0x8b,0x43,Ofs_AX,Cp);
			// idiv %%cl,%%al
			G2M(0xf6,0xf9,Cp);
			// movw %%ax,Ofs_AX(%%ebx)
			G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
		}
		else if (mode&DATA16) {
			// movw Ofs_AX(%%ebx),%%ax
			G4M(OPERoverride,0x8b,0x43,Ofs_AX,Cp);
			// movw Ofs_DX(%%ebx),%%dx
			G4M(OPERoverride,0x8b,0x53,Ofs_DX,Cp);
			// idiv %%cx,%%ax
			G3M(OPERoverride,0xf7,0xf9,Cp);
			// movw %%ax,Ofs_AX(%%ebx)
			G4M(OPERoverride,0x89,0x43,Ofs_AX,Cp);
			// movw %%dx,Ofs_DX(%%ebx)
			G4M(OPERoverride,0x89,0x53,Ofs_DX,Cp);
		}
		else {
			// movl Ofs_EAX(%%ebx),%%eax
			G3M(0x8b,0x43,Ofs_EAX,Cp);
			// movl Ofs_EDX(%%ebx),%%edx
			G3M(0x8b,0x53,Ofs_EDX,Cp);
			// idiv %%ecx,%%eax
			G2M(0xf7,0xf9,Cp);
			// movl %%eax,Ofs_EAX(%%ebx)
			G3M(0x89,0x43,Ofs_EAX,Cp);
			// movl %%edx,Ofs_EDX(%%ebx)
			G3M(0x89,0x53,Ofs_EDX,Cp);
		}
		G1(PUSHF,Cp);	// flags back on stack
		}
		break;

	case O_CBWD:
		// movl Ofs_EAX(%%ebx),%%eax
		G3M(0x8b,0x43,Ofs_EAX,Cp);
		if (mode & MBYTE) {		/* 0x98: CBW,CWDE */
			if (mode & DATA16) {	// AL->AX
				// cbw
				G2(0x9866,Cp);
				// movw %%ax,Ofs_AX(%%ebx)
				G4M(0x66,0x89,0x43,Ofs_AX,Cp);
			}
			else {			// AX->EAX
				// cwde
				// movl %%eax,Ofs_EAX(%%ebx)
				G4M(0x98,0x89,0x43,Ofs_EAX,Cp);
			}
		}
		else if	(mode &	DATA16)	{	/* 0x99: AX->DX:AX */
			// cwd
			G2(0x9966,Cp);
			// movw %%dx,Ofs_DX(%%ebx)
			G4M(0x66,0x89,0x53,Ofs_DX,Cp);
		}
		else {	/* 0x99: EAX->EDX:EAX */
			// cdq
			// movl %%edx,Ofs_EDX(%%ebx)
			G4M(0x99,0x89,0x53,Ofs_EDX,Cp);
		}
		break;
	case O_XLAT:
		// movl (seg reg offset)(%%ebx),%%edi
		G2(0x7b8b,Cp); G1(IG->p0,Cp);
		// movzbl Ofs_AL(%%ebx),%%ecx
		G4M(0x0f,0xb6,0x4b,Ofs_AL,Cp);
		// movl Ofs_EBX(%%ebx),%%eax
		G3M(0x8b,0x43,Ofs_EBX,Cp);
		// leal (%%ecx,%%eax,1),%%ecx
		G3(0x010c8d,Cp);
		if (mode & ADDR16) {
			// movzwl %%cx,%%ecx
			G3(0xC9B70F,Cp);
		}
		// leal (%%ecx,%%edi,1),%%edi
		G3M(0x8d,0x3c,0x39,Cp);
		break;

	case O_ROL:
		rcod = 0x00; goto shrot0;
	case O_ROR:
		rcod = 0x08; goto shrot0;
	case O_RCL:
		rcod = 0x10; goto shrot0;
	case O_RCR:
		rcod = 0x18; goto shrot0;
	case O_SHL:
		rcod = 0x20; goto shrot0;
	case O_SHR:
		rcod = 0x28; goto shrot0;
	case O_SAR:
		rcod = 0x38;
shrot0:
		G1(0x9d,Cp);	// get flags from stack
		if (mode & MBYTE) {
			// op al,1:	d0 c0+r
			// op al,n:	c0 c0+r	n
			// op al,cl:	d2 c0+r
			if (mode & IMMED) {
				unsigned char sh = IG->p0;
				G1(sh==1? 0xd0:0xc0,Cp);
				G1(0xc0	| rcod,Cp);
				if (sh!=1) G1(sh,Cp);
			}
			else {
				// movb Ofs_CL(%%ebx),%%cl
				G3M(0x8a,0x4b,Ofs_CL,Cp);
				// OPb %%cl,%%al
				G1(0xd2,Cp); G1(0xc0 | rcod,Cp);
			}
		}
		else {
			// op (e)ax,1:	(66) d1	c0+r
			// op (e)ax,n:	(66) c1	c0+r n
			// op (e)ax,cl:	(66) d3	c0+r
			if (mode & IMMED) {
				unsigned char sh = IG->p0;
				Gen66(mode,Cp);	G1(sh==1? 0xd1:0xc1,Cp);
				G1(0xc0	| rcod,Cp);
				if (sh!=1) G1(sh,Cp);
			}
			else {
				// movb Ofs_CL(%%ebx),%%cl
				G3M(0x8a,0x4b,Ofs_CL,Cp);
				// OP{wl} %%cl,(%%edi)
				Gen66(mode,Cp);
				G1(0xd3,Cp); G1(0xc0 | rcod,Cp);
			}
		}
		G1(0x9c,Cp);	// flags back on stack
		break;

	case O_OPAX: {	/* used by DAA,DAS,AAA,AAS,AAM,AAD */
			// movl Ofs_EAX(%%ebx),%%eax
			G4M(0x9d,0x8b,0x43,Ofs_EAX,Cp);
#ifdef __x86_64__ /* have to emulate all of them... */
			switch(IG->p1) {
			case DAA:
			case DAS: {
				int op = (IG->p1 == DAS ? 0x28 : 0);
				const unsigned char pseq[] = {
				// pushf; mov %al,%cl; add $0x66,%al
				0x9c,0x88,0xc1,0x04,0x66,
				// pushf; pop %rax; pop %rdx
				0x9c,0x58,0x5a,
				// or %dl,%al; and $0x11,%al // combine AF/CF
				0x08,0xd0,0x24,0x11,
				// mov %al,%dl; rol $4,%al
				0x88,0xc2,0xc0,0xc0,0x04,
				// imul $6,%eax // multiply 0 CF 0 0 0 AF by 6
				0x6b,0xc0,0x06};

				GNX(Cp, pseq, sizeof(pseq));
				// add/sub %al,%cl; pushf // Combine flags from
				G3M(op,0xc1,0x9c,Cp);
				// or %dl,(%rsp) // add/sub with AF/CF
				G3M(0x08,0x14,0x24,Cp);
				// movb %%cl,Ofs_EAX(%%ebx)
				G3M(0x88,0x4b,Ofs_EAX,Cp);
				break;
			}
			case AAA:
			case AAS: {
				int op = (IG->p1 == AAS ? 0x28 : 0);
				const unsigned char pseq[] = {
				// pushf; mov %eax,%ecx; and 0xf,%al
				0x9c,0x89,0xc1,0x24,0x0f,
				// add $6,%al; pop %edx
				0x04,0x06,0x5a,
				// or %dl,%al; and $0xee,%dl // ~(AF|CF)
				0x08,0xd0,0x80,0xe2,0xee,
				// and $0x10,%al; xchg %eax,%ecx; jz 1f
				0x24,0x10,0x91,0x74,0x07};
				GNX(Cp, pseq, sizeof(pseq));

				// add/sub $0x106, %ax
				G4M(0x66,op+0x05,0x06,0x01,Cp);
				// or $0x11,%dl; (AF|CF) 1: and $0xf,%al
				G3M(0x80,0xca,0x11,Cp); G2M(0x24,0x0f,Cp);
				// push %rdx; movl %%eax,Ofs_EAX(%%ebx)
				G4M(0x52,0x89,0x43,Ofs_EAX,Cp);
				break;
			}
			case AAM:
				// mov $0,%ah; mov p2,%cl
				G4M(0xb4,0x00,0xb1,IG->p2,Cp);
				// div %cl; xchg %al,%ah
				G4M(0xf6,0xf1,0x86,0xc4,Cp);
				// orb %al,%al (for flags)
				G2M(0x08,0xc0,Cp);
				// movl %%eax,Ofs_EAX(%%ebx)
				G4M(0x89,0x43,Ofs_EAX,0x9c,Cp);
				break;
			case AAD:
				// mov %al,%cl; mov %ah,%al
				G4M(0x88,0xc1,0x88,0xe0,Cp);
				// mov p2,%ah; mul %ah
				G4M(0xb4,IG->p2,0xf6,0xe4,Cp);
				// add %cl,%al; mov $0,%ah
				G4M(0x00,0xc8,0xb4,0x00,Cp);
				// movl %%eax,Ofs_EAX(%%ebx)
				G4M(0x89,0x43,Ofs_EAX,0x9c,Cp);
				break;
			default:
				error("Unimplemented O_OPAX instruction\n");
				leavedos_main(99);
			}
#else
			// get n>0,n<3 bytes from parameter stack
			G1(IG->p1,Cp);
			if (IG->p0==2) { G1(IG->p2,Cp); }
			// movl %%eax,Ofs_EAX(%%ebx)
			G4M(0x89,0x43,Ofs_EAX,0x9c,Cp);
#endif
		}
		break;

/* PUSH derived (sub-)sequences: */
	case O_PUSH1: {
		const unsigned char pseq[] = {
			// movl Ofs_ESP(%%ebx),%%ecx
			0x8b,0x4b,Ofs_ESP,
			// movl Ofs_XSS(%%ebx),%%esi
			0x8b,0x73,Ofs_XSS,
		};
		GNX(Cp, pseq, sizeof(pseq));
		} break;

	case O_PUSH2: {		/* register push only */
		const unsigned char pseq16[] = {
			// movw offs(%%ebx),%%ax
/*00*/			0x90,0x66,0x8b,0x43,0x00,
			// leal -2(%%ecx),%%ecx
/*05*/			0x8d,0x49,0xfe,
			// andl StackMask(%%ebx),%%ecx
			0x23,0x4b,Ofs_STACKM,
			// leal (%%esi,%%ecx,1),%%edx
			0x8d,0x14,0x0e,
			// movw %%ax,(%%edx,%%ebp,1)
			0x66,0x89,0x04,0x2a,
		};
		const unsigned char pseq32[] = {
			// movl offs(%%ebx),%%eax
/*00*/			0x90,0x90,0x8b,0x43,0x00,
			// leal -4(%%ecx),%%ecx
			0x8d,0x49,0xfc,
			// andl StackMask(%%ebx),%%ecx
			0x23,0x4b,Ofs_STACKM,
			// leal (%%esi,%%ecx,1),%%edx
			0x8d,0x14,0x0e,
			// movl %%eax,(%%edx,%%ebp,1)
			0x89,0x04,0x2a,
		};
		const unsigned char *p;
		unsigned char *q;
		int sz;
		if (mode&(SEGREG|DATA16)) p=pseq16,sz=sizeof(pseq16);
			else p=pseq32,sz=sizeof(pseq32);
		q=Cp; GNX(Cp, p, sz);
		q[4] = IG->p0;
		if ((mode & (SEGREG|DATA16)) == SEGREG) {
			// use pseq16 but -4 for 32-bit segreg
			// mirroring recent Intel CPUs
			q[7] = 0xfc;
		} else if (mode & MNOREG) {
			memset(q, 0x90, 5);
		} else if (mode & IMMED) {
			uint32_t v = IG->p0;
			q[0] = 0xb8; // movl $v, %%eax
			memcpy(q + 1, &v, 4);
		}
		} break;

	case O_PUSH3: {
#ifdef KEEP_ESP	/* keep high 16-bits of ESP in small-stack mode */
		const unsigned char pseq[] = {
			// movl StackMask(%%ebx),%%edx
			0x8b,0x53,Ofs_STACKM,
			// notl %%edx
			0xf7,0xd2,
			// andl Ofs_ESP(%%ebx),%%edx
			0x23,0x53,Ofs_ESP,
			// orl %%edx,%%ecx
			0x09,0xd1,
		};
		GNX(Cp, pseq, sizeof(pseq));
#endif
		// movl %%ecx,Ofs_ESP(%%ebx)
		G3M(0x89,0x4b,Ofs_ESP,Cp);
		} break;

	case O_PUSH2F: {
		const unsigned char pseq[] = {
			// movl (%%esp),%%edx	(get flags on stack)
			0x8b,0x14,0x24,
			// movl Ofs_FLAGS(%%ebx),%%eax
			0x8b,0x43,Ofs_EFLAGS,
			// andw EFLAGS_CC,%%dx	(0x8d5: OF/SF/ZF/AF/PF/CF)
			0x66,0x81,0xe2,0xd5,0x08,
			// andw	~EFLAGS_CC,%%ax
			0x66,0x25,0x2a,0xf7,
			// orw %%dx,%%ax
			0x66,0x09,0xd0,
		};
		GNX(Cp, pseq, sizeof(pseq));
#if 0		// unused "extended PVI", if used should move to separate op
		if (!V86MODE() && IOPL < 3 && (TheCPU.cr[4] & CR4_PVI)) {
		    /* This solves the DOSX 'System test 8' error.
		     * The virtualized IF is pushed instead of the
		     * real one (which is always 1). This way, tests
		     * on the pushed value of the form
		     *		cli
		     *		pushf
		     *		test 0x200,(esp)
		     * don't fail anymore. It is not clear, apart this
		     * special test case, whether pushing the virtual
		     * IF is actually useful; probably not. In any
		     * case, POPF ignores this IF on stack.
		     * Since PUSHF doesn't trap in PM, non-cpuemued
		     * dosemu will always fail this particular test.
		     */
			// rcr $10,%%eax	(IF->cy)
			G3M(0xc1,0xd8,0x0a,Cp);
			// bt $19,(_EFLAGS-TheCPU)(%ebx) (test for VIF)
			G3M(0x0f,0xba,0xa3,Cp);
			/* relative ebx offset works on x86-64 too */
			G4((unsigned char *)&_EFLAGS-CPUOFFS(0),Cp);
			// (19 from bt); rcl $10,%%eax
			G4M(0x13,0xc1,0xd0,0x0a,Cp);
		}
#endif
		if (!(mode&DATA16)) {
			// andl RETURN_MASK|EFLAGS_IF,%%eax
			G1(0x25,Cp); G4(RETURN_MASK|EFLAGS_IF,Cp);
		}
		} break;

/* POP derived (sub-)sequences: */
	case O_POP1: {
		const unsigned char pseq[] = {
			// movl Ofs_XSS(%%ebx),%%ecx
			0x8b,0x4b,Ofs_XSS,
			// movl Ofs_ESP(%%ebx),%%esi
			0x8b,0x73,Ofs_ESP,
		};
		GNX(Cp, pseq, sizeof(pseq));
		} break;

	case O_POP2: {
		const unsigned char pseq16[] = {
			// andl StackMask(%%ebx),%%esi
			0x23,0x73,Ofs_STACKM,
			// leal (%%esi,%%ecx,1),%%edx
			0x8d,0x14,0x0e,
			// movw (%%edx,%%ebp,1),%%ax
			0x66,0x8b,0x04,0x2a,
/*10*/			// leal 2(%%esi),%%esi
			0x8d,0x76,0x02,
		};
		const unsigned char pseq32[] = {
			// andl StackMask(%%ebx),%%esi
			0x23,0x73,Ofs_STACKM,
			// leal (%%esi,%%ecx,1),%%edx
			0x8d,0x14,0x0e,
			// movl (%%edx,%%ebp,1),%%eax
			0x90,0x8b,0x04,0x2a,
			// leal 4(%%esi),%%esi
			0x8d,0x76,0x04,
		};
		const unsigned char *p;
		int sz;
		if (mode&(SEGREG|DATA16)) p=pseq16,sz=sizeof(pseq16);
			else p=pseq32,sz=sizeof(pseq32);
		// for popping into memory the sequence is:
		//	first do address calculation, then pop,
		//	then store data, and last adjust stack
		GNX(Cp, p, sz);
		if ((mode & (SEGREG|DATA16)) == SEGREG) {
			// use pseq16 but +4 for 32-bit segreg
			// mirroring recent Intel CPUs
			Cp[-sz+12] = 0x04;
		}
		if (mode & MRETISP) {
			// leal IG->p0(%%esi),%%esi
			G2M(0x8d,0xb6,Cp); G4(IG->p0,Cp);
		}
		if (!(mode & MNOREG)) {
			// mov{wl} %%{e}ax,offs(%%ebx)
			Gen66(mode, Cp); G3M(0x89,0x43,IG->p0,Cp);
		}
		} break;

	case O_POP3: {
#ifdef STACK_WRAP_MP	/* mask after incrementing */
		// andl StackMask(%%ebx),%%esi
		G3M(0x23,0x73,Ofs_STACKM,Cp);
#endif
#ifdef KEEP_ESP	/* keep high 16-bits of ESP in small-stack mode */
		const unsigned char pseq[] = {
			// movl StackMask(%%ebx),%%edx
			0x8b,0x53,Ofs_STACKM,
			// notl %%edx
			0xf7,0xd2,
			// andl Ofs_ESP(%%ebx),%%edx
			0x23,0x53,Ofs_ESP,
			// orl %%edx,%%esi
			0x09,0xd6,
		};
		GNX(Cp, pseq, sizeof(pseq));
#endif
		// movl %%esi,Ofs_ESP(%%ebx)
		G3M(0x89,0x73,Ofs_ESP,Cp);
		} break;

	case O_INT: {
		unsigned char intno = IG->p0;
		int jpc = IG->p1;
		// Check bitmap, GPF if revectored, else use intno*4
		// bt intno,Ofs_int_revectored(%ebx)
		// Code offset of TheCPU.int_revectored.__map[intno>>5]
		// as Ofs_int_revectored[intno>>3] aligned to 4.
		// See http://x86.dapsen.com/html/file_module_x86_id_22.html
		G3M(0x0f,0xba,0xa3,Cp);	G4(Ofs_int_revectored+((intno>>3)&~3),Cp);
		// remaining 5 intno bits within __map dword from bt
		G1(intno&0x1f,Cp);
		// jnc skip return
		G2M(0x73,TAILSIZE+7,Cp);
		// movb EXCP0D_GPF, Ofs_ERR(%%ebx)
		G2M(0xc6,0x83,Cp); G4(Ofs_ERR,Cp); G1(EXCP0D_GPF,Cp);
		// movl {exit_addr},%%eax; mov %%eax, Ofs_KEY(%%ebx);
		G1(0xb8,Cp); G4(jpc,Cp); G3M(0x89,0x43,Ofs_KEY,Cp);
		// pop %%edx; ret
		G2M(0x5a,0xc3,Cp);
		// address to call in edi
		// movl $(inum*4), %edi
		G1(0xbf,Cp); G4(intno*4, Cp);
		break;
		}

	case O_SIM:
		// push $arg
		G1(PUSHwi,Cp); G4(IG->p1,Cp);
		// push $opc
		G1(PUSHwi,Cp); G4(IG->p0,Cp);
		// push $mode
		G1(PUSHwi,Cp); G4(mode,Cp);
		// call Ofs_SimHelper(%%ebx)
		G3M(0xff,0x53,Ofs_SimHelper(),Cp);
		// cmpl $0x0,Ofs_ERR(%rbx)
		G4M(0x83,0x7b,Ofs_ERR,0x00,Cp);
		// je skip
		G2M(JE_JZ,TAILSIZE,Cp);
		// movl {exit_addr},%%eax; movl %eax,Ofs_KEY(%ebx);
		G1(MOViax,Cp); G4(IG->p2,Cp); G3M(0x89,0x43,Ofs_KEY,Cp);
		// pop %%edx; ret
		G2M(POPdx,RET,Cp);

	case O_MOVS_SetA: {
		/* use edi for loads unless MOVSDST or REP is set */
		unsigned char modrm = mode&(MREP|MREPNE|MOVSDST) ? 0x73 : 0x7b;
		unsigned char modrm2 = mode&(MREP|MREPNE|MOVSDST) ? 0xee : 0xef;
		if (mode&ADDR16) {
		    /* The CX load has to be before the address reloads */
		    if (mode&(MREP|MREPNE)) {
			// movzwl Ofs_CX(%%ebx),%%ecx
			G4M(0x0f,0xb7,0x4b,Ofs_CX,Cp);
			rep_retry_ptr = Cp;
		    }
		    if (mode&MOVSSRC) {
			// movzwl Ofs_SI(%%ebx),%%e[sd]i
			G4M(0x0f,0xb7,modrm,Ofs_SI,Cp);
		    	if(mode & (MREPNE|MREP))
		    	{
			    /* EAX: iterations possible until address overflow
			            if DF is set. */
			    // movl %%esi,%%eax
			    G2M(MOVwfrm,0xF0,Cp);
			    if(!(mode & MBYTE))
			    {
			    	if(mode & DATA16)
			    	{
				    // shrl $1,%%eax
				    G2M(SHIFTw,0xE8,Cp);
				}
				else
				{
				    // shrl $2,%%eax
				    G3M(SHIFTwi,0xE8,2,Cp);
				}
			    }
			    // incl %%eax
#ifdef __x86_64__ // 0x40 is a REX byte, not inc
			    G2M(0xff,0xc0,Cp);
#else
			    G1(INCax,Cp);
#endif
			}
			// addl (seg reg offset)(%%ebx),%%e[sd]i
			G3M(0x03,modrm,IG->p0,Cp);
			if (mode&(MREP|MREPNE)) {
			    // addl %%[re]bp,%%[re][sd]i
			    Gen48(Cp); G2M(0x01,modrm2,Cp);
			}
		    }
		    if (mode&MOVSDST) {
			// movzwl Ofs_DI(%%ebx),%%edi
			G4M(0x0f,0xb7,0x7b,Ofs_DI,Cp);
		    	if(mode & (MREPNE|MREP))
		    	{
			    /* EDX: iterations possible until address overflow
			            if DF is set. */
			    // movl %%edi,%%edx
			    G2M(MOVwfrm,0xFA,Cp);
			    if(!(mode & MBYTE))
			    {
			    	if(mode & DATA16)
			    	{
				    // shrl $1,%%edx
				    G2M(SHIFTw,0xEA,Cp);
				}
				else
				{
				    // shrl $2,%%edx
				    G3M(SHIFTwi,0xEA,2,Cp);
				}
			    }
			    // incl %%edx
#ifdef __x86_64__ // 0x42 is a REX byte, not inc
			    G2M(0xff,0xc2,Cp);
#else
			    G1(INCdx,Cp);
#endif
			}
			// addl Ofs_XES(%%ebx),%%edi
			G3M(0x03,0x7b,Ofs_XES,Cp);
			if (mode&(MREP|MREPNE)) {
			    // addl %%[re]bp,%%[re]di
			    Gen48(Cp); G2M(0x01,0xef,Cp);
			}
		    }
		    if (mode&(MREP|MREPNE)) {
			/* Address overflow detection */
			// testl $4,Ofs_EFLAGS+1(%%ebx)
			G4M(GRP1brm,0x43,Ofs_EFLAGS+1,0x4,Cp);
			// jnz 0f (distance is 8 bytes per limit to adjust)
			G2M(JNE_JNZ,(mode&(MOVSDST|MOVSSRC)) == (MOVSDST|MOVSSRC) ?
			                0x10:0x08,Cp);
			/* correct for cleared DF */
			if(mode&MOVSSRC)
			{
			    // negl %%eax
			    G2M(GRP1wrm,0xD8,Cp);
			    // addl $(0x10000/opsize+1),%%eax
			    G2M(IMMEDwrm,0xC0,Cp); G4(0x10000/OPSIZE(mode)+1,Cp);
			}
			if(mode&MOVSDST)
			{
			    // negl %%edx
			    G2M(GRP1wrm,0xDA,Cp);
			    // addl $(0x10000/opsize+1),%%edx
			    G2M(IMMEDwrm,0xC2,Cp); G4(0x10000/OPSIZE(mode)+1,Cp);
			}
			// 0:
			/* consolidate limits to edx */
			switch(mode&(MOVSDST|MOVSSRC))
			{
			    case MOVSDST:
				/* nothing to do, limit already in edx */
				break;
			    case MOVSSRC:
			    	/* limit in eax, want it in edx */
				// xchg %%eax,%%edx
			        G1(XCHGdx,Cp);
			        break;
			    case MOVSSRC | MOVSDST:
				/* smaller limit to edx */
				// cmp %%eax,%%edx
				G2M(CMPwtrm,0xD0,Cp);
				// jb 0f
				G2M(JB_JNAE,0x01,Cp);
				// xchg %%eax,%%edx
				G1(XCHGdx,Cp);
				break;
			}
			// cmp %%ecx,%%edx
			G2M(CMPwfrm,0xCA,Cp);
			// jbe 0f
			G2M(JBE_JNA,0x02,Cp);
			// mov %%ecx,%%edx
			G2M(MOVwfrm,0xCA,Cp);
			// 0:
			// xchg %%ecx,%%edx
			G2M(XCHGwrm,0xCA,Cp);
			// sub %%ecx, %%edx
			G2M(SUBwfrm,0xCA,Cp);
		    }
		}
		else {
		    if (mode&MOVSSRC) {
			// movl (seg reg offset)(%%ebx),%%e[sd]i
			G3M(0x8b,modrm,IG->p0,Cp);
			// addl Ofs_ESI(%%ebx),%%e[sd]i
			G3M(0x03,modrm,Ofs_ESI,Cp);
		    }
		    if (mode&MOVSDST) {
			// movl Ofs_XES(%%ebx),%%edi
			G3M(0x8b,0x7b,Ofs_XES,Cp);
			// addl Ofs_EDI(%%ebx),%%edi
			G3M(0x03,0x7b,Ofs_EDI,Cp);
		    }
		    if (mode&(MREP|MREPNE)) {
			// movl Ofs_ECX(%%ebx),%%ecx
			G3M(0x8b,0x4b,Ofs_ECX,Cp);
			if (mode&MOVSSRC) {
			    // addl %%[re]bp,%%[re][sd]i
			    Gen48(Cp); G2M(0x01,modrm2,Cp);
			}
			if (mode&MOVSDST) {
			    // addl %%[re]bp,%%[re]di
			    Gen48(Cp); G2M(0x01,0xef,Cp);
			}
		    }
		} }
		break;

	case O_MOVS_MovD:
		GetDF(Cp);
		G1(REP,Cp);
		if (mode&MBYTE)	{
			G2M(MOVSb,NOP,Cp);
		}
		else if (mode&DATA16) {
			G2M(OPERoverride,MOVSw,Cp);
		}
		else {
			G2M(MOVSw,NOP,Cp);
		}
		G2M(NOP,NOP,Cp);
		G1(CLD,Cp);
		break;
	case O_MOVS_LodD:
		GetDF(Cp);
		G1(REP,Cp);
		if (mode&MBYTE)	{
			G2M(LODSb,NOP,Cp);
		}
		else if (mode&DATA16) {
			G2M(OPERoverride,LODSw,Cp);
		}
		else {
			G2M(LODSw,NOP,Cp);
		}
		G2M(NOP,NOP,Cp);
		G1(CLD,Cp);
		break;
	case O_MOVS_StoD:
		GetDF(Cp);
		G1(REP,Cp);
		if (mode&MBYTE)	{
			G2M(STOSb,NOP,Cp);
		}
		else if (mode&DATA16) {
			G2M(OPERoverride,STOSw,Cp);
		}
		else {
			G2M(STOSw,NOP,Cp);
		}
		G2M(NOP,NOP,Cp);
		G1(CLD,Cp);
		break;
	case O_MOVS_ScaD:
		CpTemp = NULL;
		G2M(JCXZ,00,Cp);
		// Pointer to the jecxz distance byte
		CpTemp = Cp-1;
		GetDF(Cp);
		G1((mode&MREP)?REP:REPNE,Cp);
		if (mode&MBYTE)	{
			G2M(SCASb,NOP,Cp);
		}
		else if (mode&DATA16) {
			G2M(OPERoverride,SCASw,Cp);
		}
		else {
			G2M(SCASw,NOP,Cp);
		}
		G2M(NOP,NOP,Cp);
		G3M(CLD,POPsi,PUSHF,Cp); // replace flags back on stack,esi=dummy
		*CpTemp = (Cp-(CpTemp+1));
		break;
	case O_MOVS_CmpD:
		if(!(mode & (MREP|MREPNE))) {
			// assumes eax=(%%esi)
			// mov %%eax, %%edx
			G2M(0x89,0xc2,Cp);
			// mov (%%edi,%%ebp,1), %%{e}a[xl]
			if (mode&MBYTE) {
				G4M(0x8a,0x04,0x2f,0x90,Cp);
			}
			else if (mode&DATA16) {
				G4M(0x66,0x8b,0x04,0x2f,Cp);
			}
			else {
				G4M(0x8b,0x04,0x2f,0x90,Cp);
			}
			G2(0x9090,Cp);
			// cmp %%eax, %%edx
			if (mode&MBYTE) {
				G2M(0x38,0xc2,Cp);
			}
			else {
				Gen66(mode,Cp);
				G2M(0x39,0xc2,Cp);
			}
			// replace flags back on stack,eax=dummy
			G2M(POPax,PUSHF,Cp);
			break;
		}
		CpTemp = NULL;
		G2M(JCXZ,00,Cp);
		// Pointer to the jecxz distance byte
		CpTemp = Cp-1;
		GetDF(Cp);
		G1((mode&MREP)?REP:REPNE,Cp);
		if (mode&MBYTE)	{
			G2M(CMPSb,NOP,Cp);
		}
		else if (mode&DATA16) {
			G2M(OPERoverride,CMPSw,Cp);
		}
		else {
			G2M(CMPSw,NOP,Cp);
		}
		G2M(NOP,NOP,Cp);
		G3M(CLD,POPax,PUSHF,Cp); // replace flags back on stack,eax=dummy
		*CpTemp = (Cp-(CpTemp+1));
		break;

	case O_MOVS_SavA:
		if (!(mode&(MREP|MREPNE))) {
		    // %%edx set to DF's increment
		    // movsbl Ofs_DF_INCREMENTS+OPSIZEBIT(mode)(%%ebx),%%edx
		    G4M(0x0f,0xbe,0x53,Ofs_DF_INCREMENTS+OPSIZEBIT(mode),Cp);
		    if(mode & MOVSSRC) {
			if (mode & ADDR16)
				G1(0x66,Cp);
			// add{wl} %{e}dx,Ofs_SI(%%ebx)
			G3M(0x01,0x53,Ofs_SI,Cp);
		    }
		    if(mode & MOVSDST) {
			if (mode & ADDR16)
				G1(0x66,Cp);
			// add{wl} %{e}dx,Ofs_DI(%%ebx)
			G3M(0x01,0x53,Ofs_DI,Cp);
		    }
		}
		else if (mode&ADDR16) {
		    if(mode & MREPCOND)
		    {
			/* it is important to *NOT* destroy the flags here, so
			   use lea instead of add. Flags are needed for termination
			   detection */
			// lea 0(%%edx,%%ecx),%%ecx	; add remaining to cx
			G3M(0x8D,0x0C,0x11,Cp);
			/* terminate immediately if rep was stopped by flags */
			// j[n]z 0f
			G2M((mode&MREP)?JE_JZ:JNE_JNZ,0x02,Cp);
			// xor %%edx,%%edx	; clear remaining
			G2M(XORwtrm,0xD2,Cp);
			// 0:
		    }
		    else if(mode & (MREP|MREPNE))
		    {
			/* use shorter add instruction for nonconditional reps */
			// add %%edx,%%ecx
			G2M(ADDwtrm,0xCA,Cp);
		    }
		    if (mode&MOVSSRC) {
			// esi = base1 + CPU_(e)SI +- n
			// subl %%[re]bp,%%[re]si
			Gen48(Cp); G2M(0x29,0xee,Cp);
			// subl (seg reg offset)(%%ebx),%%esi
			G2(0x732b,Cp); G1(IG->p0,Cp);
			// movw %%si,Ofs_SI(%%ebx)
			G4M(0x66,0x89,0x73,Ofs_SI,Cp);
		    }
		    if (mode&MOVSDST) {
			// edi = base2 + CPU_(e)DI +- n
			// subl %%[re]bp,%%[re]di
			Gen48(Cp); G2M(0x29,0xef,Cp);
			// subl Ofs_XES(%%ebx),%%edi
			G3M(0x2b,0x7b,Ofs_XES,Cp);
			// movw %%di,Ofs_DI(%%ebx)
			G4M(0x66,0x89,0x7b,Ofs_DI,Cp);
		    }
		    // continue after SI/DI overflow; store ecx
		    if (mode&(MREP|MREPNE)) {
			unsigned char * jmpbackbase;
			// or %%edx,%%edx
			G2M(ORwtrm,0xD2,Cp);
			// jnz retry
			jmpbackbase = Cp;
			G2M(JNE_JNZ,(rep_retry_ptr-jmpbackbase-2)&0xFF,Cp);
			// movw %%cx,Ofs_CX(%%ebx)
			G4M(0x66,0x89,0x4b,Ofs_CX,Cp);
		    }
		}
		else {
		    if (mode&(MREP|MREPNE)) {
			// movl %%ecx,Ofs_ECX(%%ebx)
			G3M(0x89,0x4b,Ofs_ECX,Cp);
		    }
		    if (mode&MOVSSRC) {
			// esi = base1 + CPU_(e)SI +- n
			// subl %%[re]bp,%%[re]si
			Gen48(Cp); G2M(0x29,0xee,Cp);
			// subl (seg reg offset)(%%ebx),%%esi
			G2(0x732b,Cp); G1(IG->p0,Cp);
			// movl %%esi,Ofs_ESI(%%ebx)
			G3M(0x89,0x73,Ofs_ESI,Cp);
		    }
		    if (mode&MOVSDST) {
			// edi = base2 + CPU_(e)DI +- n
			// subl %%[re]bp,%%[re]di
			Gen48(Cp); G2M(0x29,0xef,Cp);
			// subl Ofs_XES(%%ebx),%%edi
			G3M(0x2b,0x7b,Ofs_XES,Cp);
			// movl %%edi,Ofs_EDI(%%ebx)
			G3M(0x89,0x7b,Ofs_EDI,Cp);
		    }
		}
		break;

	case O_SLAHF:
		rcod = IG->p0;		// 0=LAHF 1=SAHF
		if (rcod==0) {		/* LAHF */
			// movb 0(%%esp),%%al
			G3M(0x8a,0x04,0x24,Cp);
			// movb %%al,Ofs_AH(%%ebx)
			G3M(0x88,0x43,Ofs_AH,Cp);
		}
		else {			/* SAHF */
			// movb Ofs_AH(%%ebx),%%al
			G3M(0x8a,0x43,Ofs_AH,Cp);
			// orb $2,%%al
			G2M(0x0c,0x02,Cp);
			// movb %%al,0(%%esp)
			G3M(0x88,0x04,0x24,Cp);
		}
		break;
	case O_SETFL: {
		unsigned char o1 = IG->p0;
		switch(o1) {	// these are direct on x86
		case CMC:	// xorb $1,0(%%esp)
			G4M(0x80,0x34,0x24,0x01,Cp); break;
		case CLC:	// andb $0xfe,(%%esp)
			G4M(0x80,0x24,0x24,0xfeu,Cp); break;
		case STC:	// orb $1,0(%%esp)
			G4M(0x80,0x0c,0x24,0x01,Cp); break;
		case CLD:
			// andb $0xfb,EFLAGS+1(%%ebx)
			G4M(0x80,0x63,Ofs_EFLAGS+1,0xfbu,Cp);
			// movl $0x040201,DF_INCREMENTS(%%ebx)
			G3M(0xc7,0x43,Ofs_DF_INCREMENTS,Cp);
			G4(0x040201,Cp);
			break;
		case STD:
			// orb $4,EFLAGS+1(%%ebx)
			G4M(0x80,0x4b,Ofs_EFLAGS+1,0x04,Cp);
			// movl $0xfcfeff,DF_INCREMENTS(%%ebx)
			G3M(0xc7,0x43,Ofs_DF_INCREMENTS,Cp);
			G4(0xfcfeff,Cp);
			break;
		case CLI:
			// andb $0xfd,EFLAGS+1(%%ebx)
			G4M(0x80,0x63,Ofs_EFLAGS+1,~(uint8_t)(EFLAGS_IF>>8),Cp);
			break;
		case INT:
			// clear IF & TF
			// andb $0xfc,EFLAGS+1(%%ebx)
			G4M(0x80,0x63,Ofs_EFLAGS+1,
			    ~(uint8_t)((EFLAGS_IF|EFLAGS_TF)>>8),Cp);
			break;
		} }
		break;
	case O_BSWAP: {
		// movl offs(%%ebx),%%eax
		G3M(0x8b,0x43,IG->p0,Cp);
		// bswap %%eax
		G2M(0x0f,0xc8,Cp);
		// movl %%eax,offs(%%ebx)
		G3M(0x89,0x43,IG->p0,Cp);
		}
		break;
	case O_SETCC: {
		unsigned char n = IG->p0;
		PopPushF(Cp);	// get flags from stack
		// setcc %%al
		G3M(0x0f,(0x90|(n&15)),0xc0,Cp);
		}
		break;
	case O_BITOP: {
		unsigned char n = IG->p0;
		G1(0x9d,Cp);	// get flags from stack
		switch (n) {
		case 0x03: /* BT */
		case 0x0b: /* BTS */
		case 0x13: /* BTR */
		case 0x1b: /* BTC */
			if (mode&DATA16) {
				// movzwl offs(%%ebx),%%edx
				G4M(0x0f,0xb7,0x53,IG->p1,Cp);
			}
			else {
				// movl offs(%%ebx),%%edx
				G3M(0x8b,0x53,IG->p1,Cp);
			}
			if (!(mode & RM_REG)) {
				/* add bit offset to effective address */
				// movl %%edx, %%eax
				G2M(0x89,0xd0,Cp);
				if (mode&DATA16) {
					// shrl $4, %%eax
					G3M(0xc1,0xe8,0x04,Cp);
					// leal (%%edi,%%eax,2), %%edi
					G3M(0x8d,0x3c,0x47,Cp);
				} else {
					// shrl $5, %%eax
					G3M(0xc1,0xe8,0x05,Cp);
					// leal (%%edi,%%eax,4), %%edi
					G3M(0x8d,0x3c,0x87,Cp);
				}
				// mov (%%edi,%%ebp,1), %%{e}ax
				Gen66(mode,Cp); G3M(0x8b,0x04,0x2f,Cp);
			}
			// OP{wl} %%{e}dx,%%{e}ax
			Gen66(mode,Cp);	G3M(0x0f,(n+0xa0),0xd0,Cp);
			if (!(mode & RM_REG) && n != 0x03) {
				STD_WRITE_WL(mode);
			}
			break;
		case 0x1c: /* BSF */
		case 0x1d: /* BSR */
			// OP{wl} %%{e}ax,%%{e}dx
			Gen66(mode,Cp); G3M(0x0f,(n+0xa0),0xd0,Cp);
			// jz 1f
			G2M(0x74,(mode&DATA16)?0x04:0x03,Cp);
			// mov{wl} %%{e}dx,offs(%%ebx) 1:
			Gen66(mode,Cp); G3M(0x89,0x53,IG->p1,Cp);
			break;
		case 0x20: /* BT  imm8 */
		case 0x28: /* BTS imm8 */
		case 0x30: /* BTR imm8 */
		case 0x38: /* BTC imm8 */
			// OP{wl} $immed,%%{e}ax
			Gen66(mode,Cp);	G4M(0x0f,0xba,(n|0xc0),IG->p1,Cp);
			break;
		}
		G1(0x9c,Cp);	// flags back on stack
		} break;

	case O_SHFD: {
		unsigned char l_r = IG->p0;
		G1(0x9d,Cp);	// get flags from stack
		// mov{wl} offs(%%ebx),%%{e}dx
		Gen66(mode,Cp);	G3M(0x8b,0x53,IG->p1,Cp);
		if (mode & IMMED) {
			unsigned char shc = IG->p2;
			// sh{lr}d $immed,%%{e}dx,%%{e}ax
			Gen66(mode,Cp);	G4M(0x0f,(0xa4|l_r),0xd0,shc,Cp);
		}
		else {
			// movl Ofs_ECX(%%ebx),%%ecx
			G3M(0x8b,0x4b,Ofs_ECX,Cp);
			// sh{lr}d %%cl,%%{e}dx,%%{e}ax
			Gen66(mode,Cp);	G3M(0x0f,(0xa5|l_r),0xd0,Cp);
		}
		G1(0x9c,Cp);	// flags back on stack
		} break;

	case O_RDTSC: {
		// rdtsc
		G2(0x310f,Cp);
#if 0
		if (eTimeCorrect >= 0) {
			// movl	%%eax,%%ecx
			// movl	%%edx,%%edi
			G4(0xd789c189,Cp);
			// subl	TimeStartExec.t.tl(%%ebx),%%eax
			// sbbl	TimeStartExec.t.th(%%ebx),%%edx
			G2(0x832b,Cp);
			G4((unsigned char *)&TimeStartExec.t.tl-CPUOFFS(0),Cp);
			G2(0x931b,Cp);
			G4((unsigned char *)&TimeStartExec.t.th-CPUOFFS(0),Cp);
			// addl	TheCPU.EMUtime(%%ebx),%%eax
			// adcl	TheCPU.EMUtime+4(%%ebx),%%edx
			G3M(0x03,0x43,Ofs_ETIME,Cp);
			G3M(0x13,0x53,Ofs_ETIME+4,Cp);
			// movl	%%ecx,TimeStartExec.t.tl(%%ebx)
			// movl	%%edi,TimeStartExec.t.th(%%ebx)
			G2(0x8b89,Cp);
			G4((unsigned char *)&TimeStartExec.t.tl-CPUOFFS(0),Cp);
			G2(0xbb89,Cp);
			G4((unsigned char *)&TimeStartExec.t.th-CPUOFFS(0),Cp);
			// movl	%%eax,TheCPU.EMUtime(%%ebx)
			// movl	%%edx,TheCPU.EMUtime+4(%%ebx)
			G3M(0x89,0x43,Ofs_ETIME,Cp);
			G3M(0x89,0x53,Ofs_ETIME+4,Cp);
		}
#endif
		// movl %%eax,Ofs_EAX(%%ebx)
		// movl %%edx,Ofs_EDX(%%ebx)
		G3M(0x89,0x43,Ofs_EAX,Cp);
		G3M(0x89,0x53,Ofs_EDX,Cp);
		}
		break;

	case JMP_TAILCODE: {
		/* copy tail instructions to the end of the code block */
		unsigned char *p = Cp;
		GNX(Cp, TailCode, TAILSIZE);
		/* Keep TheCPU.eip for BreakNode */
		if (mode & MPATCH)
			p[5] = p[6] = p[7] = NOP;
		*((unsigned int *)(p + TAILFIX)) = IG->p0;
		}
		break;

	case JMP_INDIRECT: {	// input: %%{e}ax = %%{e}ip
		if (mode&DATA16)
			// movz{wl} %%ax,%%eax
			G3M(0x0f,0xb7,0xc0,Cp);
		// addl Ofs_XCS(%%ebx),%%eax
		G3M(0x03,0x43,Ofs_XCS,Cp);
		// movl %%eax, Ofs_KEY(%%ebx) // signals indirect
		G3M(0x89,0x43,Ofs_KEY,Cp);
#ifdef __x86_64__
		// movl %%eax,%%edi
		G2M(0x89,0xc7,Cp);
		// movl mode,%%esi
		G1(0xbe,Cp); G4(mode,Cp);
		// lea 3+2(%%rip), %%rdx // tailcode
		G3M(0x48,0x8d,0x15,Cp); G4(3+2,Cp);
#else
		// push address of tailcode
		G1(PUSHwi,Cp); G4((uintptr_t)GetExecCodeBuf(Cp+4+1+3+2+2),Cp);
		// push mode
		G1(PUSHwi,Cp); G4(mode,Cp);
		// push ePC
		G1(PUSHax,Cp);
#endif
		// call Ofs_JmpIndirect(%%ebx)
		G3M(0xff,0x53,Ofs_JmpIndirect(),Cp);
#ifndef __x86_64__
		// need to pop arguments for i386
		G3M(POPdx,POPdx,POPdx,Cp);
#endif
		// jump to next block if a compatible one exists
		// otherwise to the tail code
		// jmp *%rax
		G2M(0xff,0xe0,Cp);
		// movl Ofs_KEY(%%ebx),%%eax
		G3M(0x8b,0x43,Ofs_KEY,Cp);
		// pop %%edx; ret
		G2M(0x5a,0xc3,Cp);
		}
		break;

	case JMP_LINK: {	// dspt
		// check if signal check is present
		if ((mode & MPATCH) && Cp[0] == 0x0f) Cp += CKSIGNSIZE;
		if (mode & MLINK) {
		    ptrdiff_t ra;
		    // patch code to directly jump to the next block
		    // b8 [npc] -> e9/eb reladr
		    ra = IG->link - (Cp + TAILFIX);
#ifdef __x86_64__
		    if (ra - 4 < INT32_MIN || ra - 4 > INT32_MAX) {
			if (debug_level('e') > 1)
			    e_printf("Linker: 64 bit jmp for ra=0x%tx\n", ra);
			G2M(0x48,0xb8,Cp); // mov adr, %%rax
			G8((uintptr_t)IG->link,Cp);
			G2M(0xff,0xe0,Cp); // jmp %%rax
		    }
		    else
#endif
		    {
			if ((ra > -127) && (ra < 128)) {
			    ra -= 1; G1(JMPsid,Cp); G1(ra,Cp);
			}
			else {
			    ra -= 4; G1(JMPd,Cp); G4(ra,Cp);
			}
		    }
		    break;
		}
		int dspt = IG->p0;
		if (mode & CKSIGN) {
		    // check signal on TAKEN or non-TAKEN branch
		    // movzwl Ofs_EXITPEND(%%ebx),%%ecx
		    G4M(0x0f,0xb7,0x4b,Ofs_EXITPEND,Cp);
		    // jecxz {continue}: exit if exitpend not 0
		    G2M(0xe3,TAILSIZE,Cp);
		    // movl {exit_addr},%%eax; movl %%eax,Ofs_KEY(%%ebx)
		    G1(0xb8,Cp); G4(dspt,Cp); G3M(0x89,0x43,Ofs_KEY,Cp);
		    // pop %%edx; ret
		    G2M(0x5a,0xc3,Cp);
	        }
		// t:	b8 [exit_pc] 5a c3
		G1(0xb8,Cp);
		G4(dspt,Cp); G2(0xc35a,Cp);
		G5(0x9090909090,Cp); // padding for 64-bit jump
		if (debug_level('e')>2) e_printf("JMP_Link %08x %08x\n", dspt, IG->p1);
		}
		break;

	case JF_LINK:
	case JB_LINK: {		// opc, PC, dspt, dspnt, link
		unsigned char opc = IG->p0;
		int sz;
		//	JCXZ:	8b 4b Ofs_ECX e3 07 or 0f b7 4b Ofs_ECX e3 07
		//	JCC:	7x 07
		// nt:	b8 [nt_pc] 5a c3
		// t:	0f b7 4b [sig] e3 07
		//	b8 [t_pc] 5a c3
		//	b8 [t_pc] 5a c3 (can be patched by node linker)
		sz = JMPTAILSIZE + (mode & CKSIGN? CKSIGNSIZE:0);
		if (opc==JCXZ) {
			if (mode&ADDR16) {
			    // movzwl Ofs_ECX(%%ebx),%%ecx
			    G4M(0x0f,0xb7,0x4b,Ofs_ECX,Cp);
			}
			else {
			    // movl Ofs_ECX(%%ebx),%%ecx
			    G3M(0x8b,0x4b,Ofs_ECX,Cp);
			}
			G2M(0xe3,sz,Cp);	// jecxz
		}
		else {
			PopPushF(Cp);	// get flags from stack
			G2M(opc,sz,Cp);	// normal condition (Jcc)
		}
		// followed by not-taken (optional CKSIGN + JMPTAILSIZE bytes)
		// followed by taken (CKSIGN for JB_LINK + JMPTAILSIZE bytes)
		}
		break;

	case JLOOP_LINK: {	// opc, PC, dspt, dspnt, link
		unsigned char opc = IG->p0;
		//	{66} dec Ofs_ECX(ebx)
		//	LOOP:	jnz t
		//	LOOPZ:  jz  nt; test 0x40,dl; jnz t
		//	LOOPNZ: jz  nt; test 0x40,dl; jz  t
		// nt:	b8 [nt_pc] c3
		// t:	8b 0d [sig] jcxz t2
		//	{66} inc Ofs_ECX(ebx)
		//	b8 [sig_pc] c3
		// t2:	b8 [t_pc] c3
		if (mode&ADDR16) {
			G4M(OPERoverride,0xff,0x4b,Ofs_ECX,Cp);
		}
		else {
			G3M(0xff,0x4b,Ofs_ECX,Cp);
		}
		/*
		 * e2 LOOP   taken = (e)cx		nt=cxz
		 * e1 LOOPZ  taken = (e)cx &&  ZF	nt=cxz||!ZF
		 * e0 LOOPNZ taken = (e)cx && !ZF	nt=cxz|| ZF
		 */
		if (opc==LOOPZ_LOOPE) {
			G2M(0x74,0x06,Cp);		// jz->nt
			// test flags (on stack)
			G4M(0xf6,0x04,0x24,0x40,Cp);
			G2M(0x75,JMPTAILSIZE,Cp);	// jnz->t
		}
		else if (opc==LOOPNZ_LOOPNE) {
			G2M(0x74,0x06,Cp);		// jz->nt
			// test flags (on stack)
			G4M(0xf6,0x04,0x24,0x40,Cp);
			G2M(0x74,JMPTAILSIZE,Cp);	// jz->t
		}
		else {
			G2M(0x75,JMPTAILSIZE,Cp);	// jnz->t
		}
		// followed by not-taken (JMPTAILSIZE bytes)
		// followed by taken (JMPTAILSIZE bytes)
		}
		break;

	}

#ifdef USE_MHPDBG
	CpTemp = CodePtr;
	if (debug_level('e')>7) do {
		static char frmtbuf[256];
		int i;
		unsigned int ref;
		int rc = dis_8086((uintptr_t)CpTemp, frmtbuf, 5, &ref, 0);
		dbug_printf("%16p: ", CpTemp);
		for (i=0; i < 11; i++) {
			if (i < rc)
				dbug_printf("%02x", CpTemp[i]);
			else
				dbug_printf("%s", "  ");
		}
		dbug_printf("%s\n", frmtbuf);
		CpTemp += rc;
	} while (CpTemp < Cp);
#endif

#if PROFILE >= 2
	if (debug_level('e')) GenTime += (GETTSC() - t0);
#endif
	return Cp;
}

/////////////////////////////////////////////////////////////////////////////

/* stack frame for compiled code:
 * esp+00	TheCPU flags
 *     04/08	return address
 *     18/20...	locals of CloseAndExec
 */
#ifdef __x86_64__
#define RE_REG(r) "%%r"#r
#define R_REG(r) "%r"#r
/* Generated code calls C functions which clobber r8-r11 */
/* r12 is used to backup rsp */
#define EXEC_CLOBBERS ,"r8","r9","r10","r11","r12"
#else
#define RE_REG(r) "%%e"#r
#define R_REG(r) "%e"#r
#define EXEC_CLOBBERS
#endif
static unsigned Exec_x86(void *SeqStart)
{
	unsigned ePC;
	void *jb;

	/* get the protected mode flags. Note that RF and VM are cleared
	 * by pushfd (but not by ints and traps) */
	unsigned long flg = getflags();

	/* pass TF=0, IF=1, DF=0 */
	flg = (flg & ~(EFLAGS_CC|EFLAGS_IF|EFLAGS_DF|EFLAGS_TF)) |
	       (EFLAGS & EFLAGS_CC) | EFLAGS_IF;

	jb = jit_base;
	InCompiledCode = 1;
	asm volatile (
		/* After modifying RSP, we can only use reg-based args. */
		"push	"RE_REG(bp)"\n"
		"mov	%[mb], "RE_REG(bp)"\n"
#ifdef __x86_64__
		"movq	%%rsp,%%r12\n"
		"addq	$-128,%%rsp\n"	/* go below red zone		*/
		"andq	$~15,%%rsp\n"	/* 16-byte stack alignment	*/
#endif
		/* things like "push 1f" will create text reloc,
		 * so no go for pie. */
		"call	1f\n"		/* save return addr		*/
		"jmp	2f\n"
		"1:\n"
		"push	%4\n"
		"jmp	*%5\n"
		"2:\n"
#ifdef __x86_64__
		"movq	%%r12,%%rsp\n"
#endif
		"pop	"RE_REG(bp)"\n"
		: "=d"(flg),"=a"(ePC),"=D"(TheCPU.mem_ref)
		: "b"(CPUOFFS(0)),"d"(flg),"a"(GetExecCodeBuf(SeqStart)),
		  [mb]"D"(jb)  // don't use "r" or can assign to %rbp
		/* Note: we need to clobber "class-less" regs (like %rbp) even
		 * if we save/restore them, to avoid gcc from allocating
		 * them to "r". But in this case we don't need to save/restore.
		 *
		 * The above Note appears troublesome, as clobbering %rbp
		 * may require -fomit-frame-pointer, which is a PITA.
		 * The alternative is to not use "r" at all, use only "m".
		 * Unfortunately in this case we can't pass do_seq_start,
		 * at least on clang (doesn't support "m" for functions),
		 * so it needs to be inlined.
		 *
		 * The above didn't work too, as w/o -fomit-frame-pointer
		 * its quite difficult to tell gcc to not use %ebp
		 * for mem refs (static storage as in 12e00030 doesn't
		 * help with -fpic on i386). So the new strategy is to
		 * avoid both "r" and "m", and use push/pop for %ebp.
		 * The trick that should allow push to work, is to use
		 * fixed reg for mem_base and move it to %ebp after push,
		 * rather than using a mem-ref with modified %esp.
		 */
		: "memory", "cc", "ecx", "esi" EXEC_CLOBBERS
	);
	InCompiledCode = 0;
	/* even though InCompiledCode is volatile, we also need a barrier */
	asm volatile ("":::"memory");
	EFLAGS = (EFLAGS & ~EFLAGS_CC) | (flg &	EFLAGS_CC);
	return ePC;
}

/////////////////////////////////////////////////////////////////////////////
