x86_64 target support

git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@1197 c046a42c-6fe2-441c-8c8c-71466251a162

x86_64 target support
git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@1197 c046a42c-6fe2-441c-8c8c-71466251a162
bellard
1 parent c4687878
Showing 11 changed files with 3678 additions and 1300 deletions
target-i386/cpu.h
target-i386/exec.h
target-i386/helper.c
target-i386/helper2.c
target-i386/op.c
target-i386/opreg_template.h
target-i386/ops_mem.h
target-i386/ops_template.h
target-i386/ops_template_mem.h
target-i386/translate-copy.c
target-i386/translate.c
@@ -20,7 +20,13 @@
 #ifndef CPU_I386_H
 #define CPU_I386_H
  
+#include "config.h"
+
+#ifdef TARGET_X86_64
+#define TARGET_LONG_BITS 64
+#else
 #define TARGET_LONG_BITS 32
+#endif
  
 /* target supports implicit self modifying code */
 #define TARGET_HAS_SMC
@@ -63,6 +69,8 @@
 #define DESC_G_MASK     (1 << 23)
 #define DESC_B_SHIFT    22
 #define DESC_B_MASK     (1 << DESC_B_SHIFT)
+#define DESC_L_SHIFT    21 /* x86_64 only : 64 bit code segment */
+#define DESC_L_MASK     (1 << DESC_L_SHIFT)
 #define DESC_AVL_MASK   (1 << 20)
 #define DESC_P_MASK     (1 << 15)
 #define DESC_DPL_SHIFT  13
@@ -125,6 +133,8 @@
 #define HF_EM_SHIFT         10
 #define HF_TS_SHIFT         11
 #define HF_IOPL_SHIFT       12 /* must be same as eflags */
+#define HF_LMA_SHIFT        14 /* only used on x86_64: long mode active */
+#define HF_CS64_SHIFT       15 /* only used on x86_64: 64 bit code segment  */
 #define HF_VM_SHIFT         17 /* must be same as eflags */
  
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
@@ -138,6 +148,8 @@
 #define HF_MP_MASK           (1 << HF_MP_SHIFT)
 #define HF_EM_MASK           (1 << HF_EM_SHIFT)
 #define HF_TS_MASK           (1 << HF_TS_SHIFT)
+#define HF_LMA_MASK          (1 << HF_LMA_SHIFT)
+#define HF_CS64_MASK         (1 << HF_CS64_SHIFT)
  
 #define CR0_PE_MASK  (1 << 0)
 #define CR0_MP_MASK  (1 << 1)
@@ -156,6 +168,9 @@
 #define CR4_PSE_MASK  (1 << 4)
 #define CR4_PAE_MASK  (1 << 5)
 #define CR4_PGE_MASK  (1 << 7)
+#define CR4_PCE_MASK  (1 << 8)
+#define CR4_OSFXSR_MASK (1 << 9)
+#define CR4_OSXMMEXCPT_MASK  (1 << 10)
  
 #define PG_PRESENT_BIT	0
 #define PG_RW_BIT	1
@@ -193,6 +208,44 @@
 #define MSR_IA32_SYSENTER_ESP           0x175
 #define MSR_IA32_SYSENTER_EIP           0x176
  
+#define MSR_EFER                        0xc0000080
+
+#define MSR_EFER_SCE   (1 << 0)
+#define MSR_EFER_LME   (1 << 8)
+#define MSR_EFER_LMA   (1 << 10)
+#define MSR_EFER_NXE   (1 << 11)
+#define MSR_EFER_FFXSR (1 << 14)
+
+#define MSR_STAR                        0xc0000081
+#define MSR_LSTAR                       0xc0000082
+#define MSR_CSTAR                       0xc0000083
+#define MSR_FMASK                       0xc0000084
+#define MSR_FSBASE                      0xc0000100
+#define MSR_GSBASE                      0xc0000101
+#define MSR_KERNELGSBASE                0xc0000102
+
+/* cpuid_features bits */
+#define CPUID_FP87 (1 << 0)
+#define CPUID_VME  (1 << 1)
+#define CPUID_DE   (1 << 2)
+#define CPUID_PSE  (1 << 3)
+#define CPUID_TSC  (1 << 4)
+#define CPUID_MSR  (1 << 5)
+#define CPUID_PAE  (1 << 6)
+#define CPUID_MCE  (1 << 7)
+#define CPUID_CX8  (1 << 8)
+#define CPUID_APIC (1 << 9)
+#define CPUID_SEP  (1 << 11) /* sysenter/sysexit */
+#define CPUID_MTRR (1 << 12)
+#define CPUID_PGE  (1 << 13)
+#define CPUID_MCA  (1 << 14)
+#define CPUID_CMOV (1 << 15)
+/* ... */
+#define CPUID_MMX  (1 << 23)
+#define CPUID_FXSR (1 << 24)
+#define CPUID_SSE  (1 << 25)
+#define CPUID_SSE2 (1 << 26)
+
 #define EXCP00_DIVZ	0
 #define EXCP01_SSTP	1
 #define EXCP02_NMI	2
@@ -219,42 +272,52 @@ enum {
     CC_OP_MULB, /* modify all flags, C, O = (CC_SRC != 0) */
     CC_OP_MULW,
     CC_OP_MULL,
+    CC_OP_MULQ,
  
     CC_OP_ADDB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_ADDW,
     CC_OP_ADDL,
+    CC_OP_ADDQ,
  
     CC_OP_ADCB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_ADCW,
     CC_OP_ADCL,
+    CC_OP_ADCQ,
  
     CC_OP_SUBB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_SUBW,
     CC_OP_SUBL,
+    CC_OP_SUBQ,
  
     CC_OP_SBBB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_SBBW,
     CC_OP_SBBL,
+    CC_OP_SBBQ,
  
     CC_OP_LOGICB, /* modify all flags, CC_DST = res */
     CC_OP_LOGICW,
     CC_OP_LOGICL,
+    CC_OP_LOGICQ,
  
     CC_OP_INCB, /* modify all flags except, CC_DST = res, CC_SRC = C */
     CC_OP_INCW,
     CC_OP_INCL,
+    CC_OP_INCQ,
  
     CC_OP_DECB, /* modify all flags except, CC_DST = res, CC_SRC = C  */
     CC_OP_DECW,
     CC_OP_DECL,
+    CC_OP_DECQ,
  
     CC_OP_SHLB, /* modify all flags, CC_DST = res, CC_SRC.msb = C */
     CC_OP_SHLW,
     CC_OP_SHLL,
+    CC_OP_SHLQ,
  
     CC_OP_SARB, /* modify all flags, CC_DST = res, CC_SRC.lsb = C */
     CC_OP_SARW,
     CC_OP_SARL,
+    CC_OP_SARQ,
  
     CC_OP_NB,
 };
@@ -271,22 +334,42 @@ typedef double CPU86_LDouble;
  
 typedef struct SegmentCache {
     uint32_t selector;
-    uint8_t *base;
+    target_ulong base;
     uint32_t limit;
     uint32_t flags;
 } SegmentCache;
  
+typedef struct {
+    union {
+        uint8_t b[16];
+        uint16_t w[8];
+        uint32_t l[4];
+        uint64_t q[2];
+    } u;
+} XMMReg;
+
+#ifdef TARGET_X86_64
+#define CPU_NB_REGS 16
+#else
+#define CPU_NB_REGS 8
+#endif
+
 typedef struct CPUX86State {
+#if TARGET_LONG_BITS > HOST_LONG_BITS
+    /* temporaries if we cannot store them in host registers */
+    target_ulong t0, t1, t2;
+#endif
+
     /* standard registers */
-    uint32_t regs[8];
-    uint32_t eip;
-    uint32_t eflags; /* eflags register. During CPU emulation, CC
+    target_ulong regs[CPU_NB_REGS];
+    target_ulong eip;
+    target_ulong eflags; /* eflags register. During CPU emulation, CC
                         flags and DF are set to zero because they are
                         stored elsewhere */
  
     /* emulator internal eflags handling */
-    uint32_t cc_src;
-    uint32_t cc_dst;
+    target_ulong cc_src;
+    target_ulong cc_dst;
     uint32_t cc_op;
     int32_t df; /* D flag : 1 if D = 0, -1 if D = 1 */
     uint32_t hflags; /* hidden flags, see HF_xxx constants */
@@ -314,10 +397,21 @@ typedef struct CPUX86State {
     SegmentCache gdt; /* only base and limit are used */
     SegmentCache idt; /* only base and limit are used */
  
+    XMMReg xmm_regs[CPU_NB_REGS];
+    XMMReg xmm_t0;
+
     /* sysenter registers */
     uint32_t sysenter_cs;
     uint32_t sysenter_esp;
     uint32_t sysenter_eip;
+#ifdef TARGET_X86_64
+    target_ulong efer;
+    target_ulong star;
+    target_ulong lstar;
+    target_ulong cstar;
+    target_ulong fmask;
+    target_ulong kernelgsbase;
+#endif
  
     /* temporary data for USE_CODE_COPY mode */
 #ifdef USE_CODE_COPY
@@ -333,8 +427,8 @@ typedef struct CPUX86State {
     int exception_is_int;
     int exception_next_eip;
     struct TranslationBlock *current_tb; /* currently executing TB */
-    uint32_t cr[5]; /* NOTE: cr1 is unused */
-    uint32_t dr[8]; /* debug registers */
+    target_ulong cr[5]; /* NOTE: cr1 is unused */
+    target_ulong dr[8]; /* debug registers */
     int interrupt_request; 
     int user_mode_only; /* user mode only simulation */
  
@@ -346,18 +440,28 @@ typedef struct CPUX86State {
        context) */
     unsigned long mem_write_pc; /* host pc at which the memory was
                                    written */
-    unsigned long mem_write_vaddr; /* target virtual addr at which the
-                                      memory was written */
+    target_ulong mem_write_vaddr; /* target virtual addr at which the
+                                     memory was written */
     /* 0 = kernel, 1 = user */
     CPUTLBEntry tlb_read[2][CPU_TLB_SIZE];
     CPUTLBEntry tlb_write[2][CPU_TLB_SIZE];
  
     /* from this point: preserved by CPU reset */
     /* ice debug support */
-    uint32_t breakpoints[MAX_BREAKPOINTS];
+    target_ulong breakpoints[MAX_BREAKPOINTS];
     int nb_breakpoints;
     int singlestep_enabled;
  
+    /* processor features (e.g. for CPUID insn) */
+    uint32_t cpuid_vendor1;
+    uint32_t cpuid_vendor2;
+    uint32_t cpuid_vendor3;
+    uint32_t cpuid_version;
+    uint32_t cpuid_features;
+
+    /* in order to simplify APIC support, we leave this pointer to the
+       user */
+    struct APICState *apic_state;
     /* user data */
     void *opaque;
 } CPUX86State;
@@ -382,7 +486,7 @@ void cpu_set_ferr(CPUX86State *s);
    cache: it synchronizes the hflags with the segment cache values */
 static inline void cpu_x86_load_seg_cache(CPUX86State *env, 
                                           int seg_reg, unsigned int selector,
-                                          uint8_t *base, unsigned int limit, 
+                                          uint32_t base, unsigned int limit, 
                                           unsigned int flags)
 {
     SegmentCache *sc;
@@ -395,27 +499,45 @@ static inline void cpu_x86_load_seg_cache(CPUX86State *env,
     sc->flags = flags;
  
     /* update the hidden flags */
-    new_hflags = (env->segs[R_CS].flags & DESC_B_MASK)
-        >> (DESC_B_SHIFT - HF_CS32_SHIFT);
-    new_hflags |= (env->segs[R_SS].flags & DESC_B_MASK)
-        >> (DESC_B_SHIFT - HF_SS32_SHIFT);
-    if (!(env->cr[0] & CR0_PE_MASK) || 
-        (env->eflags & VM_MASK) ||
-        !(new_hflags & HF_CS32_MASK)) {
-        /* XXX: try to avoid this test. The problem comes from the
-           fact that is real mode or vm86 mode we only modify the
-           'base' and 'selector' fields of the segment cache to go
-           faster. A solution may be to force addseg to one in
-           translate-i386.c. */
-        new_hflags |= HF_ADDSEG_MASK;
-    } else {
-        new_hflags |= (((unsigned long)env->segs[R_DS].base | 
-                        (unsigned long)env->segs[R_ES].base |
-                        (unsigned long)env->segs[R_SS].base) != 0) << 
-            HF_ADDSEG_SHIFT;
+    {
+        if (seg_reg == R_CS) {
+#ifdef TARGET_X86_64
+            if ((env->hflags & HF_LMA_MASK) && (flags & DESC_L_MASK)) {
+                /* long mode */
+                env->hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
+                env->hflags &= ~(HF_ADDSEG_MASK);
+            } else 
+#endif
+            {
+                /* legacy / compatibility case */
+                new_hflags = (env->segs[R_CS].flags & DESC_B_MASK)
+                    >> (DESC_B_SHIFT - HF_CS32_SHIFT);
+                env->hflags = (env->hflags & ~(HF_CS32_MASK | HF_CS64_MASK)) |
+                    new_hflags;
+            }
+        }
+        new_hflags = (env->segs[R_SS].flags & DESC_B_MASK)
+            >> (DESC_B_SHIFT - HF_SS32_SHIFT);
+        if (env->hflags & HF_CS64_MASK) {
+            /* zero base assumed for DS, ES and SS in long mode */
+        } else if (!(env->cr[0] & CR0_PE_MASK) || 
+            (env->eflags & VM_MASK) ||
+            !(new_hflags & HF_CS32_MASK)) {
+            /* XXX: try to avoid this test. The problem comes from the
+               fact that is real mode or vm86 mode we only modify the
+               'base' and 'selector' fields of the segment cache to go
+               faster. A solution may be to force addseg to one in
+               translate-i386.c. */
+            new_hflags |= HF_ADDSEG_MASK;
+        } else {
+            new_hflags |= (((unsigned long)env->segs[R_DS].base | 
+                            (unsigned long)env->segs[R_ES].base |
+                            (unsigned long)env->segs[R_SS].base) != 0) << 
+                HF_ADDSEG_SHIFT;
+        }
+        env->hflags = (env->hflags & 
+                       ~(HF_SS32_MASK | HF_ADDSEG_MASK)) | new_hflags;
     }
-    env->hflags = (env->hflags & 
-                   ~(HF_CS32_MASK | HF_SS32_MASK | HF_ADDSEG_MASK)) | new_hflags;
 }
  
 /* wrapper, just in case memory mappings must be changed */
@@ -448,6 +570,9 @@ void cpu_x86_set_a20(CPUX86State *env, int a20_state);
  
 uint64_t cpu_get_tsc(CPUX86State *env);
  
+void cpu_set_apic_base(CPUX86State *env, uint64_t val);
+uint64_t cpu_get_apic_base(CPUX86State *env);
+
 /* will be suppressed */
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
  
@@ -20,14 +20,29 @@
 #include "config.h"
 #include "dyngen-exec.h"
  
+/* XXX: factorize this mess */
+#if defined(__alpha__) || defined (__ia64__) || defined(__x86_64__)
+#define HOST_LONG_BITS 64
+#else
+#define HOST_LONG_BITS 32
+#endif
+
+#ifdef TARGET_X86_64
+#define TARGET_LONG_BITS 64
+#else
+#define TARGET_LONG_BITS 32
+#endif
+
 /* at least 4 register variables are defined */
 register struct CPUX86State *env asm(AREG0);
+
+/* XXX: use 64 bit regs if HOST_LONG_BITS == 64 */
+#if TARGET_LONG_BITS == 32
+
 register uint32_t T0 asm(AREG1);
 register uint32_t T1 asm(AREG2);
 register uint32_t T2 asm(AREG3);
  
-#define A0 T2
-
 /* if more registers are available, we define some registers too */
 #ifdef AREG4
 register uint32_t EAX asm(AREG4);
@@ -69,6 +84,17 @@ register uint32_t EDI asm(AREG11);
 #define reg_EDI
 #endif
  
+#else
+
+/* no registers can be used */
+#define T0 (env->t0)
+#define T1 (env->t1)
+#define T2 (env->t2)
+
+#endif
+
+#define A0 T2
+
 extern FILE *logfile;
 extern int loglevel;
  
@@ -136,26 +162,24 @@ void helper_movl_crN_T0(int reg);
 void helper_movl_drN_T0(int reg);
 void helper_invlpg(unsigned int addr);
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
-void cpu_x86_update_cr3(CPUX86State *env, uint32_t new_cr3);
+void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
 void cpu_x86_flush_tlb(CPUX86State *env, uint32_t addr);
-int cpu_x86_handle_mmu_fault(CPUX86State *env, uint32_t addr, 
+int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr, 
                              int is_write, int is_user, int is_softmmu);
-void tlb_fill(unsigned long addr, int is_write, int is_user, 
+void tlb_fill(target_ulong addr, int is_write, int is_user, 
               void *retaddr);
 void __hidden cpu_lock(void);
 void __hidden cpu_unlock(void);
 void do_interrupt(int intno, int is_int, int error_code, 
-                  unsigned int next_eip, int is_hw);
+                  target_ulong next_eip, int is_hw);
 void do_interrupt_user(int intno, int is_int, int error_code, 
-                       unsigned int next_eip);
+                       target_ulong next_eip);
 void raise_interrupt(int intno, int is_int, int error_code, 
                      unsigned int next_eip);
 void raise_exception_err(int exception_index, int error_code);
 void raise_exception(int exception_index);
 void __hidden cpu_loop_exit(void);
-void helper_fsave(uint8_t *ptr, int data32);
-void helper_frstor(uint8_t *ptr, int data32);
  
 void OPPROTO op_movl_eflags_T0(void);
 void OPPROTO op_movl_T0_eflags(void);
@@ -163,13 +187,20 @@ void raise_interrupt(int intno, int is_int, int error_code,
                      unsigned int next_eip);
 void raise_exception_err(int exception_index, int error_code);
 void raise_exception(int exception_index);
-void helper_divl_EAX_T0(uint32_t eip);
-void helper_idivl_EAX_T0(uint32_t eip);
+void helper_divl_EAX_T0(void);
+void helper_idivl_EAX_T0(void);
+void helper_mulq_EAX_T0(void);
+void helper_imulq_EAX_T0(void);
+void helper_imulq_T0_T1(void);
+void helper_divq_EAX_T0(void);
+void helper_idivq_EAX_T0(void);
 void helper_cmpxchg8b(void);
 void helper_cpuid(void);
 void helper_enter_level(int level, int data32);
 void helper_sysenter(void);
 void helper_sysexit(void);
+void helper_syscall(void);
+void helper_sysret(int dflag);
 void helper_rdtsc(void);
 void helper_rdmsr(void);
 void helper_wrmsr(void);
@@ -252,7 +283,7 @@ void check_iol_DX(void);
 #define stl(p, v) stl_data(p, v)
 #define stq(p, v) stq_data(p, v)
  
-static inline double ldfq(void *ptr)
+static inline double ldfq(target_ulong ptr)
 {
     union {
         double d;
@@ -262,7 +293,7 @@ static inline double ldfq(void *ptr)
     return u.d;
 }
  
-static inline void stfq(void *ptr, double v)
+static inline void stfq(target_ulong ptr, double v)
 {
     union {
         double d;
@@ -272,7 +303,7 @@ static inline void stfq(void *ptr, double v)
     stq(ptr, u.i);
 }
  
-static inline float ldfl(void *ptr)
+static inline float ldfl(target_ulong ptr)
 {
     union {
         float f;
@@ -282,7 +313,7 @@ static inline float ldfl(void *ptr)
     return u.f;
 }
  
-static inline void stfl(void *ptr, float v)
+static inline void stfl(target_ulong ptr, float v)
 {
     union {
         float f;
@@ -411,7 +442,7 @@ static inline void fpop(void)
 }
  
 #ifndef USE_X86LDOUBLE
-static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
+static inline CPU86_LDouble helper_fldt(target_ulong ptr)
 {
     CPU86_LDoubleU temp;
     int upper, e;
@@ -451,12 +482,12 @@ static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
  
 #ifdef CONFIG_USER_ONLY
  
-static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
+static inline CPU86_LDouble helper_fldt(target_ulong ptr)
 {
     return *(CPU86_LDouble *)ptr;
 }
  
-static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
+static inline void helper_fstt(CPU86_LDouble f, target_ulong ptr)
 {
     *(CPU86_LDouble *)ptr = f;
 }
@@ -465,7 +496,7 @@ static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
  
 /* we use memory access macros */
  
-static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
+static inline CPU86_LDouble helper_fldt(target_ulong ptr)
 {
     CPU86_LDoubleU temp;
  
@@ -474,7 +505,7 @@ static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
     return temp.d;
 }
  
-static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
+static inline void helper_fstt(CPU86_LDouble f, target_ulong ptr)
 {
     CPU86_LDoubleU temp;
  
@@ -522,10 +553,12 @@ void helper_fscale(void);
 void helper_fsin(void);
 void helper_fcos(void);
 void helper_fxam_ST0(void);
-void helper_fstenv(uint8_t *ptr, int data32);
-void helper_fldenv(uint8_t *ptr, int data32);
-void helper_fsave(uint8_t *ptr, int data32);
-void helper_frstor(uint8_t *ptr, int data32);
+void helper_fstenv(target_ulong ptr, int data32);
+void helper_fldenv(target_ulong ptr, int data32);
+void helper_fsave(target_ulong ptr, int data32);
+void helper_frstor(target_ulong ptr, int data32);
+void helper_fxsave(target_ulong ptr, int data64);
+void helper_fxrstor(target_ulong ptr, int data64);
 void restore_native_fp_state(CPUState *env);
 void save_native_fp_state(CPUState *env);
  
@@ -119,7 +119,7 @@ static inline int load_segment(uint32_t *e1_ptr, uint32_t *e2_ptr,
 {
     SegmentCache *dt;
     int index;
-    uint8_t *ptr;
+    target_ulong ptr;
  
     if (selector & 0x4)
         dt = &env->ldt;
@@ -143,9 +143,9 @@ static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
     return limit;
 }
  
-static inline uint8_t *get_seg_base(uint32_t e1, uint32_t e2)
+static inline uint32_t get_seg_base(uint32_t e1, uint32_t e2)
 {
-    return (uint8_t *)((e1 >> 16) | ((e2 & 0xff) << 16) | (e2 & 0xff000000));
+    return ((e1 >> 16) | ((e2 & 0xff) << 16) | (e2 & 0xff000000));
 }
  
 static inline void load_seg_cache_raw_dt(SegmentCache *sc, uint32_t e1, uint32_t e2)
@@ -160,7 +160,7 @@ static inline void load_seg_vm(int seg, int selector)
 {
     selector &= 0xffff;
     cpu_x86_load_seg_cache(env, seg, selector, 
-                           (uint8_t *)(selector << 4), 0xffff, 0);
+                           (selector << 4), 0xffff, 0);
 }
  
 static inline void get_ss_esp_from_tss(uint32_t *ss_ptr, 
@@ -258,13 +258,13 @@ static void switch_tss(int tss_selector,
                        uint32_t next_eip)
 {
     int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;
-    uint8_t *tss_base;
+    target_ulong tss_base;
     uint32_t new_regs[8], new_segs[6];
     uint32_t new_eflags, new_eip, new_cr3, new_ldt, new_trap;
     uint32_t old_eflags, eflags_mask;
     SegmentCache *dt;
     int index;
-    uint8_t *ptr;
+    target_ulong ptr;
  
     type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
 #ifdef DEBUG_PCALL
@@ -345,7 +345,7 @@ static void switch_tss(int tss_selector,
  
     /* clear busy bit (it is restartable) */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_IRET) {
-        uint8_t *ptr;
+        target_ulong ptr;
         uint32_t e2;
         ptr = env->gdt.base + (env->tr.selector & ~7);
         e2 = ldl_kernel(ptr + 4);
@@ -397,7 +397,7 @@ static void switch_tss(int tss_selector,
  
     /* set busy bit */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_CALL) {
-        uint8_t *ptr;
+        target_ulong ptr;
         uint32_t e2;
         ptr = env->gdt.base + (tss_selector & ~7);
         e2 = ldl_kernel(ptr + 4);
@@ -445,11 +445,11 @@ static void switch_tss(int tss_selector,
         cpu_x86_set_cpl(env, new_segs[R_CS] & 3);
         /* first just selectors as the rest may trigger exceptions */
         for(i = 0; i < 6; i++)
-            cpu_x86_load_seg_cache(env, i, new_segs[i], NULL, 0, 0);
+            cpu_x86_load_seg_cache(env, i, new_segs[i], 0, 0, 0);
     }
  
     env->ldt.selector = new_ldt & ~4;
-    env->ldt.base = NULL;
+    env->ldt.base = 0;
     env->ldt.limit = 0;
     env->ldt.flags = 0;
  
@@ -573,7 +573,7 @@ static inline unsigned int get_sp_mask(unsigned int e2)
  
 #define POPL(ssp, sp, sp_mask, val)\
 {\
-    val = ldl_kernel((ssp) + (sp & (sp_mask)));\
+    val = (uint32_t)ldl_kernel((ssp) + (sp & (sp_mask)));\
     sp += 4;\
 }
  
@@ -582,7 +582,7 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
                                    unsigned int next_eip, int is_hw)
 {
     SegmentCache *dt;
-    uint8_t *ptr, *ssp;
+    target_ulong ptr, ssp;
     int type, dpl, selector, ss_dpl, cpl, sp_mask;
     int has_error_code, new_stack, shift;
     uint32_t e1, e2, offset, ss, esp, ss_e1, ss_e2;
@@ -703,7 +703,7 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
         raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         new_stack = 0; /* avoid warning */
         sp_mask = 0; /* avoid warning */
-        ssp = NULL; /* avoid warning */
+        ssp = 0; /* avoid warning */
         esp = 0; /* avoid warning */
     }
  
@@ -754,10 +754,10 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
  
     if (new_stack) {
         if (env->eflags & VM_MASK) {
-            cpu_x86_load_seg_cache(env, R_ES, 0, NULL, 0, 0);
-            cpu_x86_load_seg_cache(env, R_DS, 0, NULL, 0, 0);
-            cpu_x86_load_seg_cache(env, R_FS, 0, NULL, 0, 0);
-            cpu_x86_load_seg_cache(env, R_GS, 0, NULL, 0, 0);
+            cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0, 0);
+            cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0, 0);
+            cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0, 0);
+            cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0, 0);
         }
         ss = (ss & ~3) | dpl;
         cpu_x86_load_seg_cache(env, R_SS, ss, 
@@ -780,12 +780,264 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
     env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);
 }
  
+#ifdef TARGET_X86_64
+
+#define PUSHQ(sp, val)\
+{\
+    sp -= 8;\
+    stq_kernel(sp, (val));\
+}
+
+#define POPQ(sp, val)\
+{\
+    val = ldq_kernel(sp);\
+    sp += 8;\
+}
+
+static inline target_ulong get_rsp_from_tss(int level)
+{
+    int index;
+    
+#if 0
+    printf("TR: base=" TARGET_FMT_lx " limit=%x\n", 
+           env->tr.base, env->tr.limit);
+#endif
+
+    if (!(env->tr.flags & DESC_P_MASK))
+        cpu_abort(env, "invalid tss");
+    index = 8 * level + 4;
+    if ((index + 7) > env->tr.limit)
+        raise_exception_err(EXCP0A_TSS, env->tr.selector & 0xfffc);
+    return ldq_kernel(env->tr.base + index);
+}
+
+/* 64 bit interrupt */
+static void do_interrupt64(int intno, int is_int, int error_code,
+                           target_ulong next_eip, int is_hw)
+{
+    SegmentCache *dt;
+    target_ulong ptr;
+    int type, dpl, selector, cpl, ist;
+    int has_error_code, new_stack;
+    uint32_t e1, e2, e3, ss;
+    target_ulong old_eip, esp, offset;
+
+    has_error_code = 0;
+    if (!is_int && !is_hw) {
+        switch(intno) {
+        case 8:
+        case 10:
+        case 11:
+        case 12:
+        case 13:
+        case 14:
+        case 17:
+            has_error_code = 1;
+            break;
+        }
+    }
+    if (is_int)
+        old_eip = next_eip;
+    else
+        old_eip = env->eip;
+
+    dt = &env->idt;
+    if (intno * 16 + 15 > dt->limit)
+        raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
+    ptr = dt->base + intno * 16;
+    e1 = ldl_kernel(ptr);
+    e2 = ldl_kernel(ptr + 4);
+    e3 = ldl_kernel(ptr + 8);
+    /* check gate type */
+    type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
+    switch(type) {
+    case 14: /* 386 interrupt gate */
+    case 15: /* 386 trap gate */
+        break;
+    default:
+        raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
+        break;
+    }
+    dpl = (e2 >> DESC_DPL_SHIFT) & 3;
+    cpl = env->hflags & HF_CPL_MASK;
+    /* check privledge if software int */
+    if (is_int && dpl < cpl)
+        raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
+    /* check valid bit */
+    if (!(e2 & DESC_P_MASK))
+        raise_exception_err(EXCP0B_NOSEG, intno * 16 + 2);
+    selector = e1 >> 16;
+    offset = ((target_ulong)e3 << 32) | (e2 & 0xffff0000) | (e1 & 0x0000ffff);
+    ist = e2 & 7;
+    if ((selector & 0xfffc) == 0)
+        raise_exception_err(EXCP0D_GPF, 0);
+
+    if (load_segment(&e1, &e2, selector) != 0)
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK)))
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    dpl = (e2 >> DESC_DPL_SHIFT) & 3;
+    if (dpl > cpl)
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    if (!(e2 & DESC_P_MASK))
+        raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
+    if (!(e2 & DESC_L_MASK) || (e2 & DESC_B_MASK))
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    if ((!(e2 & DESC_C_MASK) && dpl < cpl) || ist != 0) {
+        /* to inner priviledge */
+        if (ist != 0)
+            esp = get_rsp_from_tss(ist + 3);
+        else
+            esp = get_rsp_from_tss(dpl);
+        ss = 0;
+        new_stack = 1;
+    } else if ((e2 & DESC_C_MASK) || dpl == cpl) {
+        /* to same priviledge */
+        if (env->eflags & VM_MASK)
+            raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+        new_stack = 0;
+        esp = ESP & ~0xf; /* align stack */
+        dpl = cpl;
+    } else {
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+        new_stack = 0; /* avoid warning */
+        esp = 0; /* avoid warning */
+    }
+
+    PUSHQ(esp, env->segs[R_SS].selector);
+    PUSHQ(esp, ESP);
+    PUSHQ(esp, compute_eflags());
+    PUSHQ(esp, env->segs[R_CS].selector);
+    PUSHQ(esp, old_eip);
+    if (has_error_code) {
+        PUSHQ(esp, error_code);
+    }
+    
+    if (new_stack) {
+        ss = 0 | dpl;
+        cpu_x86_load_seg_cache(env, R_SS, ss, 0, 0, 0);
+    }
+    ESP = esp;
+
+    selector = (selector & ~3) | dpl;
+    cpu_x86_load_seg_cache(env, R_CS, selector, 
+                   get_seg_base(e1, e2),
+                   get_seg_limit(e1, e2),
+                   e2);
+    cpu_x86_set_cpl(env, dpl);
+    env->eip = offset;
+
+    /* interrupt gate clear IF mask */
+    if ((type & 1) == 0) {
+        env->eflags &= ~IF_MASK;
+    }
+    env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);
+}
+
+void helper_syscall(void)
+{
+    int selector;
+
+    if (!(env->efer & MSR_EFER_SCE)) {
+        raise_exception_err(EXCP06_ILLOP, 0);
+    }
+    selector = (env->star >> 32) & 0xffff;
+    if (env->hflags & HF_LMA_MASK) {
+        ECX = env->eip;
+        env->regs[11] = compute_eflags();
+
+        cpu_x86_set_cpl(env, 0);
+        cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc, 
+                           0, 0xffffffff, 
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK | DESC_L_MASK);
+        cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_W_MASK | DESC_A_MASK);
+        env->eflags &= ~env->fmask;
+        if (env->hflags & HF_CS64_MASK)
+            env->eip = env->lstar;
+        else
+            env->eip = env->cstar;
+    } else {
+        ECX = (uint32_t)env->eip;
+        
+        cpu_x86_set_cpl(env, 0);
+        cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc, 
+                           0, 0xffffffff, 
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
+        cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_W_MASK | DESC_A_MASK);
+        env->eflags &= ~(IF_MASK | RF_MASK | VM_MASK);
+        env->eip = (uint32_t)env->star;
+    }
+}
+
+void helper_sysret(int dflag)
+{
+    int cpl, selector;
+
+    cpl = env->hflags & HF_CPL_MASK;
+    if (!(env->cr[0] & CR0_PE_MASK) || cpl != 0) {
+        raise_exception_err(EXCP0D_GPF, 0);
+    }
+    selector = (env->star >> 48) & 0xffff;
+    if (env->hflags & HF_LMA_MASK) {
+        if (dflag == 2) {
+            cpu_x86_load_seg_cache(env, R_CS, (selector + 16) | 3, 
+                                   0, 0xffffffff, 
+                                   DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                                   DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                                   DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK | 
+                                   DESC_L_MASK);
+            env->eip = ECX;
+        } else {
+            cpu_x86_load_seg_cache(env, R_CS, selector | 3, 
+                                   0, 0xffffffff, 
+                                   DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                                   DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                                   DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
+            env->eip = (uint32_t)ECX;
+        }
+        cpu_x86_load_seg_cache(env, R_SS, selector + 8, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                               DESC_W_MASK | DESC_A_MASK);
+        load_eflags((uint32_t)(env->regs[11]), 0xffffffff);
+        cpu_x86_set_cpl(env, 3);
+    } else {
+        cpu_x86_load_seg_cache(env, R_CS, selector | 3, 
+                               0, 0xffffffff, 
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                               DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
+        env->eip = (uint32_t)ECX;
+        cpu_x86_load_seg_cache(env, R_SS, selector + 8, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                               DESC_W_MASK | DESC_A_MASK);
+        env->eflags |= IF_MASK;
+        cpu_x86_set_cpl(env, 3);
+    }
+}
+#endif
+
 /* real mode interrupt */
 static void do_interrupt_real(int intno, int is_int, int error_code,
                               unsigned int next_eip)
 {
     SegmentCache *dt;
-    uint8_t *ptr, *ssp;
+    target_ulong ptr, ssp;
     int selector;
     uint32_t offset, esp;
     uint32_t old_cs, old_eip;
@@ -813,16 +1065,16 @@ static void do_interrupt_real(int intno, int is_int, int error_code,
     ESP = (ESP & ~0xffff) | (esp & 0xffff);
     env->eip = offset;
     env->segs[R_CS].selector = selector;
-    env->segs[R_CS].base = (uint8_t *)(selector << 4);
+    env->segs[R_CS].base = (selector << 4);
     env->eflags &= ~(IF_MASK | TF_MASK | AC_MASK | RF_MASK);
 }
  
 /* fake user mode interrupt */
 void do_interrupt_user(int intno, int is_int, int error_code, 
-                       unsigned int next_eip)
+                       target_ulong next_eip)
 {
     SegmentCache *dt;
-    uint8_t *ptr;
+    target_ulong ptr;
     int dpl, cpl;
     uint32_t e2;
  
@@ -849,26 +1101,26 @@ void do_interrupt_user(int intno, int is_int, int error_code,
  * instruction. It is only relevant if is_int is TRUE.  
  */
 void do_interrupt(int intno, int is_int, int error_code, 
-                  unsigned int next_eip, int is_hw)
+                  target_ulong next_eip, int is_hw)
 {
 #ifdef DEBUG_PCALL
     if (loglevel & (CPU_LOG_PCALL | CPU_LOG_INT)) {
         if ((env->cr[0] & CR0_PE_MASK)) {
             static int count;
-            fprintf(logfile, "%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:%08x pc=%08x SP=%04x:%08x",
+            fprintf(logfile, "%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:" TARGET_FMT_lx " pc=" TARGET_FMT_lx " SP=%04x:" TARGET_FMT_lx,
                     count, intno, error_code, is_int,
                     env->hflags & HF_CPL_MASK,
                     env->segs[R_CS].selector, EIP,
                     (int)env->segs[R_CS].base + EIP,
                     env->segs[R_SS].selector, ESP);
             if (intno == 0x0e) {
-                fprintf(logfile, " CR2=%08x", env->cr[2]);
+                fprintf(logfile, " CR2=" TARGET_FMT_lx, env->cr[2]);
             } else {
-                fprintf(logfile, " EAX=%08x", EAX);
+                fprintf(logfile, " EAX=" TARGET_FMT_lx, EAX);
             }
             fprintf(logfile, "\n");
-#if 0
             cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
+#if 0
             {
                 int i;
                 uint8_t *ptr;
@@ -885,7 +1137,14 @@ void do_interrupt(int intno, int is_int, int error_code,
     }
 #endif
     if (env->cr[0] & CR0_PE_MASK) {
-        do_interrupt_protected(intno, is_int, error_code, next_eip, is_hw);
+#if TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            do_interrupt64(intno, is_int, error_code, next_eip, is_hw);
+        } else
+#endif
+        {
+            do_interrupt_protected(intno, is_int, error_code, next_eip, is_hw);
+        }
     } else {
         do_interrupt_real(intno, is_int, error_code, next_eip);
     }
@@ -932,20 +1191,20 @@ void raise_exception(int exception_index)
 #ifdef BUGGY_GCC_DIV64
 /* gcc 2.95.4 on PowerPC does not seem to like using __udivdi3, so we
    call it from another function */
-uint32_t div64(uint32_t *q_ptr, uint64_t num, uint32_t den)
+uint32_t div32(uint32_t *q_ptr, uint64_t num, uint32_t den)
 {
     *q_ptr = num / den;
     return num % den;
 }
  
-int32_t idiv64(int32_t *q_ptr, int64_t num, int32_t den)
+int32_t idiv32(int32_t *q_ptr, int64_t num, int32_t den)
 {
     *q_ptr = num / den;
     return num % den;
 }
 #endif
  
-void helper_divl_EAX_T0(uint32_t eip)
+void helper_divl_EAX_T0(void)
 {
     unsigned int den, q, r;
     uint64_t num;
@@ -953,20 +1212,19 @@ void helper_divl_EAX_T0(uint32_t eip)
     num = EAX | ((uint64_t)EDX << 32);
     den = T0;
     if (den == 0) {
-        EIP = eip;
         raise_exception(EXCP00_DIVZ);
     }
 #ifdef BUGGY_GCC_DIV64
-    r = div64(&q, num, den);
+    r = div32(&q, num, den);
 #else
     q = (num / den);
     r = (num % den);
 #endif
-    EAX = q;
-    EDX = r;
+    EAX = (uint32_t)q;
+    EDX = (uint32_t)r;
 }
  
-void helper_idivl_EAX_T0(uint32_t eip)
+void helper_idivl_EAX_T0(void)
 {
     int den, q, r;
     int64_t num;
@@ -974,17 +1232,16 @@ void helper_idivl_EAX_T0(uint32_t eip)
     num = EAX | ((uint64_t)EDX << 32);
     den = T0;
     if (den == 0) {
-        EIP = eip;
         raise_exception(EXCP00_DIVZ);
     }
 #ifdef BUGGY_GCC_DIV64
-    r = idiv64(&q, num, den);
+    r = idiv32(&q, num, den);
 #else
     q = (num / den);
     r = (num % den);
 #endif
-    EAX = q;
-    EDX = r;
+    EAX = (uint32_t)q;
+    EDX = (uint32_t)r;
 }
  
 void helper_cmpxchg8b(void)
@@ -993,9 +1250,9 @@ void helper_cmpxchg8b(void)
     int eflags;
  
     eflags = cc_table[CC_OP].compute_all();
-    d = ldq((uint8_t *)A0);
+    d = ldq(A0);
     if (d == (((uint64_t)EDX << 32) | EAX)) {
-        stq((uint8_t *)A0, ((uint64_t)ECX << 32) | EBX);
+        stq(A0, ((uint64_t)ECX << 32) | EBX);
         eflags |= CC_Z;
     } else {
         EDX = d >> 32;
@@ -1005,58 +1262,20 @@ void helper_cmpxchg8b(void)
     CC_SRC = eflags;
 }
  
-#define CPUID_FP87 (1 << 0)
-#define CPUID_VME  (1 << 1)
-#define CPUID_DE   (1 << 2)
-#define CPUID_PSE  (1 << 3)
-#define CPUID_TSC  (1 << 4)
-#define CPUID_MSR  (1 << 5)
-#define CPUID_PAE  (1 << 6)
-#define CPUID_MCE  (1 << 7)
-#define CPUID_CX8  (1 << 8)
-#define CPUID_APIC (1 << 9)
-#define CPUID_SEP  (1 << 11) /* sysenter/sysexit */
-#define CPUID_MTRR (1 << 12)
-#define CPUID_PGE  (1 << 13)
-#define CPUID_MCA  (1 << 14)
-#define CPUID_CMOV (1 << 15)
-/* ... */
-#define CPUID_MMX  (1 << 23)
-#define CPUID_FXSR (1 << 24)
-#define CPUID_SSE  (1 << 25)
-#define CPUID_SSE2 (1 << 26)
-
 void helper_cpuid(void)
 {
-    switch(EAX) {
+    switch((uint32_t)EAX) {
     case 0:
         EAX = 2; /* max EAX index supported */
-        EBX = 0x756e6547;
-        ECX = 0x6c65746e;
-        EDX = 0x49656e69;
+        EBX = env->cpuid_vendor1;
+        EDX = env->cpuid_vendor2;
+        ECX = env->cpuid_vendor3;
         break;
     case 1:
-        {
-            int family, model, stepping;
-            /* EAX = 1 info */
-#if 0
-            /* pentium 75-200 */
-            family = 5;
-            model = 2;
-            stepping = 11;
-#else
-            /* pentium pro */
-            family = 6;
-            model = 1;
-            stepping = 3;
-#endif
-            EAX = (family << 8) | (model << 4) | stepping;
-            EBX = 0;
-            ECX = 0;
-            EDX = CPUID_FP87 | CPUID_DE | CPUID_PSE |
-                CPUID_TSC | CPUID_MSR | CPUID_MCE |
-                CPUID_CX8 | CPUID_PGE | CPUID_CMOV;
-        }
+        EAX = env->cpuid_version;
+        EBX = 0;
+        ECX = 0;
+        EDX = env->cpuid_features;
         break;
     default:
         /* cache info: needed for Pentium Pro compatibility */
@@ -1065,12 +1284,34 @@ void helper_cpuid(void)
         ECX = 0;
         EDX = 0;
         break;
+#ifdef TARGET_X86_64
+    case 0x80000000:
+        EAX = 0x80000008;
+        EBX = env->cpuid_vendor1;
+        EDX = env->cpuid_vendor2;
+        ECX = env->cpuid_vendor3;
+        break;
+    case 0x80000001:
+        EAX = env->cpuid_features;
+        EBX = 0;
+        ECX = 0;
+        /* long mode + syscall/sysret features */
+        EDX = (env->cpuid_features & 0x0183F3FF) | (1 << 29) | (1 << 11);
+        break;
+    case 0x80000008:
+        /* virtual & phys address size in low 2 bytes. */
+        EAX = 0x00003028;
+        EBX = 0;
+        ECX = 0;
+        EDX = 0;
+        break;
+#endif
     }
 }
  
 void helper_enter_level(int level, int data32)
 {
-    uint8_t *ssp;
+    target_ulong ssp;
     uint32_t esp_mask, esp, ebp;
  
     esp_mask = get_sp_mask(env->segs[R_SS].flags);
@@ -1105,20 +1346,26 @@ void helper_lldt_T0(void)
     int selector;
     SegmentCache *dt;
     uint32_t e1, e2;
-    int index;
-    uint8_t *ptr;
+    int index, entry_limit;
+    target_ulong ptr;
  
     selector = T0 & 0xffff;
     if ((selector & 0xfffc) == 0) {
         /* XXX: NULL selector case: invalid LDT */
-        env->ldt.base = NULL;
+        env->ldt.base = 0;
         env->ldt.limit = 0;
     } else {
         if (selector & 0x4)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         dt = &env->gdt;
         index = selector & ~7;
-        if ((index + 7) > dt->limit)
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK)
+            entry_limit = 15;
+        else
+#endif            
+            entry_limit = 7;
+        if ((index + entry_limit) > dt->limit)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         ptr = dt->base + index;
         e1 = ldl_kernel(ptr);
@@ -1127,7 +1374,17 @@ void helper_lldt_T0(void)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         if (!(e2 & DESC_P_MASK))
             raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
-        load_seg_cache_raw_dt(&env->ldt, e1, e2);
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            uint32_t e3;
+            e3 = ldl_kernel(ptr + 8);
+            load_seg_cache_raw_dt(&env->ldt, e1, e2);
+            env->ldt.base |= (target_ulong)e3 << 32;
+        } else
+#endif
+        {
+            load_seg_cache_raw_dt(&env->ldt, e1, e2);
+        }
     }
     env->ldt.selector = selector;
 }
@@ -1137,13 +1394,13 @@ void helper_ltr_T0(void)
     int selector;
     SegmentCache *dt;
     uint32_t e1, e2;
-    int index, type;
-    uint8_t *ptr;
+    int index, type, entry_limit;
+    target_ulong ptr;
  
     selector = T0 & 0xffff;
     if ((selector & 0xfffc) == 0) {
-        /* NULL selector case: invalid LDT */
-        env->tr.base = NULL;
+        /* NULL selector case: invalid TR */
+        env->tr.base = 0;
         env->tr.limit = 0;
         env->tr.flags = 0;
     } else {
@@ -1151,7 +1408,13 @@ void helper_ltr_T0(void)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         dt = &env->gdt;
         index = selector & ~7;
-        if ((index + 7) > dt->limit)
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK)
+            entry_limit = 15;
+        else
+#endif            
+            entry_limit = 7;
+        if ((index + entry_limit) > dt->limit)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         ptr = dt->base + index;
         e1 = ldl_kernel(ptr);
@@ -1162,7 +1425,17 @@ void helper_ltr_T0(void)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         if (!(e2 & DESC_P_MASK))
             raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
-        load_seg_cache_raw_dt(&env->tr, e1, e2);
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            uint32_t e3;
+            e3 = ldl_kernel(ptr + 8);
+            load_seg_cache_raw_dt(&env->tr, e1, e2);
+            env->tr.base |= (target_ulong)e3 << 32;
+        } else 
+#endif
+        {
+            load_seg_cache_raw_dt(&env->tr, e1, e2);
+        }
         e2 |= DESC_TSS_BUSY_MASK;
         stl_kernel(ptr + 4, e2);
     }
@@ -1176,14 +1449,14 @@ void load_seg(int seg_reg, int selector)
     int cpl, dpl, rpl;
     SegmentCache *dt;
     int index;
-    uint8_t *ptr;
+    target_ulong ptr;
  
     selector &= 0xffff;
     if ((selector & 0xfffc) == 0) {
         /* null selector case */
         if (seg_reg == R_SS)
             raise_exception_err(EXCP0D_GPF, 0);
-        cpu_x86_load_seg_cache(env, seg_reg, selector, NULL, 0, 0);
+        cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
     } else {
  
         if (selector & 0x4)
@@ -1196,7 +1469,7 @@ void load_seg(int seg_reg, int selector)
         ptr = dt->base + index;
         e1 = ldl_kernel(ptr);
         e2 = ldl_kernel(ptr + 4);
-
+        
         if (!(e2 & DESC_S_MASK))
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         rpl = selector & 3;
@@ -1247,9 +1520,10 @@ void load_seg(int seg_reg, int selector)
 /* protected mode jump */
 void helper_ljmp_protected_T0_T1(int next_eip)
 {
-    int new_cs, new_eip, gate_cs, type;
+    int new_cs, gate_cs, type;
     uint32_t e1, e2, cpl, dpl, rpl, limit;
-
+    target_ulong new_eip;
+    
     new_cs = T0;
     new_eip = T1;
     if ((new_cs & 0xfffc) == 0)
@@ -1312,7 +1586,7 @@ void helper_ljmp_protected_T0_T1(int next_eip)
             if (((e2 & (DESC_S_MASK | DESC_CS_MASK)) != 
                  (DESC_S_MASK | DESC_CS_MASK)))
                 raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
-            if (((e2 & DESC_C_MASK) && (dpl > cpl)) ||
+            if (((e2 & DESC_C_MASK) && (dpl > cpl)) || 
                 (!(e2 & DESC_C_MASK) && (dpl != cpl)))
                 raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
             if (!(e2 & DESC_P_MASK))
@@ -1336,7 +1610,7 @@ void helper_lcall_real_T0_T1(int shift, int next_eip)
 {
     int new_cs, new_eip;
     uint32_t esp, esp_mask;
-    uint8_t *ssp;
+    target_ulong ssp;
  
     new_cs = T0;
     new_eip = T1;
@@ -1354,7 +1628,7 @@ void helper_lcall_real_T0_T1(int shift, int next_eip)
     ESP = (ESP & ~esp_mask) | (esp & esp_mask);
     env->eip = new_eip;
     env->segs[R_CS].selector = new_cs;
-    env->segs[R_CS].base = (uint8_t *)(new_cs << 4);
+    env->segs[R_CS].base = (new_cs << 4);
 }
  
 /* protected mode call */
@@ -1364,7 +1638,7 @@ void helper_lcall_protected_T0_T1(int shift, int next_eip)
     uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
     uint32_t ss, ss_e1, ss_e2, sp, type, ss_dpl, sp_mask;
     uint32_t val, limit, old_sp_mask;
-    uint8_t *ssp, *old_ssp;
+    target_ulong ssp, old_ssp;
  
     new_cs = T0;
     new_eip = T1;
@@ -1471,7 +1745,7 @@ void helper_lcall_protected_T0_T1(int shift, int next_eip)
             get_ss_esp_from_tss(&ss, &sp, dpl);
 #ifdef DEBUG_PCALL
             if (loglevel & CPU_LOG_PCALL)
-                fprintf(logfile, "new ss:esp=%04x:%08x param_count=%d ESP=%x\n", 
+                fprintf(logfile, "new ss:esp=%04x:%08x param_count=%d ESP=" TARGET_FMT_lx "\n", 
                         ss, sp, param_count, ESP);
 #endif
             if ((ss & 0xfffc) == 0)
@@ -1555,7 +1829,7 @@ void helper_lcall_protected_T0_T1(int shift, int next_eip)
 void helper_iret_real(int shift)
 {
     uint32_t sp, new_cs, new_eip, new_eflags, sp_mask;
-    uint8_t *ssp;
+    target_ulong ssp;
     int eflags_mask;
  
     sp_mask = 0xffff; /* XXXX: use SS segment size ? */
@@ -1595,7 +1869,7 @@ static inline void validate_seg(int seg_reg, int cpl)
     if (!(e2 & DESC_CS_MASK) || !(e2 & DESC_C_MASK)) {
         /* data or non conforming code segment */
         if (dpl < cpl) {
-            cpu_x86_load_seg_cache(env, seg_reg, 0, NULL, 0, 0);
+            cpu_x86_load_seg_cache(env, seg_reg, 0, 0, 0, 0);
         }
     }
 }
@@ -1603,16 +1877,31 @@ static inline void validate_seg(int seg_reg, int cpl)
 /* protected mode iret */
 static inline void helper_ret_protected(int shift, int is_iret, int addend)
 {
-    uint32_t sp, new_cs, new_eip, new_eflags, new_esp, new_ss, sp_mask;
+    uint32_t new_cs, new_eflags, new_ss;
     uint32_t new_es, new_ds, new_fs, new_gs;
     uint32_t e1, e2, ss_e1, ss_e2;
     int cpl, dpl, rpl, eflags_mask, iopl;
-    uint8_t *ssp;
+    target_ulong ssp, sp, new_eip, new_esp, sp_mask;
  
-    sp_mask = get_sp_mask(env->segs[R_SS].flags);
+#ifdef TARGET_X86_64
+    if (shift == 2)
+        sp_mask = -1;
+    else
+#endif
+        sp_mask = get_sp_mask(env->segs[R_SS].flags);
     sp = ESP;
     ssp = env->segs[R_SS].base;
     new_eflags = 0; /* avoid warning */
+#ifdef TARGET_X86_64
+    if (shift == 2) {
+        POPQ(sp, new_eip);
+        POPQ(sp, new_cs);
+        new_cs &= 0xffff;
+        if (is_iret) {
+            POPQ(sp, new_eflags);
+        }
+    } else
+#endif
     if (shift == 1) {
         /* 32 bits */
         POPL(ssp, sp, sp_mask, new_eip);
@@ -1632,7 +1921,7 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
     }
 #ifdef DEBUG_PCALL
     if (loglevel & CPU_LOG_PCALL) {
-        fprintf(logfile, "lret new %04x:%08x s=%d addend=0x%x\n",
+        fprintf(logfile, "lret new %04x:" TARGET_FMT_lx " s=%d addend=0x%x\n",
                 new_cs, new_eip, shift, addend);
         cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
     }
@@ -1660,7 +1949,7 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
         raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
  
     sp += addend;
-    if (rpl == cpl) {
+    if (rpl == cpl && !(env->hflags & HF_CS64_MASK)) {
         /* return to same priledge level */
         cpu_x86_load_seg_cache(env, R_CS, new_cs, 
                        get_seg_base(e1, e2),
@@ -1668,6 +1957,13 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
                        e2);
     } else {
         /* return to different priviledge level */
+#ifdef TARGET_X86_64
+        if (shift == 2) {
+            POPQ(sp, new_esp);
+            POPQ(sp, new_ss);
+            new_ss &= 0xffff;
+        } else
+#endif
         if (shift == 1) {
             /* 32 bits */
             POPL(ssp, sp, sp_mask, new_esp);
@@ -1680,36 +1976,49 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
         }
 #ifdef DEBUG_PCALL
         if (loglevel & CPU_LOG_PCALL) {
-            fprintf(logfile, "new ss:esp=%04x:%08x\n",
+            fprintf(logfile, "new ss:esp=%04x:" TARGET_FMT_lx "\n",
                     new_ss, new_esp);
         }
 #endif
-        
-        if ((new_ss & 3) != rpl)
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        if (load_segment(&ss_e1, &ss_e2, new_ss) != 0)
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        if (!(ss_e2 & DESC_S_MASK) ||
-            (ss_e2 & DESC_CS_MASK) ||
-            !(ss_e2 & DESC_W_MASK))
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
-        if (dpl != rpl)
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        if (!(ss_e2 & DESC_P_MASK))
-            raise_exception_err(EXCP0B_NOSEG, new_ss & 0xfffc);
+        if ((env->hflags & HF_LMA_MASK) && (new_ss & 0xfffc) == 0) {
+            /* NULL ss is allowed in long mode */
+            cpu_x86_load_seg_cache(env, R_SS, new_ss, 
+                                   0, 0xffffffff,
+                                   DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                                   DESC_S_MASK | (rpl << DESC_DPL_SHIFT) |
+                                   DESC_W_MASK | DESC_A_MASK);
+        } else {
+            if ((new_ss & 3) != rpl)
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            if (load_segment(&ss_e1, &ss_e2, new_ss) != 0)
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            if (!(ss_e2 & DESC_S_MASK) ||
+                (ss_e2 & DESC_CS_MASK) ||
+                !(ss_e2 & DESC_W_MASK))
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
+            if (dpl != rpl)
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            if (!(ss_e2 & DESC_P_MASK))
+                raise_exception_err(EXCP0B_NOSEG, new_ss & 0xfffc);
+            cpu_x86_load_seg_cache(env, R_SS, new_ss, 
+                                   get_seg_base(ss_e1, ss_e2),
+                                   get_seg_limit(ss_e1, ss_e2),
+                                   ss_e2);
+        }
  
         cpu_x86_load_seg_cache(env, R_CS, new_cs, 
                        get_seg_base(e1, e2),
                        get_seg_limit(e1, e2),
                        e2);
-        cpu_x86_load_seg_cache(env, R_SS, new_ss, 
-                       get_seg_base(ss_e1, ss_e2),
-                       get_seg_limit(ss_e1, ss_e2),
-                       ss_e2);
         cpu_x86_set_cpl(env, rpl);
         sp = new_esp;
-        sp_mask = get_sp_mask(ss_e2);
+#ifdef TARGET_X86_64
+        if (shift == 2)
+            sp_mask = -1;
+        else
+#endif
+            sp_mask = get_sp_mask(ss_e2);
  
         /* validate data segments */
         validate_seg(R_ES, cpl);
@@ -1765,6 +2074,10 @@ void helper_iret_protected(int shift, int next_eip)
  
     /* specific case for TSS */
     if (env->eflags & NT_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK)
+            raise_exception_err(EXCP0D_GPF, 0);
+#endif
         tss_selector = lduw_kernel(env->tr.base + 0);
         if (tss_selector & 4)
             raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
@@ -1793,12 +2106,12 @@ void helper_sysenter(void)
     env->eflags &= ~(VM_MASK | IF_MASK | RF_MASK);
     cpu_x86_set_cpl(env, 0);
     cpu_x86_load_seg_cache(env, R_CS, env->sysenter_cs & 0xfffc, 
-                           NULL, 0xffffffff, 
+                           0, 0xffffffff, 
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK |
                            DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_SS, (env->sysenter_cs + 8) & 0xfffc, 
-                           NULL, 0xffffffff,
+                           0, 0xffffffff,
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK |
                            DESC_W_MASK | DESC_A_MASK);
@@ -1816,12 +2129,12 @@ void helper_sysexit(void)
     }
     cpu_x86_set_cpl(env, 3);
     cpu_x86_load_seg_cache(env, R_CS, ((env->sysenter_cs + 16) & 0xfffc) | 3, 
-                           NULL, 0xffffffff, 
+                           0, 0xffffffff, 
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
                            DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_SS, ((env->sysenter_cs + 24) & 0xfffc) | 3, 
-                           NULL, 0xffffffff,
+                           0, 0xffffffff,
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
                            DESC_W_MASK | DESC_A_MASK);
@@ -1863,22 +2176,67 @@ void helper_rdtsc(void)
     uint64_t val;
  
     val = cpu_get_tsc(env);
-    EAX = val;
-    EDX = val >> 32;
+    EAX = (uint32_t)(val);
+    EDX = (uint32_t)(val >> 32);
+}
+
+#if defined(CONFIG_USER_ONLY) 
+void helper_wrmsr(void)
+{
 }
  
+void helper_rdmsr(void)
+{
+}
+#else
 void helper_wrmsr(void)
 {
-    switch(ECX) {
+    uint64_t val;
+
+    val = ((uint32_t)EAX) | ((uint64_t)((uint32_t)EDX) << 32);
+
+    switch((uint32_t)ECX) {
     case MSR_IA32_SYSENTER_CS:
-        env->sysenter_cs = EAX & 0xffff;
+        env->sysenter_cs = val & 0xffff;
         break;
     case MSR_IA32_SYSENTER_ESP:
-        env->sysenter_esp = EAX;
+        env->sysenter_esp = val;
         break;
     case MSR_IA32_SYSENTER_EIP:
-        env->sysenter_eip = EAX;
+        env->sysenter_eip = val;
+        break;
+    case MSR_IA32_APICBASE:
+        cpu_set_apic_base(env, val);
+        break;
+#ifdef TARGET_X86_64
+    case MSR_EFER:
+#define MSR_EFER_UPDATE_MASK (MSR_EFER_SCE | MSR_EFER_LME | \
+                              MSR_EFER_NXE | MSR_EFER_FFXSR)
+        env->efer = (env->efer & ~MSR_EFER_UPDATE_MASK) | 
+            (val & MSR_EFER_UPDATE_MASK);
         break;
+    case MSR_STAR:
+        env->star = val;
+        break;
+    case MSR_LSTAR:
+        env->lstar = val;
+        break;
+    case MSR_CSTAR:
+        env->cstar = val;
+        break;
+    case MSR_FMASK:
+        env->fmask = val;
+        break;
+    case MSR_FSBASE:
+        env->segs[R_FS].base = val;
+        break;
+    case MSR_GSBASE:
+        env->segs[R_GS].base = val;
+        break;
+    case MSR_KERNELGSBASE:
+        env->kernelgsbase = val;
+        break;
+#endif
     default:
         /* XXX: exception ? */
         break; 
@@ -1887,24 +2245,55 @@ void helper_wrmsr(void)
  
 void helper_rdmsr(void)
 {
-    switch(ECX) {
+    uint64_t val;
+    switch((uint32_t)ECX) {
     case MSR_IA32_SYSENTER_CS:
-        EAX = env->sysenter_cs;
-        EDX = 0;
+        val = env->sysenter_cs;
         break;
     case MSR_IA32_SYSENTER_ESP:
-        EAX = env->sysenter_esp;
-        EDX = 0;
+        val = env->sysenter_esp;
         break;
     case MSR_IA32_SYSENTER_EIP:
-        EAX = env->sysenter_eip;
-        EDX = 0;
+        val = env->sysenter_eip;
+        break;
+    case MSR_IA32_APICBASE:
+        val = cpu_get_apic_base(env);
+        break;
+#ifdef TARGET_X86_64
+    case MSR_EFER:
+        val = env->efer;
+        break;
+    case MSR_STAR:
+        val = env->star;
+        break;
+    case MSR_LSTAR:
+        val = env->lstar;
+        break;
+    case MSR_CSTAR:
+        val = env->cstar;
+        break;
+    case MSR_FMASK:
+        val = env->fmask;
+        break;
+    case MSR_FSBASE:
+        val = env->segs[R_FS].base;
+        break;
+    case MSR_GSBASE:
+        val = env->segs[R_GS].base;
         break;
+    case MSR_KERNELGSBASE:
+        val = env->kernelgsbase;
+        break;
+#endif
     default:
         /* XXX: exception ? */
+        val = 0;
         break; 
     }
+    EAX = (uint32_t)(val);
+    EDX = (uint32_t)(val >> 32);
 }
+#endif
  
 void helper_lsl(void)
 {
@@ -2055,14 +2444,14 @@ void helper_fldt_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = helper_fldt((uint8_t *)A0);
+    env->fpregs[new_fpstt] = helper_fldt(A0);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
  
 void helper_fstt_ST0_A0(void)
 {
-    helper_fstt(ST0, (uint8_t *)A0);
+    helper_fstt(ST0, A0);
 }
  
 void fpu_set_exception(int mask)
@@ -2102,11 +2491,11 @@ void helper_fbld_ST0_A0(void)
  
     val = 0;
     for(i = 8; i >= 0; i--) {
-        v = ldub((uint8_t *)A0 + i);
+        v = ldub(A0 + i);
         val = (val * 100) + ((v >> 4) * 10) + (v & 0xf);
     }
     tmp = val;
-    if (ldub((uint8_t *)A0 + 9) & 0x80)
+    if (ldub(A0 + 9) & 0x80)
         tmp = -tmp;
     fpush();
     ST0 = tmp;
@@ -2116,12 +2505,12 @@ void helper_fbst_ST0_A0(void)
 {
     CPU86_LDouble tmp;
     int v;
-    uint8_t *mem_ref, *mem_end;
+    target_ulong mem_ref, mem_end;
     int64_t val;
  
     tmp = rint(ST0);
     val = (int64_t)tmp;
-    mem_ref = (uint8_t *)A0;
+    mem_ref = A0;
     mem_end = mem_ref + 9;
     if (val < 0) {
         stb(mem_end, 0x80);
@@ -2402,7 +2791,7 @@ void helper_fxam_ST0(void)
     }
 }
  
-void helper_fstenv(uint8_t *ptr, int data32)
+void helper_fstenv(target_ulong ptr, int data32)
 {
     int fpus, fptag, exp, i;
     uint64_t mant;
@@ -2452,7 +2841,7 @@ void helper_fstenv(uint8_t *ptr, int data32)
     }
 }
  
-void helper_fldenv(uint8_t *ptr, int data32)
+void helper_fldenv(target_ulong ptr, int data32)
 {
     int i, fpus, fptag;
  
@@ -2474,7 +2863,7 @@ void helper_fldenv(uint8_t *ptr, int data32)
     }
 }
  
-void helper_fsave(uint8_t *ptr, int data32)
+void helper_fsave(target_ulong ptr, int data32)
 {
     CPU86_LDouble tmp;
     int i;
@@ -2502,7 +2891,7 @@ void helper_fsave(uint8_t *ptr, int data32)
     env->fptags[7] = 1;
 }
  
-void helper_frstor(uint8_t *ptr, int data32)
+void helper_frstor(target_ulong ptr, int data32)
 {
     CPU86_LDouble tmp;
     int i;
@@ -2517,7 +2906,78 @@ void helper_frstor(uint8_t *ptr, int data32)
     }
 }
  
-/* XXX: merge with helper_fstt ? */
+void helper_fxsave(target_ulong ptr, int data64)
+{
+    int fpus, fptag, i, nb_xmm_regs;
+    CPU86_LDouble tmp;
+    target_ulong addr;
+
+    fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
+    fptag = 0;
+    for(i = 0; i < 8; i++) {
+        fptag |= ((!env->fptags[(env->fpstt + i) & 7]) << i);
+    }
+    stw(ptr, env->fpuc);
+    stw(ptr + 2, fpus);
+    stw(ptr + 4, fptag);
+
+    addr = ptr + 0x20;
+    for(i = 0;i < 8; i++) {
+        tmp = ST(i);
+        helper_fstt(tmp, addr);
+        addr += 16;
+    }
+    
+    if (env->cr[4] & CR4_OSFXSR_MASK) {
+        /* XXX: finish it, endianness */
+        stl(ptr + 0x18, 0); /* mxcsr */
+        stl(ptr + 0x1c, 0); /* mxcsr_mask */
+        nb_xmm_regs = 8 << data64;
+        addr = ptr + 0xa0;
+        for(i = 0; i < nb_xmm_regs; i++) {
+            stq(addr, env->xmm_regs[i].u.q[0]);
+            stq(addr, env->xmm_regs[i].u.q[1]);
+            addr += 16;
+        }
+    }
+}
+
+void helper_fxrstor(target_ulong ptr, int data64)
+{
+    int i, fpus, fptag, nb_xmm_regs;
+    CPU86_LDouble tmp;
+    target_ulong addr;
+
+    env->fpuc = lduw(ptr);
+    fpus = lduw(ptr + 2);
+    fptag = ldub(ptr + 4);
+    env->fpstt = (fpus >> 11) & 7;
+    env->fpus = fpus & ~0x3800;
+    fptag ^= 0xff;
+    for(i = 0;i < 8; i++) {
+        env->fptags[(env->fpstt + i) & 7] = ((fptag >> i) & 1);
+    }
+
+    addr = ptr + 0x20;
+    for(i = 0;i < 8; i++) {
+        tmp = helper_fldt(addr);
+        ST(i) = tmp;
+        addr += 16;
+    }
+
+    if (env->cr[4] & CR4_OSFXSR_MASK) {
+        /* XXX: finish it, endianness */
+        //ldl(ptr + 0x18);
+        //ldl(ptr + 0x1c);
+        nb_xmm_regs = 8 << data64;
+        addr = ptr + 0xa0;
+        for(i = 0; i < nb_xmm_regs; i++) {
+            env->xmm_regs[i].u.q[0] = ldq(addr);
+            env->xmm_regs[i].u.q[1] = ldq(addr);
+            addr += 16;
+        }
+    }
+}
  
 #ifndef USE_X86LDOUBLE
  
@@ -2575,6 +3035,179 @@ CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper)
 }
 #endif
  
+#ifdef TARGET_X86_64
+
+//#define DEBUG_MULDIV
+
+static void add128(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b)
+{
+    *plow += a;
+    /* carry test */
+    if (*plow < a)
+        (*phigh)++;
+    *phigh += b;
+}
+
+static void neg128(uint64_t *plow, uint64_t *phigh)
+{
+    *plow = ~ *plow;
+    *phigh = ~ *phigh;
+    add128(plow, phigh, 1, 0);
+}
+
+static void mul64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b)
+{
+    uint32_t a0, a1, b0, b1;
+    uint64_t v;
+
+    a0 = a;
+    a1 = a >> 32;
+
+    b0 = b;
+    b1 = b >> 32;
+    
+    v = (uint64_t)a0 * (uint64_t)b0;
+    *plow = v;
+    *phigh = 0;
+
+    v = (uint64_t)a0 * (uint64_t)b1;
+    add128(plow, phigh, v << 32, v >> 32);
+    
+    v = (uint64_t)a1 * (uint64_t)b0;
+    add128(plow, phigh, v << 32, v >> 32);
+    
+    v = (uint64_t)a1 * (uint64_t)b1;
+    *phigh += v;
+#ifdef DEBUG_MULDIV
+    printf("mul: 0x%016llx * 0x%016llx = 0x%016llx%016llx\n",
+           a, b, *phigh, *plow);
+#endif
+}
+
+static void imul64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b)
+{
+    int sa, sb;
+    sa = (a < 0);
+    if (sa)
+        a = -a;
+    sb = (b < 0);
+    if (sb)
+        b = -b;
+    mul64(plow, phigh, a, b);
+    if (sa ^ sb) {
+        neg128(plow, phigh);
+    }
+}
+
+static void div64(uint64_t *plow, uint64_t *phigh, uint64_t b)
+{
+    uint64_t q, r, a1, a0;
+    int i, qb;
+
+    a0 = *plow;
+    a1 = *phigh;
+    if (a1 == 0) {
+        q = a0 / b;
+        r = a0 % b;
+        *plow = q;
+        *phigh = r;
+    } else {
+        /* XXX: use a better algorithm */
+        for(i = 0; i < 64; i++) {
+            if (a1 >= b) {
+                a1 -= b;
+                qb = 1;
+            } else {
+                qb = 0;
+            }
+            a1 = (a1 << 1) | (a0 >> 63);
+            a0 = (a0 << 1) | qb;
+        }
+#if defined(DEBUG_MULDIV) || 1
+        printf("div: 0x%016llx%016llx / 0x%016llx: q=0x%016llx r=0x%016llx\n",
+               *phigh, *plow, b, a0, a1);
+#endif
+        *plow = a0;
+        *phigh = a1;
+    }
+}
+
+static void idiv64(uint64_t *plow, uint64_t *phigh, uint64_t b)
+{
+    int sa, sb;
+    sa = ((int64_t)*phigh < 0);
+    if (sa)
+        neg128(plow, phigh);
+    sb = (b < 0);
+    if (sb)
+        b = -b;
+    div64(plow, phigh, b);
+    if (sa ^ sb)
+        *plow = - *plow;
+    if (sb)
+        *phigh = - *phigh;
+}
+
+void helper_mulq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+
+    mul64(&r0, &r1, EAX, T0);
+    EAX = r0;
+    EDX = r1;
+    CC_DST = r0;
+    CC_SRC = r1;
+}
+
+void helper_imulq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+
+    imul64(&r0, &r1, EAX, T0);
+    EAX = r0;
+    EDX = r1;
+    CC_DST = r0;
+    CC_SRC = (r1 != (r0 >> 63));
+}
+
+void helper_imulq_T0_T1(void)
+{
+    uint64_t r0, r1;
+
+    imul64(&r0, &r1, T0, T1);
+    T0 = r0;
+    CC_DST = r0;
+    CC_SRC = ((int64_t)r1 != ((int64_t)r0 >> 63));
+}
+
+void helper_divq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+    if (T0 == 0) {
+        raise_exception(EXCP00_DIVZ);
+    }
+    r0 = EAX;
+    r1 = EDX;
+    div64(&r0, &r1, T0);
+    EAX = r0;
+    EDX = r1;
+}
+
+void helper_idivq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+    if (T0 == 0) {
+        raise_exception(EXCP00_DIVZ);
+    }
+    r0 = EAX;
+    r1 = EDX;
+    idiv64(&r0, &r1, T0);
+    EAX = r0;
+    EDX = r1;
+}
+
+#endif
+
 #if !defined(CONFIG_USER_ONLY) 
  
 #define MMUSUFFIX _mmu
@@ -2598,7 +3231,7 @@ CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper)
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
 /* XXX: fix it to restore all registers */
-void tlb_fill(unsigned long addr, int is_write, int is_user, void *retaddr)
+void tlb_fill(target_ulong addr, int is_write, int is_user, void *retaddr)
 {
     TranslationBlock *tb;
     int ret;
@@ -77,6 +77,41 @@ CPUX86State *cpu_x86_init(void)
         asm volatile ("movl %0, %%fs" : : "r" ((1 << 3) | 7));
     }
 #endif
+    {
+        int family, model, stepping;
+#ifdef TARGET_X86_64
+        env->cpuid_vendor1 = 0x68747541; /* "Auth" */
+        env->cpuid_vendor2 = 0x69746e65; /* "enti" */
+        env->cpuid_vendor3 = 0x444d4163; /* "cAMD" */
+        family = 6;
+        model = 2;
+        stepping = 3;
+#else
+        env->cpuid_vendor1 = 0x756e6547; /* "Genu" */
+        env->cpuid_vendor2 = 0x49656e69; /* "ineI" */
+        env->cpuid_vendor3 = 0x6c65746e; /* "ntel" */
+#if 0
+        /* pentium 75-200 */
+        family = 5;
+        model = 2;
+        stepping = 11;
+#else
+        /* pentium pro */
+        family = 6;
+        model = 1;
+        stepping = 3;
+#endif
+#endif
+        env->cpuid_version = (family << 8) | (model << 4) | stepping;
+        env->cpuid_features = (CPUID_FP87 | CPUID_DE | CPUID_PSE |
+                               CPUID_TSC | CPUID_MSR | CPUID_MCE |
+                               CPUID_CX8 | CPUID_PGE | CPUID_CMOV);
+#ifdef TARGET_X86_64
+        /* currently not enabled for std i386 because not fully tested */
+        env->cpuid_features |= CPUID_APIC | CPUID_FXSR | CPUID_PAE |
+            CPUID_SSE | CPUID_SSE2;
+#endif
+    }
     cpu_single_env = env;
     cpu_reset(env);
     return env;
@@ -107,12 +142,12 @@ void cpu_reset(CPUX86State *env)
     env->tr.limit = 0xffff;
     env->tr.flags = DESC_P_MASK;
  
-    cpu_x86_load_seg_cache(env, R_CS, 0xf000, (uint8_t *)0xffff0000, 0xffff, 0); 
-    cpu_x86_load_seg_cache(env, R_DS, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_ES, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_SS, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_FS, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_GS, 0, NULL, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_CS, 0xf000, 0xffff0000, 0xffff, 0); 
+    cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_SS, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0xffff, 0);
  
     env->eip = 0xfff0;
     env->regs[R_EDX] = 0x600; /* indicate P6 processor */
@@ -136,36 +171,56 @@ void cpu_x86_close(CPUX86State *env)
 static const char *cc_op_str[] = {
     "DYNAMIC",
     "EFLAGS",
+
     "MULB",
     "MULW",
     "MULL",
+    "MULQ",
+
     "ADDB",
     "ADDW",
     "ADDL",
+    "ADDQ",
+
     "ADCB",
     "ADCW",
     "ADCL",
+    "ADCQ",
+
     "SUBB",
     "SUBW",
     "SUBL",
+    "SUBQ",
+
     "SBBB",
     "SBBW",
     "SBBL",
+    "SBBQ",
+
     "LOGICB",
     "LOGICW",
     "LOGICL",
+    "LOGICQ",
+
     "INCB",
     "INCW",
     "INCL",
+    "INCQ",
+
     "DECB",
     "DECW",
     "DECL",
+    "DECQ",
+
     "SHLB",
     "SHLW",
     "SHLL",
+    "SHLQ",
+
     "SARB",
     "SARW",
     "SARL",
+    "SARQ",
 };
  
 void cpu_dump_state(CPUState *env, FILE *f, 
@@ -177,55 +232,147 @@ void cpu_dump_state(CPUState *env, FILE *f,
     static const char *seg_name[6] = { "ES", "CS", "SS", "DS", "FS", "GS" };
  
     eflags = env->eflags;
-    cpu_fprintf(f, "EAX=%08x EBX=%08x ECX=%08x EDX=%08x\n"
-            "ESI=%08x EDI=%08x EBP=%08x ESP=%08x\n"
-            "EIP=%08x EFL=%08x [%c%c%c%c%c%c%c]    CPL=%d II=%d A20=%d\n",
-            env->regs[R_EAX], env->regs[R_EBX], env->regs[R_ECX], env->regs[R_EDX], 
-            env->regs[R_ESI], env->regs[R_EDI], env->regs[R_EBP], env->regs[R_ESP], 
-            env->eip, eflags,
-            eflags & DF_MASK ? 'D' : '-',
-            eflags & CC_O ? 'O' : '-',
-            eflags & CC_S ? 'S' : '-',
-            eflags & CC_Z ? 'Z' : '-',
-            eflags & CC_A ? 'A' : '-',
-            eflags & CC_P ? 'P' : '-',
-            eflags & CC_C ? 'C' : '-',
-            env->hflags & HF_CPL_MASK, 
-            (env->hflags >> HF_INHIBIT_IRQ_SHIFT) & 1,
-            (env->a20_mask >> 20) & 1);
-    for(i = 0; i < 6; i++) {
-        SegmentCache *sc = &env->segs[i];
-        cpu_fprintf(f, "%s =%04x %08x %08x %08x\n",
-                seg_name[i],
-                sc->selector,
-                (int)sc->base,
-                sc->limit,
-                sc->flags);
+#ifdef TARGET_X86_64
+    if (env->hflags & HF_CS64_MASK) {
+        cpu_fprintf(f, 
+                    "RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n"
+                    "RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n"
+                    "R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n"
+                    "R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n"
+                    "RIP=%016llx RFL=%08x [%c%c%c%c%c%c%c]    CPL=%d II=%d A20=%d\n",
+                    env->regs[R_EAX], 
+                    env->regs[R_EBX], 
+                    env->regs[R_ECX], 
+                    env->regs[R_EDX], 
+                    env->regs[R_ESI], 
+                    env->regs[R_EDI], 
+                    env->regs[R_EBP], 
+                    env->regs[R_ESP], 
+                    env->regs[8], 
+                    env->regs[9], 
+                    env->regs[10], 
+                    env->regs[11], 
+                    env->regs[12], 
+                    env->regs[13], 
+                    env->regs[14], 
+                    env->regs[15], 
+                    env->eip, eflags,
+                    eflags & DF_MASK ? 'D' : '-',
+                    eflags & CC_O ? 'O' : '-',
+                    eflags & CC_S ? 'S' : '-',
+                    eflags & CC_Z ? 'Z' : '-',
+                    eflags & CC_A ? 'A' : '-',
+                    eflags & CC_P ? 'P' : '-',
+                    eflags & CC_C ? 'C' : '-',
+                    env->hflags & HF_CPL_MASK, 
+                    (env->hflags >> HF_INHIBIT_IRQ_SHIFT) & 1,
+                    (env->a20_mask >> 20) & 1);
+    } else 
+#endif
+    {
+        cpu_fprintf(f, "EAX=%08x EBX=%08x ECX=%08x EDX=%08x\n"
+                    "ESI=%08x EDI=%08x EBP=%08x ESP=%08x\n"
+                    "EIP=%08x EFL=%08x [%c%c%c%c%c%c%c]    CPL=%d II=%d A20=%d\n",
+                    (uint32_t)env->regs[R_EAX], 
+                    (uint32_t)env->regs[R_EBX], 
+                    (uint32_t)env->regs[R_ECX], 
+                    (uint32_t)env->regs[R_EDX], 
+                    (uint32_t)env->regs[R_ESI], 
+                    (uint32_t)env->regs[R_EDI], 
+                    (uint32_t)env->regs[R_EBP], 
+                    (uint32_t)env->regs[R_ESP], 
+                    (uint32_t)env->eip, eflags,
+                    eflags & DF_MASK ? 'D' : '-',
+                    eflags & CC_O ? 'O' : '-',
+                    eflags & CC_S ? 'S' : '-',
+                    eflags & CC_Z ? 'Z' : '-',
+                    eflags & CC_A ? 'A' : '-',
+                    eflags & CC_P ? 'P' : '-',
+                    eflags & CC_C ? 'C' : '-',
+                    env->hflags & HF_CPL_MASK, 
+                    (env->hflags >> HF_INHIBIT_IRQ_SHIFT) & 1,
+                    (env->a20_mask >> 20) & 1);
+    }
+
+#ifdef TARGET_X86_64
+    if (env->hflags & HF_LMA_MASK) {
+        for(i = 0; i < 6; i++) {
+            SegmentCache *sc = &env->segs[i];
+            cpu_fprintf(f, "%s =%04x %016llx %08x %08x\n",
+                        seg_name[i],
+                        sc->selector,
+                        sc->base,
+                        sc->limit,
+                        sc->flags);
+        }
+        cpu_fprintf(f, "LDT=%04x %016llx %08x %08x\n",
+                    env->ldt.selector,
+                    env->ldt.base,
+                    env->ldt.limit,
+                    env->ldt.flags);
+        cpu_fprintf(f, "TR =%04x %016llx %08x %08x\n",
+                    env->tr.selector,
+                    env->tr.base,
+                    env->tr.limit,
+                    env->tr.flags);
+        cpu_fprintf(f, "GDT=     %016llx %08x\n",
+                    env->gdt.base, env->gdt.limit);
+        cpu_fprintf(f, "IDT=     %016llx %08x\n",
+                    env->idt.base, env->idt.limit);
+        cpu_fprintf(f, "CR0=%08x CR2=%016llx CR3=%016llx CR4=%08x\n",
+                    (uint32_t)env->cr[0], 
+                    env->cr[2], 
+                    env->cr[3], 
+                    (uint32_t)env->cr[4]);
+    } else
+#endif
+    {
+        for(i = 0; i < 6; i++) {
+            SegmentCache *sc = &env->segs[i];
+            cpu_fprintf(f, "%s =%04x %08x %08x %08x\n",
+                        seg_name[i],
+                        sc->selector,
+                        (uint32_t)sc->base,
+                        sc->limit,
+                        sc->flags);
+        }
+        cpu_fprintf(f, "LDT=%04x %08x %08x %08x\n",
+                    env->ldt.selector,
+                    (uint32_t)env->ldt.base,
+                    env->ldt.limit,
+                    env->ldt.flags);
+        cpu_fprintf(f, "TR =%04x %08x %08x %08x\n",
+                    env->tr.selector,
+                    (uint32_t)env->tr.base,
+                    env->tr.limit,
+                    env->tr.flags);
+        cpu_fprintf(f, "GDT=     %08x %08x\n",
+                    (uint32_t)env->gdt.base, env->gdt.limit);
+        cpu_fprintf(f, "IDT=     %08x %08x\n",
+                    (uint32_t)env->idt.base, env->idt.limit);
+        cpu_fprintf(f, "CR0=%08x CR2=%08x CR3=%08x CR4=%08x\n",
+                    (uint32_t)env->cr[0], 
+                    (uint32_t)env->cr[2], 
+                    (uint32_t)env->cr[3], 
+                    (uint32_t)env->cr[4]);
     }
-    cpu_fprintf(f, "LDT=%04x %08x %08x %08x\n",
-            env->ldt.selector,
-            (int)env->ldt.base,
-            env->ldt.limit,
-            env->ldt.flags);
-    cpu_fprintf(f, "TR =%04x %08x %08x %08x\n",
-            env->tr.selector,
-            (int)env->tr.base,
-            env->tr.limit,
-            env->tr.flags);
-    cpu_fprintf(f, "GDT=     %08x %08x\n",
-            (int)env->gdt.base, env->gdt.limit);
-    cpu_fprintf(f, "IDT=     %08x %08x\n",
-            (int)env->idt.base, env->idt.limit);
-    cpu_fprintf(f, "CR0=%08x CR2=%08x CR3=%08x CR4=%08x\n",
-            env->cr[0], env->cr[2], env->cr[3], env->cr[4]);
-    
     if (flags & X86_DUMP_CCOP) {
         if ((unsigned)env->cc_op < CC_OP_NB)
             snprintf(cc_op_name, sizeof(cc_op_name), "%s", cc_op_str[env->cc_op]);
         else
             snprintf(cc_op_name, sizeof(cc_op_name), "[%d]", env->cc_op);
-        cpu_fprintf(f, "CCS=%08x CCD=%08x CCO=%-8s\n",
-                env->cc_src, env->cc_dst, cc_op_name);
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_CS64_MASK) {
+            cpu_fprintf(f, "CCS=%016llx CCD=%016llx CCO=%-8s\n",
+                        env->cc_src, env->cc_dst, 
+                        cc_op_name);
+        } else 
+#endif
+        {
+            cpu_fprintf(f, "CCS=%08x CCD=%08x CCO=%-8s\n",
+                        (uint32_t)env->cc_src, (uint32_t)env->cc_dst, 
+                        cc_op_name);
+        }
     }
     if (flags & X86_DUMP_FPU) {
         cpu_fprintf(f, "ST0=%f ST1=%f ST2=%f ST3=%f\n", 
@@ -274,6 +421,24 @@ void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0)
         (env->cr[0] & (CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK))) {
         tlb_flush(env, 1);
     }
+
+#ifdef TARGET_X86_64
+    if (!(env->cr[0] & CR0_PG_MASK) && (new_cr0 & CR0_PG_MASK) &&
+        (env->efer & MSR_EFER_LME)) {
+        /* enter in long mode */
+        /* XXX: generate an exception */
+        if (!(env->cr[4] & CR4_PAE_MASK))
+            return;
+        env->efer |= MSR_EFER_LMA;
+        env->hflags |= HF_LMA_MASK;
+    } else if ((env->cr[0] & CR0_PG_MASK) && !(new_cr0 & CR0_PG_MASK) &&
+               (env->efer & MSR_EFER_LMA)) {
+        /* exit long mode */
+        env->efer &= ~MSR_EFER_LMA;
+        env->hflags &= ~(HF_LMA_MASK | HF_CS64_MASK);
+        env->eip &= 0xffffffff;
+    }
+#endif
     env->cr[0] = new_cr0 | CR0_ET_MASK;
  
     /* update PE flag in hidden flags */
@@ -286,12 +451,12 @@ void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0)
         ((new_cr0 << (HF_MP_SHIFT - 1)) & (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK));
 }
  
-void cpu_x86_update_cr3(CPUX86State *env, uint32_t new_cr3)
+void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3)
 {
     env->cr[3] = new_cr3;
     if (env->cr[0] & CR0_PG_MASK) {
 #if defined(DEBUG_MMU)
-        printf("CR3 update: CR3=%08x\n", new_cr3);
+        printf("CR3 update: CR3=" TARGET_FMT_lx "\n", new_cr3);
 #endif
         tlb_flush(env, 0);
     }
@@ -300,7 +465,7 @@ void cpu_x86_update_cr3(CPUX86State *env, uint32_t new_cr3)
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 {
 #if defined(DEBUG_MMU)
-    printf("CR4 update: CR4=%08x\n", env->cr[4]);
+    printf("CR4 update: CR4=%08x\n", (uint32_t)env->cr[4]);
 #endif
     if ((new_cr4 & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK)) !=
         (env->cr[4] & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK))) {
@@ -315,22 +480,51 @@ void cpu_x86_flush_tlb(CPUX86State *env, uint32_t addr)
     tlb_flush_page(env, addr);
 }
  
+static inline uint8_t *get_phys_mem_ptr(target_phys_addr_t addr)
+{
+    /* XXX: incorrect */
+    return phys_ram_base + addr;
+}
+
+/* WARNING: addr must be aligned */
+uint32_t ldl_phys_aligned(target_phys_addr_t addr)
+{
+    uint8_t *ptr;
+    uint32_t val;
+    ptr = get_phys_mem_ptr(addr);
+    if (!ptr)
+        val = 0;
+    else
+        val = ldl_raw(ptr);
+    return val;
+}
+
+void stl_phys_aligned(target_phys_addr_t addr, uint32_t val)
+{
+    uint8_t *ptr;
+    ptr = get_phys_mem_ptr(addr);
+    if (!ptr)
+        return;
+    stl_raw(ptr, val);
+}
+
 /* return value:
    -1 = cannot handle fault 
    0  = nothing more to do 
    1  = generate PF fault
    2  = soft MMU activation required for this block
 */
-int cpu_x86_handle_mmu_fault(CPUX86State *env, uint32_t addr, 
+int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr, 
                              int is_write, int is_user, int is_softmmu)
 {
-    uint8_t *pde_ptr, *pte_ptr;
-    uint32_t pde, pte, virt_addr, ptep;
+    uint32_t pdpe_addr, pde_addr, pte_addr;
+    uint32_t pde, pte, ptep, pdpe;
     int error_code, is_dirty, prot, page_size, ret;
-    unsigned long paddr, vaddr, page_offset;
+    unsigned long paddr, page_offset;
+    target_ulong vaddr, virt_addr;
  
 #if defined(DEBUG_MMU)
-    printf("MMU fault: addr=0x%08x w=%d u=%d eip=%08x\n", 
+    printf("MMU fault: addr=" TARGET_FMT_lx " w=%d u=%d eip=" TARGET_FMT_lx "\n", 
            addr, is_write, is_user, env->eip);
 #endif
     is_write &= 1;
@@ -349,90 +543,166 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, uint32_t addr,
         goto do_mapping;
     }
  
-    /* page directory entry */
-    pde_ptr = phys_ram_base + 
-        (((env->cr[3] & ~0xfff) + ((addr >> 20) & ~3)) & env->a20_mask);
-    pde = ldl_raw(pde_ptr);
-    if (!(pde & PG_PRESENT_MASK)) {
-        error_code = 0;
-        goto do_fault;
-    }
-    /* if PSE bit is set, then we use a 4MB page */
-    if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-        if (is_user) {
-            if (!(pde & PG_USER_MASK))
-                goto do_fault_protect;
-            if (is_write && !(pde & PG_RW_MASK))
-                goto do_fault_protect;
-        } else {
-            if ((env->cr[0] & CR0_WP_MASK) && 
-                is_write && !(pde & PG_RW_MASK)) 
-                goto do_fault_protect;
+
+    if (env->cr[4] & CR4_PAE_MASK) {
+        /* XXX: we only use 32 bit physical addresses */
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            uint32_t pml4e_addr, pml4e;
+            int32_t sext;
+
+            /* XXX: handle user + rw rights */
+            /* XXX: handle NX flag */
+            /* test virtual address sign extension */
+            sext = (int64_t)addr >> 47;
+            if (sext != 0 && sext != -1) {
+                error_code = 0;
+                goto do_fault;
+            }
+            
+            pml4e_addr = ((env->cr[3] & ~0xfff) + (((addr >> 39) & 0x1ff) << 3)) & 
+                env->a20_mask;
+            pml4e = ldl_phys_aligned(pml4e_addr);
+            if (!(pml4e & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
+            if (!(pml4e & PG_ACCESSED_MASK)) {
+                pml4e |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pml4e_addr, pml4e);
+            }
+            
+            pdpe_addr = ((pml4e & ~0xfff) + (((addr >> 30) & 0x1ff) << 3)) & 
+                env->a20_mask;
+            pdpe = ldl_phys_aligned(pdpe_addr);
+            if (!(pdpe & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
+            if (!(pdpe & PG_ACCESSED_MASK)) {
+                pdpe |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pdpe_addr, pdpe);
+            }
+        } else 
+#endif
+        {
+            pdpe_addr = ((env->cr[3] & ~0x1f) + ((addr >> 30) << 3)) & 
+                env->a20_mask;
+            pdpe = ldl_phys_aligned(pdpe_addr);
+            if (!(pdpe & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
         }
-        is_dirty = is_write && !(pde & PG_DIRTY_MASK);
-        if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
-            pde |= PG_ACCESSED_MASK;
-            if (is_dirty)
-                pde |= PG_DIRTY_MASK;
-            stl_raw(pde_ptr, pde);
+
+        pde_addr = ((pdpe & ~0xfff) + (((addr >> 21) & 0x1ff) << 3)) &
+            env->a20_mask;
+        pde = ldl_phys_aligned(pde_addr);
+        if (!(pde & PG_PRESENT_MASK)) {
+            error_code = 0;
+            goto do_fault;
         }
-        
-        pte = pde & ~0x003ff000; /* align to 4MB */
-        ptep = pte;
-        page_size = 4096 * 1024;
-        virt_addr = addr & ~0x003fffff;
-    } else {
-        if (!(pde & PG_ACCESSED_MASK)) {
-            pde |= PG_ACCESSED_MASK;
-            stl_raw(pde_ptr, pde);
+        if (pde & PG_PSE_MASK) {
+            /* 2 MB page */
+            page_size = 2048 * 1024;
+            goto handle_big_page;
+        } else {
+            /* 4 KB page */
+            if (!(pde & PG_ACCESSED_MASK)) {
+                pde |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pde_addr, pde);
+            }
+            pte_addr = ((pde & ~0xfff) + (((addr >> 12) & 0x1ff) << 3)) &
+                env->a20_mask;
+            goto handle_4k_page;
         }
-
+    } else {
         /* page directory entry */
-        pte_ptr = phys_ram_base + 
-            (((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask);
-        pte = ldl_raw(pte_ptr);
-        if (!(pte & PG_PRESENT_MASK)) {
+        pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & ~3)) & 
+            env->a20_mask;
+        pde = ldl_phys_aligned(pde_addr);
+        if (!(pde & PG_PRESENT_MASK)) {
             error_code = 0;
             goto do_fault;
         }
-        /* combine pde and pte user and rw protections */
-        ptep = pte & pde;
-        if (is_user) {
-            if (!(ptep & PG_USER_MASK))
-                goto do_fault_protect;
-            if (is_write && !(ptep & PG_RW_MASK))
-                goto do_fault_protect;
+        /* if PSE bit is set, then we use a 4MB page */
+        if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+            page_size = 4096 * 1024;
+        handle_big_page:
+            if (is_user) {
+                if (!(pde & PG_USER_MASK))
+                    goto do_fault_protect;
+                if (is_write && !(pde & PG_RW_MASK))
+                    goto do_fault_protect;
+            } else {
+                if ((env->cr[0] & CR0_WP_MASK) && 
+                    is_write && !(pde & PG_RW_MASK)) 
+                    goto do_fault_protect;
+            }
+            is_dirty = is_write && !(pde & PG_DIRTY_MASK);
+            if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
+                pde |= PG_ACCESSED_MASK;
+                if (is_dirty)
+                    pde |= PG_DIRTY_MASK;
+                stl_phys_aligned(pde_addr, pde);
+            }
+        
+            pte = pde & ~( (page_size - 1) & ~0xfff); /* align to page_size */
+            ptep = pte;
+            virt_addr = addr & ~(page_size - 1);
         } else {
-            if ((env->cr[0] & CR0_WP_MASK) &&
-                is_write && !(ptep & PG_RW_MASK)) 
-                goto do_fault_protect;
-        }
-        is_dirty = is_write && !(pte & PG_DIRTY_MASK);
-        if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
-            pte |= PG_ACCESSED_MASK;
-            if (is_dirty)
-                pte |= PG_DIRTY_MASK;
-            stl_raw(pte_ptr, pte);
+            if (!(pde & PG_ACCESSED_MASK)) {
+                pde |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pde_addr, pde);
+            }
+
+            /* page directory entry */
+            pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & 
+                env->a20_mask;
+        handle_4k_page:
+            pte = ldl_phys_aligned(pte_addr);
+            if (!(pte & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
+            /* combine pde and pte user and rw protections */
+            ptep = pte & pde;
+            if (is_user) {
+                if (!(ptep & PG_USER_MASK))
+                    goto do_fault_protect;
+                if (is_write && !(ptep & PG_RW_MASK))
+                    goto do_fault_protect;
+            } else {
+                if ((env->cr[0] & CR0_WP_MASK) &&
+                    is_write && !(ptep & PG_RW_MASK)) 
+                    goto do_fault_protect;
+            }
+            is_dirty = is_write && !(pte & PG_DIRTY_MASK);
+            if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
+                pte |= PG_ACCESSED_MASK;
+                if (is_dirty)
+                    pte |= PG_DIRTY_MASK;
+                stl_phys_aligned(pte_addr, pte);
+            }
+            page_size = 4096;
+            virt_addr = addr & ~0xfff;
         }
-        page_size = 4096;
-        virt_addr = addr & ~0xfff;
-    }
  
-    /* the page can be put in the TLB */
-    prot = PAGE_READ;
-    if (pte & PG_DIRTY_MASK) {
-        /* only set write access if already dirty... otherwise wait
-           for dirty access */
-        if (is_user) {
-            if (ptep & PG_RW_MASK)
-                prot |= PAGE_WRITE;
-        } else {
-            if (!(env->cr[0] & CR0_WP_MASK) ||
-                (ptep & PG_RW_MASK))
-                prot |= PAGE_WRITE;
+        /* the page can be put in the TLB */
+        prot = PAGE_READ;
+        if (pte & PG_DIRTY_MASK) {
+            /* only set write access if already dirty... otherwise wait
+               for dirty access */
+            if (is_user) {
+                if (ptep & PG_RW_MASK)
+                    prot |= PAGE_WRITE;
+            } else {
+                if (!(env->cr[0] & CR0_WP_MASK) ||
+                    (ptep & PG_RW_MASK))
+                    prot |= PAGE_WRITE;
+            }
         }
     }
-
  do_mapping:
     pte = pte & env->a20_mask;
  
@@ -22,7 +22,7 @@
 #include "exec.h"
  
 /* n must be a constant to be efficient */
-static inline int lshift(int x, int n)
+static inline target_long lshift(target_long x, int n)
 {
     if (n >= 0)
         return x << n;
@@ -80,6 +80,58 @@ static inline int lshift(int x, int n)
 #undef REG
 #undef REGNAME
  
+#ifdef TARGET_X86_64
+
+#define REG (env->regs[8])
+#define REGNAME _R8
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[9])
+#define REGNAME _R9
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[10])
+#define REGNAME _R10
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[11])
+#define REGNAME _R11
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[12])
+#define REGNAME _R12
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[13])
+#define REGNAME _R13
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[14])
+#define REGNAME _R14
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[15])
+#define REGNAME _R15
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#endif
+
 /* operations with flags */
  
 /* update flags with T0 and T1 (add/sub case) */
@@ -170,6 +222,13 @@ void OPPROTO op_bswapl_T0(void)
     T0 = bswap32(T0);
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_bswapq_T0(void)
+{
+    T0 = bswap64(T0);
+}
+#endif
+
 /* multiply/divide */
  
 /* XXX: add eflags optimizations */
@@ -179,7 +238,7 @@ void OPPROTO op_mulb_AL_T0(void)
 {
     unsigned int res;
     res = (uint8_t)EAX * (uint8_t)T0;
-    EAX = (EAX & 0xffff0000) | res;
+    EAX = (EAX & ~0xffff) | res;
     CC_DST = res;
     CC_SRC = (res & 0xff00);
 }
@@ -188,7 +247,7 @@ void OPPROTO op_imulb_AL_T0(void)
 {
     int res;
     res = (int8_t)EAX * (int8_t)T0;
-    EAX = (EAX & 0xffff0000) | (res & 0xffff);
+    EAX = (EAX & ~0xffff) | (res & 0xffff);
     CC_DST = res;
     CC_SRC = (res != (int8_t)res);
 }
@@ -197,8 +256,8 @@ void OPPROTO op_mulw_AX_T0(void)
 {
     unsigned int res;
     res = (uint16_t)EAX * (uint16_t)T0;
-    EAX = (EAX & 0xffff0000) | (res & 0xffff);
-    EDX = (EDX & 0xffff0000) | ((res >> 16) & 0xffff);
+    EAX = (EAX & ~0xffff) | (res & 0xffff);
+    EDX = (EDX & ~0xffff) | ((res >> 16) & 0xffff);
     CC_DST = res;
     CC_SRC = res >> 16;
 }
@@ -207,8 +266,8 @@ void OPPROTO op_imulw_AX_T0(void)
 {
     int res;
     res = (int16_t)EAX * (int16_t)T0;
-    EAX = (EAX & 0xffff0000) | (res & 0xffff);
-    EDX = (EDX & 0xffff0000) | ((res >> 16) & 0xffff);
+    EAX = (EAX & ~0xffff) | (res & 0xffff);
+    EDX = (EDX & ~0xffff) | ((res >> 16) & 0xffff);
     CC_DST = res;
     CC_SRC = (res != (int16_t)res);
 }
@@ -217,10 +276,10 @@ void OPPROTO op_mull_EAX_T0(void)
 {
     uint64_t res;
     res = (uint64_t)((uint32_t)EAX) * (uint64_t)((uint32_t)T0);
-    EAX = res;
-    EDX = res >> 32;
-    CC_DST = res;
-    CC_SRC = res >> 32;
+    EAX = (uint32_t)res;
+    EDX = (uint32_t)(res >> 32);
+    CC_DST = (uint32_t)res;
+    CC_SRC = (uint32_t)(res >> 32);
 }
  
 void OPPROTO op_imull_EAX_T0(void)
@@ -251,6 +310,23 @@ void OPPROTO op_imull_T0_T1(void)
     CC_SRC = (res != (int32_t)res);
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_mulq_EAX_T0(void)
+{
+    helper_mulq_EAX_T0();
+}
+
+void OPPROTO op_imulq_EAX_T0(void)
+{
+    helper_imulq_EAX_T0();
+}
+
+void OPPROTO op_imulq_T0_T1(void)
+{
+    helper_imulq_T0_T1();
+}
+#endif
+
 /* division, flags are undefined */
 /* XXX: add exceptions for overflow */
  
@@ -261,12 +337,11 @@ void OPPROTO op_divb_AL_T0(void)
     num = (EAX & 0xffff);
     den = (T0 & 0xff);
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xff;
     r = (num % den) & 0xff;
-    EAX = (EAX & 0xffff0000) | (r << 8) | q;
+    EAX = (EAX & ~0xffff) | (r << 8) | q;
 }
  
 void OPPROTO op_idivb_AL_T0(void)
@@ -276,12 +351,11 @@ void OPPROTO op_idivb_AL_T0(void)
     num = (int16_t)EAX;
     den = (int8_t)T0;
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xff;
     r = (num % den) & 0xff;
-    EAX = (EAX & 0xffff0000) | (r << 8) | q;
+    EAX = (EAX & ~0xffff) | (r << 8) | q;
 }
  
 void OPPROTO op_divw_AX_T0(void)
@@ -291,13 +365,12 @@ void OPPROTO op_divw_AX_T0(void)
     num = (EAX & 0xffff) | ((EDX & 0xffff) << 16);
     den = (T0 & 0xffff);
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xffff;
     r = (num % den) & 0xffff;
-    EAX = (EAX & 0xffff0000) | q;
-    EDX = (EDX & 0xffff0000) | r;
+    EAX = (EAX & ~0xffff) | q;
+    EDX = (EDX & ~0xffff) | r;
 }
  
 void OPPROTO op_idivw_AX_T0(void)
@@ -307,30 +380,47 @@ void OPPROTO op_idivw_AX_T0(void)
     num = (EAX & 0xffff) | ((EDX & 0xffff) << 16);
     den = (int16_t)T0;
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xffff;
     r = (num % den) & 0xffff;
-    EAX = (EAX & 0xffff0000) | q;
-    EDX = (EDX & 0xffff0000) | r;
+    EAX = (EAX & ~0xffff) | q;
+    EDX = (EDX & ~0xffff) | r;
 }
  
 void OPPROTO op_divl_EAX_T0(void)
 {
-    helper_divl_EAX_T0(PARAM1);
+    helper_divl_EAX_T0();
 }
  
 void OPPROTO op_idivl_EAX_T0(void)
 {
-    helper_idivl_EAX_T0(PARAM1);
+    helper_idivl_EAX_T0();
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_divq_EAX_T0(void)
+{
+    helper_divq_EAX_T0();
+}
+
+void OPPROTO op_idivq_EAX_T0(void)
+{
+    helper_idivq_EAX_T0();
+}
+#endif
+
 /* constant load & misc op */
  
+/* XXX: consistent names */
+void OPPROTO op_movl_T0_imu(void)
+{
+    T0 = (uint32_t)PARAM1;
+}
+
 void OPPROTO op_movl_T0_im(void)
 {
-    T0 = PARAM1;
+    T0 = (int32_t)PARAM1;
 }
  
 void OPPROTO op_addl_T0_im(void)
@@ -353,9 +443,14 @@ void OPPROTO op_movl_T0_T1(void)
     T0 = T1;
 }
  
+void OPPROTO op_movl_T1_imu(void)
+{
+    T1 = (uint32_t)PARAM1;
+}
+
 void OPPROTO op_movl_T1_im(void)
 {
-    T1 = PARAM1;
+    T1 = (int32_t)PARAM1;
 }
  
 void OPPROTO op_addl_T1_im(void)
@@ -370,19 +465,95 @@ void OPPROTO op_movl_T1_A0(void)
  
 void OPPROTO op_movl_A0_im(void)
 {
-    A0 = PARAM1;
+    A0 = (uint32_t)PARAM1;
 }
  
 void OPPROTO op_addl_A0_im(void)
 {
-    A0 += PARAM1;
+    A0 = (uint32_t)(A0 + PARAM1);
+}
+
+void OPPROTO op_movl_A0_seg(void)
+{
+    A0 = (uint32_t)*(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_addl_A0_seg(void)
+{
+    A0 = (uint32_t)(A0 + *(target_ulong *)((char *)env + PARAM1));
 }
  
 void OPPROTO op_addl_A0_AL(void)
 {
-    A0 += (EAX & 0xff);
+    A0 = (uint32_t)(A0 + (EAX & 0xff));
+}
+
+#ifdef WORDS_BIGENDIAN
+typedef union UREG64 {
+    struct { uint16_t v3, v2, v1, v0; } w;
+    struct { uint32_t v1, v0; } l;
+    uint64_t q;
+} UREG64;
+#else
+typedef union UREG64 {
+    struct { uint16_t v0, v1, v2, v3; } w;
+    struct { uint32_t v0, v1; } l;
+    uint64_t q;
+} UREG64;
+#endif
+
+#ifdef TARGET_X86_64
+
+#define PARAMQ1 \
+({\
+    UREG64 __p;\
+    __p.l.v1 = PARAM1;\
+    __p.l.v0 = PARAM2;\
+    __p.q;\
+}) 
+
+void OPPROTO op_movq_T0_im64(void)
+{
+    T0 = PARAMQ1;
 }
  
+void OPPROTO op_movq_A0_im(void)
+{
+    A0 = (int32_t)PARAM1;
+}
+
+void OPPROTO op_movq_A0_im64(void)
+{
+    A0 = PARAMQ1;
+}
+
+void OPPROTO op_addq_A0_im(void)
+{
+    A0 = (A0 + (int32_t)PARAM1);
+}
+
+void OPPROTO op_addq_A0_im64(void)
+{
+    A0 = (A0 + PARAMQ1);
+}
+
+void OPPROTO op_movq_A0_seg(void)
+{
+    A0 = *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_addq_A0_seg(void)
+{
+    A0 += *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_addq_A0_AL(void)
+{
+    A0 = (A0 + (EAX & 0xff));
+}
+
+#endif
+
 void OPPROTO op_andl_A0_ffff(void)
 {
     A0 = A0 & 0xffff;
@@ -401,29 +572,29 @@ void OPPROTO op_andl_A0_ffff(void)
 #include "ops_mem.h"
 #endif
  
-/* used for bit operations */
+/* indirect jump */
  
-void OPPROTO op_add_bitw_A0_T1(void)
+void OPPROTO op_jmp_T0(void)
 {
-    A0 += ((int16_t)T1 >> 4) << 1;
+    EIP = T0;
 }
  
-void OPPROTO op_add_bitl_A0_T1(void)
+void OPPROTO op_movl_eip_im(void)
 {
-    A0 += ((int32_t)T1 >> 5) << 2;
+    EIP = (uint32_t)PARAM1;
 }
  
-/* indirect jump */
-
-void OPPROTO op_jmp_T0(void)
+#ifdef TARGET_X86_64
+void OPPROTO op_movq_eip_im(void)
 {
-    EIP = T0;
+    EIP = (int32_t)PARAM1;
 }
  
-void OPPROTO op_jmp_im(void)
+void OPPROTO op_movq_eip_im64(void)
 {
-    EIP = PARAM1;
+    EIP = PARAMQ1;
 }
+#endif
  
 void OPPROTO op_hlt(void)
 {
@@ -505,11 +676,10 @@ void OPPROTO op_sti_vm(void)
 void OPPROTO op_boundw(void)
 {
     int low, high, v;
-    low = ldsw((uint8_t *)A0);
-    high = ldsw((uint8_t *)A0 + 2);
+    low = ldsw(A0);
+    high = ldsw(A0 + 2);
     v = (int16_t)T0;
     if (v < low || v > high) {
-        EIP = PARAM1;
         raise_exception(EXCP05_BOUND);
     }
     FORCE_RET();
@@ -518,11 +688,10 @@ void OPPROTO op_boundw(void)
 void OPPROTO op_boundl(void)
 {
     int low, high, v;
-    low = ldl((uint8_t *)A0);
-    high = ldl((uint8_t *)A0 + 4);
+    low = ldl(A0);
+    high = ldl(A0 + 4);
     v = T0;
     if (v < low || v > high) {
-        EIP = PARAM1;
         raise_exception(EXCP05_BOUND);
     }
     FORCE_RET();
@@ -533,11 +702,6 @@ void OPPROTO op_cmpxchg8b(void)
     helper_cmpxchg8b();
 }
  
-void OPPROTO op_jmp(void)
-{
-    JUMP_TB(op_jmp, PARAM1, 0, PARAM2);
-}
-
 void OPPROTO op_movl_T0_0(void)
 {
     T0 = 0;
@@ -564,6 +728,14 @@ void OPPROTO op_exit_tb(void)
 #include "ops_template.h"
 #undef SHIFT
  
+#ifdef TARGET_X86_64
+
+#define SHIFT 3
+#include "ops_template.h"
+#undef SHIFT
+
+#endif
+
 /* sign extend */
  
 void OPPROTO op_movsbl_T0_T0(void)
@@ -581,6 +753,11 @@ void OPPROTO op_movswl_T0_T0(void)
     T0 = (int16_t)T0;
 }
  
+void OPPROTO op_movslq_T0_T0(void)
+{
+    T0 = (int32_t)T0;
+}
+
 void OPPROTO op_movzwl_T0_T0(void)
 {
     T0 = (uint16_t)T0;
@@ -591,9 +768,16 @@ void OPPROTO op_movswl_EAX_AX(void)
     EAX = (int16_t)EAX;
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_movslq_RAX_EAX(void)
+{
+    EAX = (int32_t)EAX;
+}
+#endif
+
 void OPPROTO op_movsbw_AX_AL(void)
 {
-    EAX = (EAX & 0xffff0000) | ((int8_t)EAX & 0xffff);
+    EAX = (EAX & ~0xffff) | ((int8_t)EAX & 0xffff);
 }
  
 void OPPROTO op_movslq_EDX_EAX(void)
@@ -603,14 +787,21 @@ void OPPROTO op_movslq_EDX_EAX(void)
  
 void OPPROTO op_movswl_DX_AX(void)
 {
-    EDX = (EDX & 0xffff0000) | (((int16_t)EAX >> 15) & 0xffff);
+    EDX = (EDX & ~0xffff) | (((int16_t)EAX >> 15) & 0xffff);
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO op_movsqo_RDX_RAX(void)
+{
+    EDX = (int64_t)EAX >> 63;
 }
+#endif
  
 /* string ops helpers */
  
 void OPPROTO op_addl_ESI_T0(void)
 {
-    ESI += T0;
+    ESI = (uint32_t)(ESI + T0);
 }
  
 void OPPROTO op_addw_ESI_T0(void)
@@ -620,7 +811,7 @@ void OPPROTO op_addw_ESI_T0(void)
  
 void OPPROTO op_addl_EDI_T0(void)
 {
-    EDI += T0;
+    EDI = (uint32_t)(EDI + T0);
 }
  
 void OPPROTO op_addw_EDI_T0(void)
@@ -630,7 +821,7 @@ void OPPROTO op_addw_EDI_T0(void)
  
 void OPPROTO op_decl_ECX(void)
 {
-    ECX--;
+    ECX = (uint32_t)(ECX - 1);
 }
  
 void OPPROTO op_decw_ECX(void)
@@ -638,6 +829,23 @@ void OPPROTO op_decw_ECX(void)
     ECX = (ECX & ~0xffff) | ((ECX - 1) & 0xffff);
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_addq_ESI_T0(void)
+{
+    ESI = (ESI + T0);
+}
+
+void OPPROTO op_addq_EDI_T0(void)
+{
+    EDI = (EDI + T0);
+}
+
+void OPPROTO op_decq_ECX(void)
+{
+    ECX--;
+}
+#endif
+
 /* push/pop utils */
  
 void op_addl_A0_SS(void)
@@ -647,22 +855,22 @@ void op_addl_A0_SS(void)
  
 void op_subl_A0_2(void)
 {
-    A0 -= 2;
+    A0 = (uint32_t)(A0 - 2);
 }
  
 void op_subl_A0_4(void)
 {
-    A0 -= 4;
+    A0 = (uint32_t)(A0 - 4);
 }
  
 void op_addl_ESP_4(void)
 {
-    ESP += 4;
+    ESP = (uint32_t)(ESP + 4);
 }
  
 void op_addl_ESP_2(void)
 {
-    ESP += 2;
+    ESP = (uint32_t)(ESP + 2);
 }
  
 void op_addw_ESP_4(void)
@@ -677,7 +885,7 @@ void op_addw_ESP_2(void)
  
 void op_addl_ESP_im(void)
 {
-    ESP += PARAM1;
+    ESP = (uint32_t)(ESP + PARAM1);
 }
  
 void op_addw_ESP_im(void)
@@ -685,6 +893,23 @@ void op_addw_ESP_im(void)
     ESP = (ESP & ~0xffff) | ((ESP + PARAM1) & 0xffff);
 }
  
+#ifdef TARGET_X86_64
+void op_subq_A0_8(void)
+{
+    A0 -= 8;
+}
+
+void op_addq_ESP_8(void)
+{
+    ESP += 8;
+}
+
+void op_addq_ESP_im(void)
+{
+    ESP += PARAM1;
+}
+#endif
+
 void OPPROTO op_rdtsc(void)
 {
     helper_rdtsc();
@@ -710,6 +935,18 @@ void OPPROTO op_sysexit(void)
     helper_sysexit();
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_syscall(void)
+{
+    helper_syscall();
+}
+
+void OPPROTO op_sysret(void)
+{
+    helper_sysret(PARAM1);
+}
+#endif
+
 void OPPROTO op_rdmsr(void)
 {
     helper_rdmsr();
@@ -868,7 +1105,7 @@ void OPPROTO op_movl_seg_T0_vm(void)
     /* env->segs[] access */
     sc = (SegmentCache *)((char *)env + PARAM1);
     sc->selector = selector;
-    sc->base = (void *)(selector << 4);
+    sc->base = (selector << 4);
 }
  
 void OPPROTO op_movl_T0_seg(void)
@@ -876,16 +1113,6 @@ void OPPROTO op_movl_T0_seg(void)
     T0 = env->segs[PARAM1].selector;
 }
  
-void OPPROTO op_movl_A0_seg(void)
-{
-    A0 = *(unsigned long *)((char *)env + PARAM1);
-}
-
-void OPPROTO op_addl_A0_seg(void)
-{
-    A0 += *(unsigned long *)((char *)env + PARAM1);
-}
-
 void OPPROTO op_lsl(void)
 {
     helper_lsl();
@@ -1006,6 +1233,26 @@ void OPPROTO op_movl_env_T1(void)
     *(uint32_t *)((char *)env + PARAM1) = T1;
 }
  
+void OPPROTO op_movtl_T0_env(void)
+{
+    T0 = *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_movtl_env_T0(void)
+{
+    *(target_ulong *)((char *)env + PARAM1) = T0;
+}
+
+void OPPROTO op_movtl_T1_env(void)
+{
+    T1 = *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_movtl_env_T1(void)
+{
+    *(target_ulong *)((char *)env + PARAM1) = T1;
+}
+
 void OPPROTO op_clts(void)
 {
     env->cr[0] &= ~CR0_TS_MASK;
@@ -1014,25 +1261,31 @@ void OPPROTO op_clts(void)
  
 /* flags handling */
  
-/* slow jumps cases : in order to avoid calling a function with a
-   pointer (which can generate a stack frame on PowerPC), we use
-   op_setcc to set T0 and then call op_jcc. */
-void OPPROTO op_jcc(void)
+void OPPROTO op_goto_tb0(void)
 {
-    if (T0)
-        JUMP_TB(op_jcc, PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(op_jcc, PARAM1, 1, PARAM3);
-    FORCE_RET();
+    GOTO_TB(op_goto_tb0, 0);
+}
+
+void OPPROTO op_goto_tb1(void)
+{
+    GOTO_TB(op_goto_tb1, 1);
+}
+
+void OPPROTO op_jmp_label(void)
+{
+    GOTO_LABEL_PARAM(1);
 }
  
-void OPPROTO op_jcc_im(void)
+void OPPROTO op_jnz_T0_label(void)
 {
     if (T0)
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
-    FORCE_RET();
+        GOTO_LABEL_PARAM(1);
+}
+
+void OPPROTO op_jz_T0_label(void)
+{
+    if (!T0)
+        GOTO_LABEL_PARAM(1);
 }
  
 /* slow set cases (compute x86 flags) */
@@ -1299,6 +1552,28 @@ CCTable cc_table[CC_OP_NB] = {
     [CC_OP_SARB] = { compute_all_sarb, compute_c_sarl },
     [CC_OP_SARW] = { compute_all_sarw, compute_c_sarl },
     [CC_OP_SARL] = { compute_all_sarl, compute_c_sarl },
+
+#ifdef TARGET_X86_64
+    [CC_OP_MULQ] = { compute_all_mulq, compute_c_mull },
+
+    [CC_OP_ADDQ] = { compute_all_addq, compute_c_addq  },
+
+    [CC_OP_ADCQ] = { compute_all_adcq, compute_c_adcq  },
+
+    [CC_OP_SUBQ] = { compute_all_subq, compute_c_subq  },
+    
+    [CC_OP_SBBQ] = { compute_all_sbbq, compute_c_sbbq  },
+    
+    [CC_OP_LOGICQ] = { compute_all_logicq, compute_c_logicq },
+    
+    [CC_OP_INCQ] = { compute_all_incq, compute_c_incl },
+
+    [CC_OP_DECQ] = { compute_all_decq, compute_c_incl },
+
+    [CC_OP_SHLQ] = { compute_all_shlq, compute_c_shlq },
+
+    [CC_OP_SARQ] = { compute_all_sarq, compute_c_sarl },
+#endif
 };
  
 /* floating point support. Some of the code for complicated x87
@@ -1330,20 +1605,20 @@ double qemu_rint(double x)
 void OPPROTO op_flds_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldl((void *)A0);
+    FP_CONVERT.i32 = ldl(A0);
     FT0 = FP_CONVERT.f;
 #else
-    FT0 = ldfl((void *)A0);
+    FT0 = ldfl(A0);
 #endif
 }
  
 void OPPROTO op_fldl_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = ldq((void *)A0);
+    FP_CONVERT.i64 = ldq(A0);
     FT0 = FP_CONVERT.d;
 #else
-    FT0 = ldfq((void *)A0);
+    FT0 = ldfq(A0);
 #endif
 }
  
@@ -1352,17 +1627,17 @@ void OPPROTO op_fldl_FT0_A0(void)
  
 void helper_fild_FT0_A0(void)
 {
-    FT0 = (CPU86_LDouble)ldsw((void *)A0);
+    FT0 = (CPU86_LDouble)ldsw(A0);
 }
  
 void helper_fildl_FT0_A0(void)
 {
-    FT0 = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    FT0 = (CPU86_LDouble)((int32_t)ldl(A0));
 }
  
 void helper_fildll_FT0_A0(void)
 {
-    FT0 = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    FT0 = (CPU86_LDouble)((int64_t)ldq(A0));
 }
  
 void OPPROTO op_fild_FT0_A0(void)
@@ -1385,30 +1660,30 @@ void OPPROTO op_fildll_FT0_A0(void)
 void OPPROTO op_fild_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldsw((void *)A0);
+    FP_CONVERT.i32 = ldsw(A0);
     FT0 = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    FT0 = (CPU86_LDouble)ldsw((void *)A0);
+    FT0 = (CPU86_LDouble)ldsw(A0);
 #endif
 }
  
 void OPPROTO op_fildl_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = (int32_t) ldl((void *)A0);
+    FP_CONVERT.i32 = (int32_t) ldl(A0);
     FT0 = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    FT0 = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    FT0 = (CPU86_LDouble)((int32_t)ldl(A0));
 #endif
 }
  
 void OPPROTO op_fildll_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = (int64_t) ldq((void *)A0);
+    FP_CONVERT.i64 = (int64_t) ldq(A0);
     FT0 = (CPU86_LDouble)FP_CONVERT.i64;
 #else
-    FT0 = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    FT0 = (CPU86_LDouble)((int64_t)ldq(A0));
 #endif
 }
 #endif
@@ -1420,10 +1695,10 @@ void OPPROTO op_flds_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldl((void *)A0);
+    FP_CONVERT.i32 = ldl(A0);
     env->fpregs[new_fpstt] = FP_CONVERT.f;
 #else
-    env->fpregs[new_fpstt] = ldfl((void *)A0);
+    env->fpregs[new_fpstt] = ldfl(A0);
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1434,10 +1709,10 @@ void OPPROTO op_fldl_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = ldq((void *)A0);
+    FP_CONVERT.i64 = ldq(A0);
     env->fpregs[new_fpstt] = FP_CONVERT.d;
 #else
-    env->fpregs[new_fpstt] = ldfq((void *)A0);
+    env->fpregs[new_fpstt] = ldfq(A0);
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1455,7 +1730,7 @@ void helper_fild_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw((void *)A0);
+    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw(A0);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
@@ -1464,7 +1739,7 @@ void helper_fildl_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl(A0));
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
@@ -1473,7 +1748,7 @@ void helper_fildll_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq(A0));
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
@@ -1500,10 +1775,10 @@ void OPPROTO op_fild_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldsw((void *)A0);
+    FP_CONVERT.i32 = ldsw(A0);
     env->fpregs[new_fpstt] = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw((void *)A0);
+    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw(A0);
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1514,10 +1789,10 @@ void OPPROTO op_fildl_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = (int32_t) ldl((void *)A0);
+    FP_CONVERT.i32 = (int32_t) ldl(A0);
     env->fpregs[new_fpstt] = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl(A0));
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1528,10 +1803,10 @@ void OPPROTO op_fildll_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = (int64_t) ldq((void *)A0);
+    FP_CONVERT.i64 = (int64_t) ldq(A0);
     env->fpregs[new_fpstt] = (CPU86_LDouble)FP_CONVERT.i64;
 #else
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq(A0));
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1545,15 +1820,15 @@ void OPPROTO op_fsts_ST0_A0(void)
 {
 #ifdef USE_FP_CONVERT
     FP_CONVERT.f = (float)ST0;
-    stfl((void *)A0, FP_CONVERT.f);
+    stfl(A0, FP_CONVERT.f);
 #else
-    stfl((void *)A0, (float)ST0);
+    stfl(A0, (float)ST0);
 #endif
 }
  
 void OPPROTO op_fstl_ST0_A0(void)
 {
-    stfq((void *)A0, (double)ST0);
+    stfq(A0, (double)ST0);
 }
  
 void OPPROTO op_fstt_ST0_A0(void)
@@ -1574,7 +1849,7 @@ void OPPROTO op_fist_ST0_A0(void)
     val = lrint(d);
     if (val != (int16_t)val)
         val = -32768;
-    stw((void *)A0, val);
+    stw(A0, val);
 }
  
 void OPPROTO op_fistl_ST0_A0(void)
@@ -1588,7 +1863,7 @@ void OPPROTO op_fistl_ST0_A0(void)
  
     d = ST0;
     val = lrint(d);
-    stl((void *)A0, val);
+    stl(A0, val);
 }
  
 void OPPROTO op_fistll_ST0_A0(void)
@@ -1602,7 +1877,7 @@ void OPPROTO op_fistll_ST0_A0(void)
  
     d = ST0;
     val = llrint(d);
-    stq((void *)A0, val);
+    stq(A0, val);
 }
  
 void OPPROTO op_fbld_ST0_A0(void)
@@ -1934,25 +2209,25 @@ void OPPROTO op_fnstsw_A0(void)
 {
     int fpus;
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
-    stw((void *)A0, fpus);
+    stw(A0, fpus);
 }
  
 void OPPROTO op_fnstsw_EAX(void)
 {
     int fpus;
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
-    EAX = (EAX & 0xffff0000) | fpus;
+    EAX = (EAX & ~0xffff) | fpus;
 }
  
 void OPPROTO op_fnstcw_A0(void)
 {
-    stw((void *)A0, env->fpuc);
+    stw(A0, env->fpuc);
 }
  
 void OPPROTO op_fldcw_A0(void)
 {
     int rnd_type;
-    env->fpuc = lduw((void *)A0);
+    env->fpuc = lduw(A0);
     /* set rounding mode */
     switch(env->fpuc & RC_MASK) {
     default:
@@ -2001,22 +2276,22 @@ void OPPROTO op_fninit(void)
  
 void OPPROTO op_fnstenv_A0(void)
 {
-    helper_fstenv((uint8_t *)A0, PARAM1);
+    helper_fstenv(A0, PARAM1);
 }
  
 void OPPROTO op_fldenv_A0(void)
 {
-    helper_fldenv((uint8_t *)A0, PARAM1);
+    helper_fldenv(A0, PARAM1);
 }
  
 void OPPROTO op_fnsave_A0(void)
 {
-    helper_fsave((uint8_t *)A0, PARAM1);
+    helper_fsave(A0, PARAM1);
 }
  
 void OPPROTO op_frstor_A0(void)
 {
-    helper_frstor((uint8_t *)A0, PARAM1);
+    helper_frstor(A0, PARAM1);
 }
  
 /* threading support */
@@ -2030,3 +2305,30 @@ void OPPROTO op_unlock(void)
     cpu_unlock();
 }
  
+/* SSE support */
+static inline void memcpy16(void *d, void *s)
+{
+    ((uint32_t *)d)[0] = ((uint32_t *)s)[0];
+    ((uint32_t *)d)[1] = ((uint32_t *)s)[1];
+    ((uint32_t *)d)[2] = ((uint32_t *)s)[2];
+    ((uint32_t *)d)[3] = ((uint32_t *)s)[3];
+}
+
+void OPPROTO op_movo(void)
+{
+    /* XXX: badly generated code */
+    XMMReg *d, *s;
+    d = (XMMReg *)((char *)env + PARAM1);
+    s = (XMMReg *)((char *)env + PARAM2);
+    memcpy16(d, s);
+}
+
+void OPPROTO op_fxsave_A0(void)
+{
+    helper_fxsave(A0, PARAM1);
+}
+
+void OPPROTO op_fxrstor_A0(void)
+{
+    helper_fxrstor(A0, PARAM1);
+}
@@ -20,29 +20,56 @@
  */
 void OPPROTO glue(op_movl_A0,REGNAME)(void)
 {
-    A0 = REG;
+    A0 = (uint32_t)REG;
 }
  
 void OPPROTO glue(op_addl_A0,REGNAME)(void)
 {
-    A0 += REG;
+    A0 = (uint32_t)(A0 + REG);
 }
  
 void OPPROTO glue(glue(op_addl_A0,REGNAME),_s1)(void)
 {
-    A0 += REG << 1;
+    A0 = (uint32_t)(A0 + (REG << 1));
 }
  
 void OPPROTO glue(glue(op_addl_A0,REGNAME),_s2)(void)
 {
-    A0 += REG << 2;
+    A0 = (uint32_t)(A0 + (REG << 2));
 }
  
 void OPPROTO glue(glue(op_addl_A0,REGNAME),_s3)(void)
 {
-    A0 += REG << 3;
+    A0 = (uint32_t)(A0 + (REG << 3));
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(op_movq_A0,REGNAME)(void)
+{
+    A0 = REG;
+}
+
+void OPPROTO glue(op_addq_A0,REGNAME)(void)
+{
+    A0 = (A0 + REG);
+}
+
+void OPPROTO glue(glue(op_addq_A0,REGNAME),_s1)(void)
+{
+    A0 = (A0 + (REG << 1));
+}
+
+void OPPROTO glue(glue(op_addq_A0,REGNAME),_s2)(void)
+{
+    A0 = (A0 + (REG << 2));
 }
  
+void OPPROTO glue(glue(op_addq_A0,REGNAME),_s3)(void)
+{
+    A0 = (A0 + (REG << 3));
+}
+#endif
+
 void OPPROTO glue(op_movl_T0,REGNAME)(void)
 {
     T0 = REG;
@@ -65,72 +92,99 @@ void OPPROTO glue(op_movh_T1,REGNAME)(void)
  
 void OPPROTO glue(glue(op_movl,REGNAME),_T0)(void)
 {
-    REG = T0;
+    REG = (uint32_t)T0;
 }
  
 void OPPROTO glue(glue(op_movl,REGNAME),_T1)(void)
 {
-    REG = T1;
+    REG = (uint32_t)T1;
 }
  
 void OPPROTO glue(glue(op_movl,REGNAME),_A0)(void)
 {
+    REG = (uint32_t)A0;
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(glue(op_movq,REGNAME),_T0)(void)
+{
+    REG = T0;
+}
+
+void OPPROTO glue(glue(op_movq,REGNAME),_T1)(void)
+{
+    REG = T1;
+}
+
+void OPPROTO glue(glue(op_movq,REGNAME),_A0)(void)
+{
     REG = A0;
 }
+#endif
  
 /* mov T1 to REG if T0 is true */
 void OPPROTO glue(glue(op_cmovw,REGNAME),_T1_T0)(void)
 {
     if (T0)
-        REG = (REG & 0xffff0000) | (T1 & 0xffff);
+        REG = (REG & ~0xffff) | (T1 & 0xffff);
     FORCE_RET();
 }
  
 void OPPROTO glue(glue(op_cmovl,REGNAME),_T1_T0)(void)
 {
     if (T0)
+        REG = (uint32_t)T1;
+    FORCE_RET();
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(glue(op_cmovq,REGNAME),_T1_T0)(void)
+{
+    if (T0)
         REG = T1;
     FORCE_RET();
 }
+#endif
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movw,REGNAME),_T0)(void)
 {
-    REG = (REG & 0xffff0000) | (T0 & 0xffff);
+    REG = (REG & ~0xffff) | (T0 & 0xffff);
 }
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movw,REGNAME),_T1)(void)
 {
-    REG = (REG & 0xffff0000) | (T1 & 0xffff);
+    REG = (REG & ~0xffff) | (T1 & 0xffff);
 }
  
 /* NOTE: A0 high order bits are ignored */
 void OPPROTO glue(glue(op_movw,REGNAME),_A0)(void)
 {
-    REG = (REG & 0xffff0000) | (A0 & 0xffff);
+    REG = (REG & ~0xffff) | (A0 & 0xffff);
 }
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movb,REGNAME),_T0)(void)
 {
-    REG = (REG & 0xffffff00) | (T0 & 0xff);
+    REG = (REG & ~0xff) | (T0 & 0xff);
 }
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movh,REGNAME),_T0)(void)
 {
-    REG = (REG & 0xffff00ff) | ((T0 & 0xff) << 8);
+    REG = (REG & ~0xff00) | ((T0 & 0xff) << 8);
 }
  
 /* NOTE: T1 high order bits are ignored */
 void OPPROTO glue(glue(op_movb,REGNAME),_T1)(void)
 {
-    REG = (REG & 0xffffff00) | (T1 & 0xff);
+    REG = (REG & ~0xff) | (T1 & 0xff);
 }
  
 /* NOTE: T1 high order bits are ignored */
 void OPPROTO glue(glue(op_movh,REGNAME),_T1)(void)
 {
-    REG = (REG & 0xffff00ff) | ((T1 & 0xff) << 8);
+    REG = (REG & ~0xff00) | ((T1 & 0xff) << 8);
 }
+
 void OPPROTO glue(glue(op_ldub, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldub, MEMSUFFIX)((uint8_t *)A0);
+    T0 = glue(ldub, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsb, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldsb, MEMSUFFIX)((int8_t *)A0);
+    T0 = glue(ldsb, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_lduw, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(lduw, MEMSUFFIX)((uint8_t *)A0);
+    T0 = glue(lduw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsw, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldsw, MEMSUFFIX)((int8_t *)A0);
+    T0 = glue(ldsw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldl, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldl, MEMSUFFIX)((uint8_t *)A0);
+    T0 = (uint32_t)glue(ldl, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldub, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldub, MEMSUFFIX)((uint8_t *)A0);
+    T1 = glue(ldub, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsb, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldsb, MEMSUFFIX)((int8_t *)A0);
+    T1 = glue(ldsb, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_lduw, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(lduw, MEMSUFFIX)((uint8_t *)A0);
+    T1 = glue(lduw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsw, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldsw, MEMSUFFIX)((int8_t *)A0);
+    T1 = glue(ldsw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldl, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldl, MEMSUFFIX)((uint8_t *)A0);
+    T1 = glue(ldl, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_stb, MEMSUFFIX), _T0_A0)(void)
 {
-    glue(stb, MEMSUFFIX)((uint8_t *)A0, T0);
+    glue(stb, MEMSUFFIX)(A0, T0);
 }
  
 void OPPROTO glue(glue(op_stw, MEMSUFFIX), _T0_A0)(void)
 {
-    glue(stw, MEMSUFFIX)((uint8_t *)A0, T0);
+    glue(stw, MEMSUFFIX)(A0, T0);
 }
  
 void OPPROTO glue(glue(op_stl, MEMSUFFIX), _T0_A0)(void)
 {
-    glue(stl, MEMSUFFIX)((uint8_t *)A0, T0);
+    glue(stl, MEMSUFFIX)(A0, T0);
 }
  
 #if 0
 void OPPROTO glue(glue(op_stb, MEMSUFFIX), _T1_A0)(void)
 {
-    glue(stb, MEMSUFFIX)((uint8_t *)A0, T1);
+    glue(stb, MEMSUFFIX)(A0, T1);
 }
 #endif
  
 void OPPROTO glue(glue(op_stw, MEMSUFFIX), _T1_A0)(void)
 {
-    glue(stw, MEMSUFFIX)((uint8_t *)A0, T1);
+    glue(stw, MEMSUFFIX)(A0, T1);
 }
  
 void OPPROTO glue(glue(op_stl, MEMSUFFIX), _T1_A0)(void)
 {
-    glue(stl, MEMSUFFIX)((uint8_t *)A0, T1);
+    glue(stl, MEMSUFFIX)(A0, T1);
 }
  
+/* SSE support */
+void OPPROTO glue(glue(op_ldo, MEMSUFFIX), _env_A0)(void)
+{
+    XMMReg *p;
+    p = (XMMReg *)((char *)env + PARAM1);
+    /* XXX: host endianness ? */
+    p->u.q[0] = glue(ldq, MEMSUFFIX)(A0);
+    p->u.q[1] = glue(ldq, MEMSUFFIX)(A0 + 8);
+}
+
+void OPPROTO glue(glue(op_sto, MEMSUFFIX), _env_A0)(void)
+{
+    XMMReg *p;
+    p = (XMMReg *)((char *)env + PARAM1);
+    /* XXX: host endianness ? */
+    glue(stq, MEMSUFFIX)(A0, p->u.q[0]);
+    glue(stq, MEMSUFFIX)(A0 + 8, p->u.q[1]);
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(glue(op_ldsl, MEMSUFFIX), _T0_A0)(void)
+{
+    T0 = (int32_t)glue(ldl, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_ldsl, MEMSUFFIX), _T1_A0)(void)
+{
+    T1 = (int32_t)glue(ldl, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_ldq, MEMSUFFIX), _T0_A0)(void)
+{
+    T0 = glue(ldq, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_ldq, MEMSUFFIX), _T1_A0)(void)
+{
+    T1 = glue(ldq, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_stq, MEMSUFFIX), _T0_A0)(void)
+{
+    glue(stq, MEMSUFFIX)(A0, T0);
+}
+
+void OPPROTO glue(glue(op_stq, MEMSUFFIX), _T1_A0)(void)
+{
+    glue(stq, MEMSUFFIX)(A0, T1);
+}
+#endif
+
 #undef MEMSUFFIX
@@ -20,7 +20,12 @@
  */
 #define DATA_BITS (1 << (3 + SHIFT))
 #define SHIFT_MASK (DATA_BITS - 1)
-#define SIGN_MASK (1 << (DATA_BITS - 1))
+#define SIGN_MASK (((target_ulong)1) << (DATA_BITS - 1))
+#if DATA_BITS <= 32
+#define SHIFT1_MASK 0x1f
+#else
+#define SHIFT1_MASK 0x3f
+#endif
  
 #if DATA_BITS == 8
 #define SUFFIX b
@@ -37,6 +42,11 @@
 #define DATA_TYPE uint32_t
 #define DATA_STYPE int32_t
 #define DATA_MASK 0xffffffff
+#elif DATA_BITS == 64
+#define SUFFIX q
+#define DATA_TYPE uint64_t
+#define DATA_STYPE int64_t
+#define DATA_MASK 0xffffffffffffffff
 #else
 #error unhandled operand size
 #endif
@@ -46,7 +56,7 @@
 static int glue(compute_all_add, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_SRC;
     src2 = CC_DST - CC_SRC;
     cf = (DATA_TYPE)CC_DST < (DATA_TYPE)src1;
@@ -60,7 +70,8 @@ static int glue(compute_all_add, SUFFIX)(void)
  
 static int glue(compute_c_add, SUFFIX)(void)
 {
-    int src1, cf;
+    int cf;
+    target_long src1;
     src1 = CC_SRC;
     cf = (DATA_TYPE)CC_DST < (DATA_TYPE)src1;
     return cf;
@@ -69,7 +80,7 @@ static int glue(compute_c_add, SUFFIX)(void)
 static int glue(compute_all_adc, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_SRC;
     src2 = CC_DST - CC_SRC - 1;
     cf = (DATA_TYPE)CC_DST <= (DATA_TYPE)src1;
@@ -83,7 +94,8 @@ static int glue(compute_all_adc, SUFFIX)(void)
  
 static int glue(compute_c_adc, SUFFIX)(void)
 {
-    int src1, cf;
+    int cf;
+    target_long src1;
     src1 = CC_SRC;
     cf = (DATA_TYPE)CC_DST <= (DATA_TYPE)src1;
     return cf;
@@ -92,7 +104,7 @@ static int glue(compute_c_adc, SUFFIX)(void)
 static int glue(compute_all_sub, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 < (DATA_TYPE)src2;
@@ -106,7 +118,8 @@ static int glue(compute_all_sub, SUFFIX)(void)
  
 static int glue(compute_c_sub, SUFFIX)(void)
 {
-    int src1, src2, cf;
+    int cf;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 < (DATA_TYPE)src2;
@@ -116,7 +129,7 @@ static int glue(compute_c_sub, SUFFIX)(void)
 static int glue(compute_all_sbb, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC + 1;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 <= (DATA_TYPE)src2;
@@ -130,7 +143,8 @@ static int glue(compute_all_sbb, SUFFIX)(void)
  
 static int glue(compute_c_sbb, SUFFIX)(void)
 {
-    int src1, src2, cf;
+    int cf;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC + 1;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 <= (DATA_TYPE)src2;
@@ -157,7 +171,7 @@ static int glue(compute_c_logic, SUFFIX)(void)
 static int glue(compute_all_inc, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST - 1;
     src2 = 1;
     cf = CC_SRC;
@@ -179,7 +193,7 @@ static int glue(compute_c_inc, SUFFIX)(void)
 static int glue(compute_all_dec, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + 1;
     src2 = 1;
     cf = CC_SRC;
@@ -187,7 +201,7 @@ static int glue(compute_all_dec, SUFFIX)(void)
     af = (CC_DST ^ src1 ^ src2) & 0x10;
     zf = ((DATA_TYPE)CC_DST == 0) << 6;
     sf = lshift(CC_DST, 8 - DATA_BITS) & 0x80;
-    of = ((CC_DST & DATA_MASK) == ((uint32_t)SIGN_MASK - 1)) << 11;
+    of = ((CC_DST & DATA_MASK) == ((target_ulong)SIGN_MASK - 1)) << 11;
     return cf | pf | af | zf | sf | of;
 }
  
@@ -256,71 +270,66 @@ static int glue(compute_all_mul, SUFFIX)(void)
  
 void OPPROTO glue(op_jb_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_TYPE)src1 < (DATA_TYPE)src2)
-        JUMP_TB(glue(op_jb_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jb_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jz_sub, SUFFIX)(void)
 {
     if ((DATA_TYPE)CC_DST == 0)
-        JUMP_TB(glue(op_jz_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jz_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
+    FORCE_RET();
+}
+
+void OPPROTO glue(op_jnz_sub, SUFFIX)(void)
+{
+    if ((DATA_TYPE)CC_DST != 0)
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jbe_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_TYPE)src1 <= (DATA_TYPE)src2)
-        JUMP_TB(glue(op_jbe_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jbe_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_js_sub, SUFFIX)(void)
 {
     if (CC_DST & SIGN_MASK)
-        JUMP_TB(glue(op_js_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_js_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jl_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_STYPE)src1 < (DATA_STYPE)src2)
-        JUMP_TB(glue(op_jl_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jl_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jle_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_STYPE)src1 <= (DATA_STYPE)src2)
-        JUMP_TB(glue(op_jle_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jle_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
@@ -330,50 +339,33 @@ void OPPROTO glue(op_jle_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_loopnz, SUFFIX)(void)
 {
-    unsigned int tmp;
     int eflags;
     eflags = cc_table[CC_OP].compute_all();
-    tmp = (ECX - 1) & DATA_MASK;
-    ECX = (ECX & ~DATA_MASK) | tmp;
-    if (tmp != 0 && !(eflags & CC_Z))
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX != 0 && !(eflags & CC_Z))
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_loopz, SUFFIX)(void)
 {
-    unsigned int tmp;
     int eflags;
     eflags = cc_table[CC_OP].compute_all();
-    tmp = (ECX - 1) & DATA_MASK;
-    ECX = (ECX & ~DATA_MASK) | tmp;
-    if (tmp != 0 && (eflags & CC_Z))
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX != 0 && (eflags & CC_Z))
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
-void OPPROTO glue(op_loop, SUFFIX)(void)
+void OPPROTO glue(op_jz_ecx, SUFFIX)(void)
 {
-    unsigned int tmp;
-    tmp = (ECX - 1) & DATA_MASK;
-    ECX = (ECX & ~DATA_MASK) | tmp;
-    if (tmp != 0)
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX == 0)
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
-void OPPROTO glue(op_jecxz, SUFFIX)(void)
+void OPPROTO glue(op_jnz_ecx, SUFFIX)(void)
 {
-    if ((DATA_TYPE)ECX == 0)
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX != 0)
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
@@ -383,7 +375,7 @@ void OPPROTO glue(op_jecxz, SUFFIX)(void)
  
 void OPPROTO glue(op_setb_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -397,7 +389,7 @@ void OPPROTO glue(op_setz_T0_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_setbe_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -411,7 +403,7 @@ void OPPROTO glue(op_sets_T0_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_setl_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -420,7 +412,7 @@ void OPPROTO glue(op_setl_T0_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_setle_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -432,7 +424,7 @@ void OPPROTO glue(op_setle_T0_sub, SUFFIX)(void)
 void OPPROTO glue(glue(op_shl, SUFFIX), _T0_T1)(void)
 {
     int count;
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
     T0 = T0 << count;
     FORCE_RET();
 }
@@ -440,7 +432,7 @@ void OPPROTO glue(glue(op_shl, SUFFIX), _T0_T1)(void)
 void OPPROTO glue(glue(op_shr, SUFFIX), _T0_T1)(void)
 {
     int count;
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
     T0 &= DATA_MASK;
     T0 = T0 >> count;
     FORCE_RET();
@@ -448,8 +440,10 @@ void OPPROTO glue(glue(op_shr, SUFFIX), _T0_T1)(void)
  
 void OPPROTO glue(glue(op_sar, SUFFIX), _T0_T1)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     src = (DATA_STYPE)T0;
     T0 = src >> count;
     FORCE_RET();
@@ -484,7 +478,7 @@ void OPPROTO glue(glue(op_bts, SUFFIX), _T0_T1_cc)(void)
     int count;
     count = T1 & SHIFT_MASK;
     T1 = T0 >> count;
-    T0 |= (1 << count);
+    T0 |= (((target_long)1) << count);
 }
  
 void OPPROTO glue(glue(op_btr, SUFFIX), _T0_T1_cc)(void)
@@ -492,7 +486,7 @@ void OPPROTO glue(glue(op_btr, SUFFIX), _T0_T1_cc)(void)
     int count;
     count = T1 & SHIFT_MASK;
     T1 = T0 >> count;
-    T0 &= ~(1 << count);
+    T0 &= ~(((target_long)1) << count);
 }
  
 void OPPROTO glue(glue(op_btc, SUFFIX), _T0_T1_cc)(void)
@@ -500,12 +494,19 @@ void OPPROTO glue(glue(op_btc, SUFFIX), _T0_T1_cc)(void)
     int count;
     count = T1 & SHIFT_MASK;
     T1 = T0 >> count;
-    T0 ^= (1 << count);
+    T0 ^= (((target_long)1) << count);
+}
+
+void OPPROTO glue(glue(op_add_bit, SUFFIX), _A0_T1)(void)
+{
+    A0 += ((DATA_STYPE)T1 >> (3 + SHIFT)) << SHIFT;
 }
  
 void OPPROTO glue(glue(op_bsf, SUFFIX), _T0_cc)(void)
 {
-    int res, count;
+    int count;
+    target_long res;
+    
     res = T0 & DATA_MASK;
     if (res != 0) {
         count = 0;
@@ -523,7 +524,9 @@ void OPPROTO glue(glue(op_bsf, SUFFIX), _T0_cc)(void)
  
 void OPPROTO glue(glue(op_bsr, SUFFIX), _T0_cc)(void)
 {
-    int res, count;
+    int count;
+    target_long res;
+
     res = T0 & DATA_MASK;
     if (res != 0) {
         count = DATA_BITS - 1;
@@ -555,70 +558,8 @@ void OPPROTO glue(op_movl_T0_Dshift, SUFFIX)(void)
     T0 = DF << SHIFT;
 }
  
-void OPPROTO glue(op_string_jz_sub, SUFFIX)(void)
-{
-    if ((DATA_TYPE)CC_DST == 0)
-        JUMP_TB2(glue(op_string_jz_sub, SUFFIX), PARAM1, 3);
-    FORCE_RET();
-}
-
-void OPPROTO glue(op_string_jnz_sub, SUFFIX)(void)
-{
-    if ((DATA_TYPE)CC_DST != 0)
-        JUMP_TB2(glue(op_string_jnz_sub, SUFFIX), PARAM1, 3);
-    FORCE_RET();
-}
-
-void OPPROTO glue(glue(op_string_jz_sub, SUFFIX), _im)(void)
-{
-    if ((DATA_TYPE)CC_DST == 0) {
-        EIP = PARAM1;
-        if (env->eflags & TF_MASK) {
-            raise_exception(EXCP01_SSTP);
-        }
-        T0 = 0;
-        EXIT_TB();
-    }
-    FORCE_RET();
-}
-
-void OPPROTO glue(glue(op_string_jnz_sub, SUFFIX), _im)(void)
-{
-    if ((DATA_TYPE)CC_DST != 0) {
-        EIP = PARAM1;
-        if (env->eflags & TF_MASK) {
-            raise_exception(EXCP01_SSTP);
-        }
-        T0 = 0;
-        EXIT_TB();
-    }
-    FORCE_RET();
-}
-
-#if DATA_BITS >= 16
-void OPPROTO glue(op_jz_ecx, SUFFIX)(void)
-{
-    if ((DATA_TYPE)ECX == 0)
-        JUMP_TB(glue(op_jz_ecx, SUFFIX), PARAM1, 1, PARAM2);
-    FORCE_RET();
-}
-
-void OPPROTO glue(glue(op_jz_ecx, SUFFIX), _im)(void)
-{
-    if ((DATA_TYPE)ECX == 0) {
-        EIP = PARAM1;
-        if (env->eflags & TF_MASK) {
-            raise_exception(EXCP01_SSTP);
-        }
-        T0 = 0;
-        EXIT_TB();
-    }
-    FORCE_RET();
-}
-#endif
-
 /* port I/O */
-
+#if DATA_BITS <= 32
 void OPPROTO glue(glue(op_out, SUFFIX), _T0_T1)(void)
 {
     glue(cpu_out, SUFFIX)(env, T0, T1 & DATA_MASK);
@@ -648,9 +589,11 @@ void OPPROTO glue(glue(op_check_io, SUFFIX), _DX)(void)
 {
     glue(glue(check_io, SUFFIX), _DX)();
 }
+#endif
  
 #undef DATA_BITS
 #undef SHIFT_MASK
+#undef SHIFT1_MASK
 #undef SIGN_MASK
 #undef DATA_TYPE
 #undef DATA_STYPE
@@ -28,6 +28,8 @@
 #define MEM_SUFFIX w_raw
 #elif DATA_BITS == 32
 #define MEM_SUFFIX l_raw
+#elif DATA_BITS == 64
+#define MEM_SUFFIX q_raw
 #endif
  
 #elif MEM_WRITE == 1
@@ -38,6 +40,8 @@
 #define MEM_SUFFIX w_kernel
 #elif DATA_BITS == 32
 #define MEM_SUFFIX l_kernel
+#elif DATA_BITS == 64
+#define MEM_SUFFIX q_kernel
 #endif
  
 #elif MEM_WRITE == 2
@@ -48,6 +52,8 @@
 #define MEM_SUFFIX w_user
 #elif DATA_BITS == 32
 #define MEM_SUFFIX l_user
+#elif DATA_BITS == 64
+#define MEM_SUFFIX q_user
 #endif
  
 #else
@@ -64,14 +70,16 @@
  
 void OPPROTO glue(glue(op_rol, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
+    int count;
+    target_long src;
+
     count = T1 & SHIFT_MASK;
     if (count) {
         src = T0;
         T0 &= DATA_MASK;
         T0 = (T0 << count) | (T0 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #else
         /* gcc 3.2 workaround. This is really a bug in gcc. */
         asm volatile("" : : "r" (T0));
@@ -86,14 +94,16 @@ void OPPROTO glue(glue(op_rol, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_ror, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
+    int count;
+    target_long src;
+
     count = T1 & SHIFT_MASK;
     if (count) {
         src = T0;
         T0 &= DATA_MASK;
         T0 = (T0 >> count) | (T0 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #else
         /* gcc 3.2 workaround. This is really a bug in gcc. */
         asm volatile("" : : "r" (T0));
@@ -114,7 +124,7 @@ void OPPROTO glue(glue(op_rol, MEM_SUFFIX), _T0_T1)(void)
         T0 &= DATA_MASK;
         T0 = (T0 << count) | (T0 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     }
     FORCE_RET();
@@ -128,7 +138,7 @@ void OPPROTO glue(glue(op_ror, MEM_SUFFIX), _T0_T1)(void)
         T0 &= DATA_MASK;
         T0 = (T0 >> count) | (T0 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     }
     FORCE_RET();
@@ -136,10 +146,11 @@ void OPPROTO glue(glue(op_ror, MEM_SUFFIX), _T0_T1)(void)
  
 void OPPROTO glue(glue(op_rcl, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, res, eflags;
-    unsigned int src;
+    int count, eflags;
+    target_ulong src;
+    target_long res;
  
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
 #if DATA_BITS == 16
     count = rclw_table[count];
 #elif DATA_BITS == 8
@@ -154,7 +165,7 @@ void OPPROTO glue(glue(op_rcl, MEM_SUFFIX), _T0_T1_cc)(void)
             res |= T0 >> (DATA_BITS + 1 - count);
         T0 = res;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = (eflags & ~(CC_C | CC_O)) |
             (lshift(src ^ T0, 11 - (DATA_BITS - 1)) & CC_O) | 
@@ -166,10 +177,11 @@ void OPPROTO glue(glue(op_rcl, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_rcr, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, res, eflags;
-    unsigned int src;
+    int count, eflags;
+    target_ulong src;
+    target_long res;
  
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
 #if DATA_BITS == 16
     count = rclw_table[count];
 #elif DATA_BITS == 8
@@ -184,7 +196,7 @@ void OPPROTO glue(glue(op_rcr, MEM_SUFFIX), _T0_T1_cc)(void)
             res |= T0 << (DATA_BITS + 1 - count);
         T0 = res;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = (eflags & ~(CC_C | CC_O)) |
             (lshift(src ^ T0, 11 - (DATA_BITS - 1)) & CC_O) | 
@@ -196,13 +208,15 @@ void OPPROTO glue(glue(op_rcr, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_shl, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     if (count) {
         src = (DATA_TYPE)T0 << (count - 1);
         T0 = T0 << count;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = src;
         CC_DST = T0;
@@ -213,14 +227,16 @@ void OPPROTO glue(glue(op_shl, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_shr, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     if (count) {
         T0 &= DATA_MASK;
         src = T0 >> (count - 1);
         T0 = T0 >> count;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = src;
         CC_DST = T0;
@@ -231,14 +247,16 @@ void OPPROTO glue(glue(op_shr, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_sar, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     if (count) {
         src = (DATA_STYPE)T0;
         T0 = src >> count;
         src = src >> (count - 1);
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = src;
         CC_DST = T0;
@@ -262,7 +280,7 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_im_cc)(void)
         res |= T1 << (count - 16);
     T0 = res >> 16;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -282,7 +300,7 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
           res |= T1 << (count - 16);
         T0 = res >> 16;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -304,7 +322,7 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_im_cc)(void)
         res |= T1 << (32 - count);
     T0 = res;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -325,7 +343,7 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
             res |= T1 << (32 - count);
         T0 = res;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -335,17 +353,19 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
 }
 #endif
  
-#if DATA_BITS == 32
+#if DATA_BITS >= 32
 void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_im_cc)(void)
 {
-    int count, tmp;
+    int count;
+    target_long tmp;
+
     count = PARAM1;
     T0 &= DATA_MASK;
     T1 &= DATA_MASK;
     tmp = T0 << (count - 1);
     T0 = (T0 << count) | (T1 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -353,15 +373,17 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_im_cc)(void)
  
 void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
 {
-    int count, tmp;
-    count = ECX & 0x1f;
+    int count;
+    target_long tmp;
+
+    count = ECX & SHIFT1_MASK;
     if (count) {
         T0 &= DATA_MASK;
         T1 &= DATA_MASK;
         tmp = T0 << (count - 1);
         T0 = (T0 << count) | (T1 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -372,14 +394,16 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
  
 void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_im_cc)(void)
 {
-    int count, tmp;
+    int count;
+    target_long tmp;
+
     count = PARAM1;
     T0 &= DATA_MASK;
     T1 &= DATA_MASK;
     tmp = T0 >> (count - 1);
     T0 = (T0 >> count) | (T1 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -388,15 +412,17 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_im_cc)(void)
  
 void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
 {
-    int count, tmp;
-    count = ECX & 0x1f;
+    int count;
+    target_long tmp;
+
+    count = ECX & SHIFT1_MASK;
     if (count) {
         T0 &= DATA_MASK;
         T1 &= DATA_MASK;
         tmp = T0 >> (count - 1);
         T0 = (T0 >> count) | (T1 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -414,11 +440,11 @@ void OPPROTO glue(glue(op_adc, MEM_SUFFIX), _T0_T1_cc)(void)
     cf = cc_table[CC_OP].compute_c();
     T0 = T0 + T1 + cf;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = T1;
     CC_DST = T0;
-    CC_OP = CC_OP_ADDB + SHIFT + cf * 3;
+    CC_OP = CC_OP_ADDB + SHIFT + cf * 4;
 }
  
 void OPPROTO glue(glue(op_sbb, MEM_SUFFIX), _T0_T1_cc)(void)
@@ -427,23 +453,23 @@ void OPPROTO glue(glue(op_sbb, MEM_SUFFIX), _T0_T1_cc)(void)
     cf = cc_table[CC_OP].compute_c();
     T0 = T0 - T1 - cf;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = T1;
     CC_DST = T0;
-    CC_OP = CC_OP_SUBB + SHIFT + cf * 3;
+    CC_OP = CC_OP_SUBB + SHIFT + cf * 4;
 }
  
 void OPPROTO glue(glue(op_cmpxchg, MEM_SUFFIX), _T0_T1_EAX_cc)(void)
 {
-    unsigned int src, dst;
+    target_ulong src, dst;
  
     src = T0;
     dst = EAX - T0;
     if ((DATA_TYPE)dst == 0) {
         T0 = T1;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     } else {
         EAX = (EAX & ~DATA_MASK) | (T0 & DATA_MASK);
@@ -57,7 +57,7 @@ typedef struct DisasContext {
     int override; /* -1 if no override */
     int prefix;
     int aflag, dflag;
-    uint8_t *pc; /* pc = eip + cs_base */
+    target_ulong pc; /* pc = eip + cs_base */
     int is_jmp; /* 1 = means jump (stop translation), 2 means CPU
                    static state change (stop translation) */
     /* code output */
@@ -65,7 +65,7 @@ typedef struct DisasContext {
     uint8_t *gen_code_start;
  
     /* current block context */
-    uint8_t *cs_base; /* base of CS segment */
+    target_ulong cs_base; /* base of CS segment */
     int pe;     /* protected mode */
     int code32; /* 32 bit code segment */
     int f_st;   /* currently unused */
@@ -277,7 +277,7 @@ static inline uint32_t insn_get(DisasContext *s, int ot)
    be stopped.  */
 static int disas_insn(DisasContext *s)
 {
-    uint8_t *pc_start, *pc_tmp, *pc_start_insn;
+    target_ulong pc_start, pc_tmp, pc_start_insn;
     int b, prefixes, aflag, dflag, next_eip, val;
     int ot;
     int modrm, mod, op, rm;
@@ -789,6 +789,8 @@ static int disas_insn(DisasContext *s)
                 break;
             case 0x1e: /* fcomi */
                 break;
+            case 0x28: /* ffree sti */
+                break;
             case 0x2a: /* fst sti */
                 break;
             case 0x2b: /* fstp sti */
@@ -1176,9 +1178,9 @@ static inline int gen_intermediate_code_internal(CPUState *env,
                                                  uint8_t *tc_ptr)
 {
     DisasContext dc1, *dc = &dc1;
-    uint8_t *pc_insn, *pc_start, *gen_code_end;
+    target_ulong pc_insn, pc_start, cs_base;
+    uint8_t *gen_code_end;
     int flags, ret;
-    uint8_t *cs_base;
  
     if (env->nb_breakpoints > 0 ||
         env->singlestep_enabled)
@@ -1197,8 +1199,8 @@ static inline int gen_intermediate_code_internal(CPUState *env,
     dc->gen_code_start = gen_code_ptr;
  
     /* generate intermediate code */
-    pc_start = (uint8_t *)tb->pc;
-    cs_base = (uint8_t *)tb->cs_base;
+    pc_start = tb->pc;
+    cs_base = tb->cs_base;
     dc->pc = pc_start;
     dc->cs_base = cs_base;
     dc->pe = (flags >> HF_PE_SHIFT) & 1;
@@ -1249,7 +1251,7 @@ static inline int gen_intermediate_code_internal(CPUState *env,
         fprintf(logfile, "IN: COPY: %s fpu=%d\n", 
                 lookup_symbol(pc_start),
                 tb->cflags & CF_TB_FP_USED ? 1 : 0);
-	disas(logfile, pc_start, dc->pc - pc_start, 0, !dc->code32);
+	target_disas(logfile, pc_start, dc->pc - pc_start, !dc->code32);
         fprintf(logfile, "\n");
     }
 #endif
@@ -39,18 +39,45 @@ static uint32_t *gen_opparam_ptr;
 #define PREFIX_DATA   0x08
 #define PREFIX_ADR    0x10
  
+#ifdef TARGET_X86_64
+#define X86_64_ONLY(x) x
+#define X86_64_DEF(x...) x
+#define CODE64(s) ((s)->code64)
+#define REX_X(s) ((s)->rex_x)
+#define REX_B(s) ((s)->rex_b)
+/* XXX: gcc generates push/pop in some opcodes, so we cannot use them */
+#if 1
+#define BUGGY_64(x) NULL
+#endif
+#else
+#define X86_64_ONLY(x) NULL
+#define X86_64_DEF(x...)
+#define CODE64(s) 0
+#define REX_X(s) 0
+#define REX_B(s) 0
+#endif
+
+#ifdef TARGET_X86_64
+static int x86_64_hregs;
+#endif
+
 typedef struct DisasContext {
     /* current insn context */
     int override; /* -1 if no override */
     int prefix;
     int aflag, dflag;
-    uint8_t *pc; /* pc = eip + cs_base */
+    target_ulong pc; /* pc = eip + cs_base */
     int is_jmp; /* 1 = means jump (stop translation), 2 means CPU
                    static state change (stop translation) */
     /* current block context */
-    uint8_t *cs_base; /* base of CS segment */
+    target_ulong cs_base; /* base of CS segment */
     int pe;     /* protected mode */
     int code32; /* 32 bit code segment */
+#ifdef TARGET_X86_64
+    int lma;    /* long mode active */
+    int code64; /* 64 bit code segment */
+    int rex_x, rex_b;
+#endif
     int ss32;   /* 32 bit stack segment */
     int cc_op;  /* current CC operation */
     int addseg; /* non zero if either DS/ES/SS have a non zero base */
@@ -65,10 +92,13 @@ typedef struct DisasContext {
     int flags; /* all execution flags */
     struct TranslationBlock *tb;
     int popl_esp_hack; /* for correct popl with esp base handling */
+    int rip_offset; /* only used in x86_64, but left for simplicity */
+    int cpuid_features;
 } DisasContext;
  
 static void gen_eob(DisasContext *s);
-static void gen_jmp(DisasContext *s, unsigned int eip);
+static void gen_jmp(DisasContext *s, target_ulong eip);
+static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num);
  
 /* i386 arith/logic operations */
 enum {
@@ -121,103 +151,182 @@ enum {
     OR_EBP,
     OR_ESI,
     OR_EDI,
-    OR_TMP0,    /* temporary operand register */
+
+    OR_TMP0 = 16,    /* temporary operand register */
     OR_TMP1,
     OR_A0, /* temporary register used when doing address evaluation */
-    OR_ZERO, /* fixed zero register */
-    NB_OREGS,
 };
  
-static GenOpFunc *gen_op_mov_reg_T0[3][8] = {
+#ifdef TARGET_X86_64
+
+#define NB_OP_SIZES 4
+
+#define DEF_REGS(prefix, suffix) \
+  prefix ## EAX ## suffix,\
+  prefix ## ECX ## suffix,\
+  prefix ## EDX ## suffix,\
+  prefix ## EBX ## suffix,\
+  prefix ## ESP ## suffix,\
+  prefix ## EBP ## suffix,\
+  prefix ## ESI ## suffix,\
+  prefix ## EDI ## suffix,\
+  prefix ## R8 ## suffix,\
+  prefix ## R9 ## suffix,\
+  prefix ## R10 ## suffix,\
+  prefix ## R11 ## suffix,\
+  prefix ## R12 ## suffix,\
+  prefix ## R13 ## suffix,\
+  prefix ## R14 ## suffix,\
+  prefix ## R15 ## suffix,
+
+#define DEF_BREGS(prefixb, prefixh, suffix)             \
+                                                        \
+static void prefixb ## ESP ## suffix ## _wrapper(void)  \
+{                                                       \
+    if (x86_64_hregs)                                 \
+        prefixb ## ESP ## suffix ();                    \
+    else                                                \
+        prefixh ## EAX ## suffix ();                    \
+}                                                       \
+                                                        \
+static void prefixb ## EBP ## suffix ## _wrapper(void)  \
+{                                                       \
+    if (x86_64_hregs)                                 \
+        prefixb ## EBP ## suffix ();                    \
+    else                                                \
+        prefixh ## ECX ## suffix ();                    \
+}                                                       \
+                                                        \
+static void prefixb ## ESI ## suffix ## _wrapper(void)  \
+{                                                       \
+    if (x86_64_hregs)                                 \
+        prefixb ## ESI ## suffix ();                    \
+    else                                                \
+        prefixh ## EDX ## suffix ();                    \
+}                                                       \
+                                                        \
+static void prefixb ## EDI ## suffix ## _wrapper(void)  \
+{                                                       \
+    if (x86_64_hregs)                                 \
+        prefixb ## EDI ## suffix ();                    \
+    else                                                \
+        prefixh ## EBX ## suffix ();                    \
+}
+
+DEF_BREGS(gen_op_movb_, gen_op_movh_, _T0)
+DEF_BREGS(gen_op_movb_, gen_op_movh_, _T1)
+DEF_BREGS(gen_op_movl_T0_, gen_op_movh_T0_, )
+DEF_BREGS(gen_op_movl_T1_, gen_op_movh_T1_, )
+
+#else /* !TARGET_X86_64 */
+
+#define NB_OP_SIZES 3
+
+#define DEF_REGS(prefix, suffix) \
+  prefix ## EAX ## suffix,\
+  prefix ## ECX ## suffix,\
+  prefix ## EDX ## suffix,\
+  prefix ## EBX ## suffix,\
+  prefix ## ESP ## suffix,\
+  prefix ## EBP ## suffix,\
+  prefix ## ESI ## suffix,\
+  prefix ## EDI ## suffix,
+
+#endif /* !TARGET_X86_64 */
+
+static GenOpFunc *gen_op_mov_reg_T0[NB_OP_SIZES][CPU_NB_REGS] = {
     [OT_BYTE] = {
         gen_op_movb_EAX_T0,
         gen_op_movb_ECX_T0,
         gen_op_movb_EDX_T0,
         gen_op_movb_EBX_T0,
+#ifdef TARGET_X86_64
+        gen_op_movb_ESP_T0_wrapper,
+        gen_op_movb_EBP_T0_wrapper,
+        gen_op_movb_ESI_T0_wrapper,
+        gen_op_movb_EDI_T0_wrapper,
+        gen_op_movb_R8_T0,
+        gen_op_movb_R9_T0,
+        gen_op_movb_R10_T0,
+        gen_op_movb_R11_T0,
+        gen_op_movb_R12_T0,
+        gen_op_movb_R13_T0,
+        gen_op_movb_R14_T0,
+        gen_op_movb_R15_T0,
+#else
         gen_op_movh_EAX_T0,
         gen_op_movh_ECX_T0,
         gen_op_movh_EDX_T0,
         gen_op_movh_EBX_T0,
+#endif
     },
     [OT_WORD] = {
-        gen_op_movw_EAX_T0,
-        gen_op_movw_ECX_T0,
-        gen_op_movw_EDX_T0,
-        gen_op_movw_EBX_T0,
-        gen_op_movw_ESP_T0,
-        gen_op_movw_EBP_T0,
-        gen_op_movw_ESI_T0,
-        gen_op_movw_EDI_T0,
+        DEF_REGS(gen_op_movw_, _T0)
     },
     [OT_LONG] = {
-        gen_op_movl_EAX_T0,
-        gen_op_movl_ECX_T0,
-        gen_op_movl_EDX_T0,
-        gen_op_movl_EBX_T0,
-        gen_op_movl_ESP_T0,
-        gen_op_movl_EBP_T0,
-        gen_op_movl_ESI_T0,
-        gen_op_movl_EDI_T0,
+        DEF_REGS(gen_op_movl_, _T0)
     },
+#ifdef TARGET_X86_64
+    [OT_QUAD] = {
+        DEF_REGS(gen_op_movq_, _T0)
+    },
+#endif
 };
  
-static GenOpFunc *gen_op_mov_reg_T1[3][8] = {
+static GenOpFunc *gen_op_mov_reg_T1[NB_OP_SIZES][CPU_NB_REGS] = {
     [OT_BYTE] = {
         gen_op_movb_EAX_T1,
         gen_op_movb_ECX_T1,
         gen_op_movb_EDX_T1,
         gen_op_movb_EBX_T1,
+#ifdef TARGET_X86_64
+        gen_op_movb_ESP_T1_wrapper,
+        gen_op_movb_EBP_T1_wrapper,
+        gen_op_movb_ESI_T1_wrapper,
+        gen_op_movb_EDI_T1_wrapper,
+        gen_op_movb_R8_T1,
+        gen_op_movb_R9_T1,
+        gen_op_movb_R10_T1,
+        gen_op_movb_R11_T1,
+        gen_op_movb_R12_T1,
+        gen_op_movb_R13_T1,
+        gen_op_movb_R14_T1,
+        gen_op_movb_R15_T1,
+#else
         gen_op_movh_EAX_T1,
         gen_op_movh_ECX_T1,
         gen_op_movh_EDX_T1,
         gen_op_movh_EBX_T1,
+#endif
     },
     [OT_WORD] = {
-        gen_op_movw_EAX_T1,
-        gen_op_movw_ECX_T1,
-        gen_op_movw_EDX_T1,
-        gen_op_movw_EBX_T1,
-        gen_op_movw_ESP_T1,
-        gen_op_movw_EBP_T1,
-        gen_op_movw_ESI_T1,
-        gen_op_movw_EDI_T1,
+        DEF_REGS(gen_op_movw_, _T1)
     },
     [OT_LONG] = {
-        gen_op_movl_EAX_T1,
-        gen_op_movl_ECX_T1,
-        gen_op_movl_EDX_T1,
-        gen_op_movl_EBX_T1,
-        gen_op_movl_ESP_T1,
-        gen_op_movl_EBP_T1,
-        gen_op_movl_ESI_T1,
-        gen_op_movl_EDI_T1,
+        DEF_REGS(gen_op_movl_, _T1)
+    },
+#ifdef TARGET_X86_64
+    [OT_QUAD] = {
+        DEF_REGS(gen_op_movq_, _T1)
     },
+#endif
 };
  
-static GenOpFunc *gen_op_mov_reg_A0[2][8] = {
+static GenOpFunc *gen_op_mov_reg_A0[NB_OP_SIZES - 1][CPU_NB_REGS] = {
     [0] = {
-        gen_op_movw_EAX_A0,
-        gen_op_movw_ECX_A0,
-        gen_op_movw_EDX_A0,
-        gen_op_movw_EBX_A0,
-        gen_op_movw_ESP_A0,
-        gen_op_movw_EBP_A0,
-        gen_op_movw_ESI_A0,
-        gen_op_movw_EDI_A0,
+        DEF_REGS(gen_op_movw_, _A0)
     },
     [1] = {
-        gen_op_movl_EAX_A0,
-        gen_op_movl_ECX_A0,
-        gen_op_movl_EDX_A0,
-        gen_op_movl_EBX_A0,
-        gen_op_movl_ESP_A0,
-        gen_op_movl_EBP_A0,
-        gen_op_movl_ESI_A0,
-        gen_op_movl_EDI_A0,
+        DEF_REGS(gen_op_movl_, _A0)
+    },
+#ifdef TARGET_X86_64
+    [2] = {
+        DEF_REGS(gen_op_movq_, _A0)
     },
+#endif
 };
  
-static GenOpFunc *gen_op_mov_TN_reg[3][2][8] = 
+static GenOpFunc *gen_op_mov_TN_reg[NB_OP_SIZES][2][CPU_NB_REGS] = 
 {
     [OT_BYTE] = {
         {
@@ -225,144 +334,133 @@ static GenOpFunc *gen_op_mov_TN_reg[3][2][8] =
             gen_op_movl_T0_ECX,
             gen_op_movl_T0_EDX,
             gen_op_movl_T0_EBX,
+#ifdef TARGET_X86_64
+            gen_op_movl_T0_ESP_wrapper,
+            gen_op_movl_T0_EBP_wrapper,
+            gen_op_movl_T0_ESI_wrapper,
+            gen_op_movl_T0_EDI_wrapper,
+            gen_op_movl_T0_R8,
+            gen_op_movl_T0_R9,
+            gen_op_movl_T0_R10,
+            gen_op_movl_T0_R11,
+            gen_op_movl_T0_R12,
+            gen_op_movl_T0_R13,
+            gen_op_movl_T0_R14,
+            gen_op_movl_T0_R15,
+#else
             gen_op_movh_T0_EAX,
             gen_op_movh_T0_ECX,
             gen_op_movh_T0_EDX,
             gen_op_movh_T0_EBX,
+#endif
         },
         {
             gen_op_movl_T1_EAX,
             gen_op_movl_T1_ECX,
             gen_op_movl_T1_EDX,
             gen_op_movl_T1_EBX,
+#ifdef TARGET_X86_64
+            gen_op_movl_T1_ESP_wrapper,
+            gen_op_movl_T1_EBP_wrapper,
+            gen_op_movl_T1_ESI_wrapper,
+            gen_op_movl_T1_EDI_wrapper,
+            gen_op_movl_T1_R8,
+            gen_op_movl_T1_R9,
+            gen_op_movl_T1_R10,
+            gen_op_movl_T1_R11,
+            gen_op_movl_T1_R12,
+            gen_op_movl_T1_R13,
+            gen_op_movl_T1_R14,
+            gen_op_movl_T1_R15,
+#else
             gen_op_movh_T1_EAX,
             gen_op_movh_T1_ECX,
             gen_op_movh_T1_EDX,
             gen_op_movh_T1_EBX,
+#endif
         },
     },
     [OT_WORD] = {
         {
-            gen_op_movl_T0_EAX,
-            gen_op_movl_T0_ECX,
-            gen_op_movl_T0_EDX,
-            gen_op_movl_T0_EBX,
-            gen_op_movl_T0_ESP,
-            gen_op_movl_T0_EBP,
-            gen_op_movl_T0_ESI,
-            gen_op_movl_T0_EDI,
+            DEF_REGS(gen_op_movl_T0_, )
         },
         {
-            gen_op_movl_T1_EAX,
-            gen_op_movl_T1_ECX,
-            gen_op_movl_T1_EDX,
-            gen_op_movl_T1_EBX,
-            gen_op_movl_T1_ESP,
-            gen_op_movl_T1_EBP,
-            gen_op_movl_T1_ESI,
-            gen_op_movl_T1_EDI,
+            DEF_REGS(gen_op_movl_T1_, )
         },
     },
     [OT_LONG] = {
         {
-            gen_op_movl_T0_EAX,
-            gen_op_movl_T0_ECX,
-            gen_op_movl_T0_EDX,
-            gen_op_movl_T0_EBX,
-            gen_op_movl_T0_ESP,
-            gen_op_movl_T0_EBP,
-            gen_op_movl_T0_ESI,
-            gen_op_movl_T0_EDI,
+            DEF_REGS(gen_op_movl_T0_, )
         },
         {
-            gen_op_movl_T1_EAX,
-            gen_op_movl_T1_ECX,
-            gen_op_movl_T1_EDX,
-            gen_op_movl_T1_EBX,
-            gen_op_movl_T1_ESP,
-            gen_op_movl_T1_EBP,
-            gen_op_movl_T1_ESI,
-            gen_op_movl_T1_EDI,
+            DEF_REGS(gen_op_movl_T1_, )
         },
     },
+#ifdef TARGET_X86_64
+    [OT_QUAD] = {
+        {
+            DEF_REGS(gen_op_movl_T0_, )
+        },
+        {
+            DEF_REGS(gen_op_movl_T1_, )
+        },
+    },
+#endif
 };
  
-static GenOpFunc *gen_op_movl_A0_reg[8] = {
-    gen_op_movl_A0_EAX,
-    gen_op_movl_A0_ECX,
-    gen_op_movl_A0_EDX,
-    gen_op_movl_A0_EBX,
-    gen_op_movl_A0_ESP,
-    gen_op_movl_A0_EBP,
-    gen_op_movl_A0_ESI,
-    gen_op_movl_A0_EDI,
+static GenOpFunc *gen_op_movl_A0_reg[CPU_NB_REGS] = {
+    DEF_REGS(gen_op_movl_A0_, )
 };
  
-static GenOpFunc *gen_op_addl_A0_reg_sN[4][8] = {
+static GenOpFunc *gen_op_addl_A0_reg_sN[4][CPU_NB_REGS] = {
     [0] = {
-        gen_op_addl_A0_EAX,
-        gen_op_addl_A0_ECX,
-        gen_op_addl_A0_EDX,
-        gen_op_addl_A0_EBX,
-        gen_op_addl_A0_ESP,
-        gen_op_addl_A0_EBP,
-        gen_op_addl_A0_ESI,
-        gen_op_addl_A0_EDI,
+        DEF_REGS(gen_op_addl_A0_, )
     },
     [1] = {
-        gen_op_addl_A0_EAX_s1,
-        gen_op_addl_A0_ECX_s1,
-        gen_op_addl_A0_EDX_s1,
-        gen_op_addl_A0_EBX_s1,
-        gen_op_addl_A0_ESP_s1,
-        gen_op_addl_A0_EBP_s1,
-        gen_op_addl_A0_ESI_s1,
-        gen_op_addl_A0_EDI_s1,
+        DEF_REGS(gen_op_addl_A0_, _s1)
     },
     [2] = {
-        gen_op_addl_A0_EAX_s2,
-        gen_op_addl_A0_ECX_s2,
-        gen_op_addl_A0_EDX_s2,
-        gen_op_addl_A0_EBX_s2,
-        gen_op_addl_A0_ESP_s2,
-        gen_op_addl_A0_EBP_s2,
-        gen_op_addl_A0_ESI_s2,
-        gen_op_addl_A0_EDI_s2,
+        DEF_REGS(gen_op_addl_A0_, _s2)
     },
     [3] = {
-        gen_op_addl_A0_EAX_s3,
-        gen_op_addl_A0_ECX_s3,
-        gen_op_addl_A0_EDX_s3,
-        gen_op_addl_A0_EBX_s3,
-        gen_op_addl_A0_ESP_s3,
-        gen_op_addl_A0_EBP_s3,
-        gen_op_addl_A0_ESI_s3,
-        gen_op_addl_A0_EDI_s3,
+        DEF_REGS(gen_op_addl_A0_, _s3)
     },
 };
  
-static GenOpFunc *gen_op_cmov_reg_T1_T0[2][8] = {
+#ifdef TARGET_X86_64
+static GenOpFunc *gen_op_movq_A0_reg[CPU_NB_REGS] = {
+    DEF_REGS(gen_op_movq_A0_, )
+};
+
+static GenOpFunc *gen_op_addq_A0_reg_sN[4][CPU_NB_REGS] = {
     [0] = {
-        gen_op_cmovw_EAX_T1_T0,
-        gen_op_cmovw_ECX_T1_T0,
-        gen_op_cmovw_EDX_T1_T0,
-        gen_op_cmovw_EBX_T1_T0,
-        gen_op_cmovw_ESP_T1_T0,
-        gen_op_cmovw_EBP_T1_T0,
-        gen_op_cmovw_ESI_T1_T0,
-        gen_op_cmovw_EDI_T1_T0,
+        DEF_REGS(gen_op_addq_A0_, )
     },
     [1] = {
-        gen_op_cmovl_EAX_T1_T0,
-        gen_op_cmovl_ECX_T1_T0,
-        gen_op_cmovl_EDX_T1_T0,
-        gen_op_cmovl_EBX_T1_T0,
-        gen_op_cmovl_ESP_T1_T0,
-        gen_op_cmovl_EBP_T1_T0,
-        gen_op_cmovl_ESI_T1_T0,
-        gen_op_cmovl_EDI_T1_T0,
+        DEF_REGS(gen_op_addq_A0_, _s1)
+    },
+    [2] = {
+        DEF_REGS(gen_op_addq_A0_, _s2)
+    },
+    [3] = {
+        DEF_REGS(gen_op_addq_A0_, _s3)
     },
 };
+#endif
+
+static GenOpFunc *gen_op_cmov_reg_T1_T0[NB_OP_SIZES - 1][CPU_NB_REGS] = {
+    [0] = {
+        DEF_REGS(gen_op_cmovw_, _T1_T0)
+    },
+    [1] = {
+        DEF_REGS(gen_op_cmovl_, _T1_T0)
+    },
+#ifdef TARGET_X86_64
+    [2] = {
+        DEF_REGS(gen_op_cmovq_, _T1_T0)
+    },
+#endif
+};
  
 static GenOpFunc *gen_op_arith_T0_T1_cc[8] = {
     NULL,
@@ -387,13 +485,17 @@ static GenOpFunc *gen_op_arith_T0_T1_cc[8] = {
     {\
         gen_op_adcl ## SUFFIX ## _T0_T1_cc,\
         gen_op_sbbl ## SUFFIX ## _T0_T1_cc,\
+    },\
+    {\
+        X86_64_ONLY(gen_op_adcq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_sbbq ## SUFFIX ## _T0_T1_cc),\
     },
  
-static GenOpFunc *gen_op_arithc_T0_T1_cc[3][2] = {
+static GenOpFunc *gen_op_arithc_T0_T1_cc[4][2] = {
     DEF_ARITHC( )
 };
  
-static GenOpFunc *gen_op_arithc_mem_T0_T1_cc[9][2] = {
+static GenOpFunc *gen_op_arithc_mem_T0_T1_cc[3 * 4][2] = {
     DEF_ARITHC(_raw)
 #ifndef CONFIG_USER_ONLY
     DEF_ARITHC(_kernel)
@@ -415,14 +517,14 @@ static const int cc_op_arithb[8] = {
 #define DEF_CMPXCHG(SUFFIX)\
     gen_op_cmpxchgb ## SUFFIX ## _T0_T1_EAX_cc,\
     gen_op_cmpxchgw ## SUFFIX ## _T0_T1_EAX_cc,\
-    gen_op_cmpxchgl ## SUFFIX ## _T0_T1_EAX_cc,
-
+    gen_op_cmpxchgl ## SUFFIX ## _T0_T1_EAX_cc,\
+    X86_64_ONLY(gen_op_cmpxchgq ## SUFFIX ## _T0_T1_EAX_cc),
  
-static GenOpFunc *gen_op_cmpxchg_T0_T1_EAX_cc[3] = {
+static GenOpFunc *gen_op_cmpxchg_T0_T1_EAX_cc[4] = {
     DEF_CMPXCHG( )
 };
  
-static GenOpFunc *gen_op_cmpxchg_mem_T0_T1_EAX_cc[9] = {
+static GenOpFunc *gen_op_cmpxchg_mem_T0_T1_EAX_cc[3 * 4] = {
     DEF_CMPXCHG(_raw)
 #ifndef CONFIG_USER_ONLY
     DEF_CMPXCHG(_kernel)
@@ -460,13 +562,23 @@ static GenOpFunc *gen_op_cmpxchg_mem_T0_T1_EAX_cc[9] = {
         gen_op_shrl ## SUFFIX ## _T0_T1_cc,\
         gen_op_shll ## SUFFIX ## _T0_T1_cc,\
         gen_op_sarl ## SUFFIX ## _T0_T1_cc,\
+    },\
+    {\
+        X86_64_ONLY(gen_op_rolq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_rorq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_rclq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_rcrq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_shlq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_shrq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_shlq ## SUFFIX ## _T0_T1_cc),\
+        X86_64_ONLY(gen_op_sarq ## SUFFIX ## _T0_T1_cc),\
     },
  
-static GenOpFunc *gen_op_shift_T0_T1_cc[3][8] = {
+static GenOpFunc *gen_op_shift_T0_T1_cc[4][8] = {
     DEF_SHIFT( )
 };
  
-static GenOpFunc *gen_op_shift_mem_T0_T1_cc[9][8] = {
+static GenOpFunc *gen_op_shift_mem_T0_T1_cc[3 * 4][8] = {
     DEF_SHIFT(_raw)
 #ifndef CONFIG_USER_ONLY
     DEF_SHIFT(_kernel)
@@ -486,18 +598,19 @@ static GenOpFunc *gen_op_shift_mem_T0_T1_cc[9][8] = {
     {\
         gen_op_shldl ## SUFFIX ## _T0_T1_ ## op ## _cc,\
         gen_op_shrdl ## SUFFIX ## _T0_T1_ ## op ## _cc,\
+    },\
+    {\
     },
  
-
-static GenOpFunc1 *gen_op_shiftd_T0_T1_im_cc[3][2] = {
+static GenOpFunc1 *gen_op_shiftd_T0_T1_im_cc[4][2] = {
     DEF_SHIFTD(, im)
 };
  
-static GenOpFunc *gen_op_shiftd_T0_T1_ECX_cc[3][2] = {
+static GenOpFunc *gen_op_shiftd_T0_T1_ECX_cc[4][2] = {
     DEF_SHIFTD(, ECX)
 };
  
-static GenOpFunc1 *gen_op_shiftd_mem_T0_T1_im_cc[9][2] = {
+static GenOpFunc1 *gen_op_shiftd_mem_T0_T1_im_cc[3 * 4][2] = {
     DEF_SHIFTD(_raw, im)
 #ifndef CONFIG_USER_ONLY
     DEF_SHIFTD(_kernel, im)
@@ -505,7 +618,7 @@ static GenOpFunc1 *gen_op_shiftd_mem_T0_T1_im_cc[9][2] = {
 #endif
 };
  
-static GenOpFunc *gen_op_shiftd_mem_T0_T1_ECX_cc[9][2] = {
+static GenOpFunc *gen_op_shiftd_mem_T0_T1_ECX_cc[3 * 4][2] = {
     DEF_SHIFTD(_raw, ECX)
 #ifndef CONFIG_USER_ONLY
     DEF_SHIFTD(_kernel, ECX)
@@ -513,7 +626,7 @@ static GenOpFunc *gen_op_shiftd_mem_T0_T1_ECX_cc[9][2] = {
 #endif
 };
  
-static GenOpFunc *gen_op_btx_T0_T1_cc[2][4] = {
+static GenOpFunc *gen_op_btx_T0_T1_cc[3][4] = {
     [0] = {
         gen_op_btw_T0_T1_cc,
         gen_op_btsw_T0_T1_cc,
@@ -526,9 +639,23 @@ static GenOpFunc *gen_op_btx_T0_T1_cc[2][4] = {
         gen_op_btrl_T0_T1_cc,
         gen_op_btcl_T0_T1_cc,
     },
+#ifdef TARGET_X86_64
+    [2] = {
+        gen_op_btq_T0_T1_cc,
+        gen_op_btsq_T0_T1_cc,
+        gen_op_btrq_T0_T1_cc,
+        gen_op_btcq_T0_T1_cc,
+    },
+#endif
+};
+
+static GenOpFunc *gen_op_add_bit_A0_T1[3] = {
+    gen_op_add_bitw_A0_T1,
+    gen_op_add_bitl_A0_T1,
+    X86_64_ONLY(gen_op_add_bitq_A0_T1),
 };
  
-static GenOpFunc *gen_op_bsx_T0_cc[2][2] = {
+static GenOpFunc *gen_op_bsx_T0_cc[3][2] = {
     [0] = {
         gen_op_bsfw_T0_cc,
         gen_op_bsrw_T0_cc,
@@ -537,109 +664,158 @@ static GenOpFunc *gen_op_bsx_T0_cc[2][2] = {
         gen_op_bsfl_T0_cc,
         gen_op_bsrl_T0_cc,
     },
+#ifdef TARGET_X86_64
+    [2] = {
+        gen_op_bsfq_T0_cc,
+        gen_op_bsrq_T0_cc,
+    },
+#endif
 };
  
-static GenOpFunc *gen_op_lds_T0_A0[3 * 3] = {
+static GenOpFunc *gen_op_lds_T0_A0[3 * 4] = {
     gen_op_ldsb_raw_T0_A0,
     gen_op_ldsw_raw_T0_A0,
+    X86_64_ONLY(gen_op_ldsl_raw_T0_A0),
     NULL,
 #ifndef CONFIG_USER_ONLY
     gen_op_ldsb_kernel_T0_A0,
     gen_op_ldsw_kernel_T0_A0,
+    X86_64_ONLY(gen_op_ldsl_kernel_T0_A0),
     NULL,
  
     gen_op_ldsb_user_T0_A0,
     gen_op_ldsw_user_T0_A0,
+    X86_64_ONLY(gen_op_ldsl_user_T0_A0),
     NULL,
 #endif
 };
  
-static GenOpFunc *gen_op_ldu_T0_A0[3 * 3] = {
+static GenOpFunc *gen_op_ldu_T0_A0[3 * 4] = {
     gen_op_ldub_raw_T0_A0,
     gen_op_lduw_raw_T0_A0,
     NULL,
+    NULL,
  
 #ifndef CONFIG_USER_ONLY
     gen_op_ldub_kernel_T0_A0,
     gen_op_lduw_kernel_T0_A0,
     NULL,
+    NULL,
  
     gen_op_ldub_user_T0_A0,
     gen_op_lduw_user_T0_A0,
     NULL,
+    NULL,
 #endif
 };
  
 /* sign does not matter, except for lidt/lgdt call (TODO: fix it) */
-static GenOpFunc *gen_op_ld_T0_A0[3 * 3] = {
+static GenOpFunc *gen_op_ld_T0_A0[3 * 4] = {
     gen_op_ldub_raw_T0_A0,
     gen_op_lduw_raw_T0_A0,
     gen_op_ldl_raw_T0_A0,
+    X86_64_ONLY(gen_op_ldq_raw_T0_A0),
  
 #ifndef CONFIG_USER_ONLY
     gen_op_ldub_kernel_T0_A0,
     gen_op_lduw_kernel_T0_A0,
     gen_op_ldl_kernel_T0_A0,
+    X86_64_ONLY(gen_op_ldq_kernel_T0_A0),
  
     gen_op_ldub_user_T0_A0,
     gen_op_lduw_user_T0_A0,
     gen_op_ldl_user_T0_A0,
+    X86_64_ONLY(gen_op_ldq_user_T0_A0),
 #endif
 };
  
-static GenOpFunc *gen_op_ld_T1_A0[3 * 3] = {
+static GenOpFunc *gen_op_ld_T1_A0[3 * 4] = {
     gen_op_ldub_raw_T1_A0,
     gen_op_lduw_raw_T1_A0,
     gen_op_ldl_raw_T1_A0,
+    X86_64_ONLY(gen_op_ldq_raw_T1_A0),
  
 #ifndef CONFIG_USER_ONLY
     gen_op_ldub_kernel_T1_A0,
     gen_op_lduw_kernel_T1_A0,
     gen_op_ldl_kernel_T1_A0,
+    X86_64_ONLY(gen_op_ldq_kernel_T1_A0),
  
     gen_op_ldub_user_T1_A0,
     gen_op_lduw_user_T1_A0,
     gen_op_ldl_user_T1_A0,
+    X86_64_ONLY(gen_op_ldq_user_T1_A0),
 #endif
 };
  
-static GenOpFunc *gen_op_st_T0_A0[3 * 3] = {
+static GenOpFunc *gen_op_st_T0_A0[3 * 4] = {
     gen_op_stb_raw_T0_A0,
     gen_op_stw_raw_T0_A0,
     gen_op_stl_raw_T0_A0,
+    X86_64_ONLY(gen_op_stq_raw_T0_A0),
  
 #ifndef CONFIG_USER_ONLY
     gen_op_stb_kernel_T0_A0,
     gen_op_stw_kernel_T0_A0,
     gen_op_stl_kernel_T0_A0,
+    X86_64_ONLY(gen_op_stq_kernel_T0_A0),
  
     gen_op_stb_user_T0_A0,
     gen_op_stw_user_T0_A0,
     gen_op_stl_user_T0_A0,
+    X86_64_ONLY(gen_op_stq_user_T0_A0),
 #endif
 };
  
-static GenOpFunc *gen_op_st_T1_A0[3 * 3] = {
+static GenOpFunc *gen_op_st_T1_A0[3 * 4] = {
     NULL,
     gen_op_stw_raw_T1_A0,
     gen_op_stl_raw_T1_A0,
+    X86_64_ONLY(gen_op_stq_raw_T1_A0),
  
 #ifndef CONFIG_USER_ONLY
     NULL,
     gen_op_stw_kernel_T1_A0,
     gen_op_stl_kernel_T1_A0,
+    X86_64_ONLY(gen_op_stq_kernel_T1_A0),
  
     NULL,
     gen_op_stw_user_T1_A0,
     gen_op_stl_user_T1_A0,
+    X86_64_ONLY(gen_op_stq_user_T1_A0),
 #endif
 };
  
+static inline void gen_jmp_im(target_ulong pc)
+{
+#ifdef TARGET_X86_64
+    if (pc == (uint32_t)pc) {
+        gen_op_movl_eip_im(pc);
+    } else if (pc == (int32_t)pc) {
+        gen_op_movq_eip_im(pc);
+    } else {
+        gen_op_movq_eip_im64(pc >> 32, pc);
+    }
+#else
+    gen_op_movl_eip_im(pc);
+#endif
+}
+
 static inline void gen_string_movl_A0_ESI(DisasContext *s)
 {
     int override;
  
     override = s->override;
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        if (override >= 0) {
+            gen_op_movq_A0_seg(offsetof(CPUX86State,segs[override].base));
+            gen_op_addq_A0_reg_sN[0][R_ESI]();
+        } else {
+            gen_op_movq_A0_reg[R_ESI]();
+        }
+    } else
+#endif
     if (s->aflag) {
         /* 32 bit address */
         if (s->addseg && override < 0)
@@ -662,6 +838,11 @@ static inline void gen_string_movl_A0_ESI(DisasContext *s)
  
 static inline void gen_string_movl_A0_EDI(DisasContext *s)
 {
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_movq_A0_reg[R_EDI]();
+    } else
+#endif
     if (s->aflag) {
         if (s->addseg) {
             gen_op_movl_A0_seg(offsetof(CPUX86State,segs[R_ES].base));
@@ -676,58 +857,43 @@ static inline void gen_string_movl_A0_EDI(DisasContext *s)
     }
 }
  
-static GenOpFunc *gen_op_movl_T0_Dshift[3] = {
+static GenOpFunc *gen_op_movl_T0_Dshift[4] = {
     gen_op_movl_T0_Dshiftb,
     gen_op_movl_T0_Dshiftw,
     gen_op_movl_T0_Dshiftl,
+    X86_64_ONLY(gen_op_movl_T0_Dshiftq),
 };
  
-static GenOpFunc2 *gen_op_jz_ecx[2] = {
-    gen_op_jz_ecxw,
-    gen_op_jz_ecxl,
+static GenOpFunc1 *gen_op_jnz_ecx[3] = {
+    gen_op_jnz_ecxw,
+    gen_op_jnz_ecxl,
+    X86_64_ONLY(gen_op_jnz_ecxq),
 };
  
-static GenOpFunc1 *gen_op_jz_ecx_im[2] = {
-    gen_op_jz_ecxw_im,
-    gen_op_jz_ecxl_im,
+static GenOpFunc1 *gen_op_jz_ecx[3] = {
+    gen_op_jz_ecxw,
+    gen_op_jz_ecxl,
+    X86_64_ONLY(gen_op_jz_ecxq),
 };
  
-static GenOpFunc *gen_op_dec_ECX[2] = {
+static GenOpFunc *gen_op_dec_ECX[3] = {
     gen_op_decw_ECX,
     gen_op_decl_ECX,
+    X86_64_ONLY(gen_op_decq_ECX),
 };
  
-#ifdef USE_DIRECT_JUMP
-typedef GenOpFunc GenOpFuncTB2;
-#define gen_op_string_jnz_sub(nz, ot, tb) gen_op_string_jnz_sub2[nz][ot]()
-#else
-typedef GenOpFunc1 GenOpFuncTB2;
-#define gen_op_string_jnz_sub(nz, ot, tb) gen_op_string_jnz_sub2[nz][ot](tb)
-#endif
-
-static GenOpFuncTB2 *gen_op_string_jnz_sub2[2][3] = {
+static GenOpFunc1 *gen_op_string_jnz_sub[2][4] = {
     {
-        gen_op_string_jnz_subb,
-        gen_op_string_jnz_subw,
-        gen_op_string_jnz_subl,
+        gen_op_jnz_subb,
+        gen_op_jnz_subw,
+        gen_op_jnz_subl,
+        X86_64_ONLY(gen_op_jnz_subq),
     },
     {
-        gen_op_string_jz_subb,
-        gen_op_string_jz_subw,
-        gen_op_string_jz_subl,
-    },
-};
-
-static GenOpFunc1 *gen_op_string_jnz_sub_im[2][3] = {
-    {
-        gen_op_string_jnz_subb_im,
-        gen_op_string_jnz_subw_im,
-        gen_op_string_jnz_subl_im,
-    },
-    {
-        gen_op_string_jz_subb_im,
-        gen_op_string_jz_subw_im,
-        gen_op_string_jz_subl_im,
+        gen_op_jz_subb,
+        gen_op_jz_subw,
+        gen_op_jz_subl,
+        X86_64_ONLY(gen_op_jz_subq),
     },
 };
  
@@ -767,12 +933,12 @@ static GenOpFunc *gen_check_io_DX[3] = {
     gen_op_check_iol_DX,
 };
  
-static void gen_check_io(DisasContext *s, int ot, int use_dx, int cur_eip)
+static void gen_check_io(DisasContext *s, int ot, int use_dx, target_ulong cur_eip)
 {
     if (s->pe && (s->cpl > s->iopl || s->vm86)) {
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        gen_op_jmp_im(cur_eip);
+        gen_jmp_im(cur_eip);
         if (use_dx)
             gen_check_io_DX[ot]();
         else
@@ -787,6 +953,12 @@ static inline void gen_movs(DisasContext *s, int ot)
     gen_string_movl_A0_EDI(s);
     gen_op_st_T0_A0[ot + s->mem_index]();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_ESI_T0();
+        gen_op_addq_EDI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_ESI_T0();
         gen_op_addl_EDI_T0();
@@ -804,15 +976,19 @@ static inline void gen_update_cc_op(DisasContext *s)
     }
 }
  
-static inline void gen_jz_ecx_string(DisasContext *s, unsigned int next_eip)
+/* XXX: does not work with gdbstub "ice" single step - not a
+   serious problem */
+static int gen_jz_ecx_string(DisasContext *s, target_ulong next_eip)
 {
-    if (s->jmp_opt) {
-        gen_op_jz_ecx[s->aflag]((long)s->tb, next_eip);
-    } else {
-        /* XXX: does not work with gdbstub "ice" single step - not a
-           serious problem */
-        gen_op_jz_ecx_im[s->aflag](next_eip);
-    }
+    int l1, l2;
+
+    l1 = gen_new_label();
+    l2 = gen_new_label();
+    gen_op_jnz_ecx[s->aflag](l1);
+    gen_set_label(l2);
+    gen_jmp_tb(s, next_eip, 1);
+    gen_set_label(l1);
+    return l2;
 }
  
 static inline void gen_stos(DisasContext *s, int ot)
@@ -821,6 +997,11 @@ static inline void gen_stos(DisasContext *s, int ot)
     gen_string_movl_A0_EDI(s);
     gen_op_st_T0_A0[ot + s->mem_index]();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_EDI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_EDI_T0();
     } else {
@@ -834,6 +1015,11 @@ static inline void gen_lods(DisasContext *s, int ot)
     gen_op_ld_T0_A0[ot + s->mem_index]();
     gen_op_mov_reg_T0[ot][R_EAX]();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_ESI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_ESI_T0();
     } else {
@@ -848,6 +1034,11 @@ static inline void gen_scas(DisasContext *s, int ot)
     gen_op_ld_T1_A0[ot + s->mem_index]();
     gen_op_cmpl_T0_T1_cc();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_EDI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_EDI_T0();
     } else {
@@ -863,6 +1054,12 @@ static inline void gen_cmps(DisasContext *s, int ot)
     gen_op_ld_T1_A0[ot + s->mem_index]();
     gen_op_cmpl_T0_T1_cc();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_ESI_T0();
+        gen_op_addq_EDI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_ESI_T0();
         gen_op_addl_EDI_T0();
@@ -880,6 +1077,11 @@ static inline void gen_ins(DisasContext *s, int ot)
     gen_op_in_DX_T0[ot]();
     gen_op_st_T0_A0[ot + s->mem_index]();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_EDI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_EDI_T0();
     } else {
@@ -893,6 +1095,11 @@ static inline void gen_outs(DisasContext *s, int ot)
     gen_op_ld_T0_A0[ot + s->mem_index]();
     gen_op_out_DX_T0[ot]();
     gen_op_movl_T0_Dshift[ot]();
+#ifdef TARGET_X86_64
+    if (s->aflag == 2) {
+        gen_op_addq_ESI_T0();
+    } else 
+#endif
     if (s->aflag) {
         gen_op_addl_ESI_T0();
     } else {
@@ -904,36 +1111,35 @@ static inline void gen_outs(DisasContext *s, int ot)
    instruction */
 #define GEN_REPZ(op)                                                          \
 static inline void gen_repz_ ## op(DisasContext *s, int ot,                   \
-                                 unsigned int cur_eip, unsigned int next_eip) \
+                                 target_ulong cur_eip, target_ulong next_eip) \
 {                                                                             \
+    int l2;\
     gen_update_cc_op(s);                                                      \
-    gen_jz_ecx_string(s, next_eip);                                           \
+    l2 = gen_jz_ecx_string(s, next_eip);                                      \
     gen_ ## op(s, ot);                                                        \
     gen_op_dec_ECX[s->aflag]();                                               \
     /* a loop would cause two single step exceptions if ECX = 1               \
        before rep string_insn */                                              \
     if (!s->jmp_opt)                                                          \
-        gen_op_jz_ecx_im[s->aflag](next_eip);                                 \
+        gen_op_jz_ecx[s->aflag](l2);                                          \
     gen_jmp(s, cur_eip);                                                      \
 }
  
 #define GEN_REPZ2(op)                                                         \
 static inline void gen_repz_ ## op(DisasContext *s, int ot,                   \
-                                   unsigned int cur_eip,                      \
-                                   unsigned int next_eip,                     \
+                                   target_ulong cur_eip,                      \
+                                   target_ulong next_eip,                     \
                                    int nz)                                    \
 {                                                                             \
+    int l2;\
     gen_update_cc_op(s);                                                      \
-    gen_jz_ecx_string(s, next_eip);                                           \
+    l2 = gen_jz_ecx_string(s, next_eip);                                      \
     gen_ ## op(s, ot);                                                        \
     gen_op_dec_ECX[s->aflag]();                                               \
     gen_op_set_cc_op(CC_OP_SUBB + ot);                                        \
+    gen_op_string_jnz_sub[nz][ot](l2);\
     if (!s->jmp_opt)                                                          \
-        gen_op_string_jnz_sub_im[nz][ot](next_eip);                           \
-    else                                                                      \
-        gen_op_string_jnz_sub(nz, ot, (long)s->tb);                           \
-    if (!s->jmp_opt)                                                          \
-        gen_op_jz_ecx_im[s->aflag](next_eip);                                 \
+        gen_op_jz_ecx[s->aflag](l2);                                          \
     gen_jmp(s, cur_eip);                                                      \
 }
  
@@ -956,7 +1162,7 @@ enum {
     JCC_LE,
 };
  
-static GenOpFunc3 *gen_jcc_sub[3][8] = {
+static GenOpFunc1 *gen_jcc_sub[4][8] = {
     [OT_BYTE] = {
         NULL,
         gen_op_jb_subb,
@@ -987,20 +1193,37 @@ static GenOpFunc3 *gen_jcc_sub[3][8] = {
         gen_op_jl_subl,
         gen_op_jle_subl,
     },
+#ifdef TARGET_X86_64
+    [OT_QUAD] = {
+        NULL,
+        BUGGY_64(gen_op_jb_subq),
+        gen_op_jz_subq,
+        BUGGY_64(gen_op_jbe_subq),
+        gen_op_js_subq,
+        NULL,
+        BUGGY_64(gen_op_jl_subq),
+        BUGGY_64(gen_op_jle_subq),
+    },
+#endif
 };
-static GenOpFunc2 *gen_op_loop[2][4] = {
+static GenOpFunc1 *gen_op_loop[3][4] = {
     [0] = {
         gen_op_loopnzw,
         gen_op_loopzw,
-        gen_op_loopw,
-        gen_op_jecxzw,
+        gen_op_jnz_ecxw,
     },
     [1] = {
         gen_op_loopnzl,
         gen_op_loopzl,
-        gen_op_loopl,
-        gen_op_jecxzl,
+        gen_op_jnz_ecxl,
+    },
+#ifdef TARGET_X86_64
+    [2] = {
+        gen_op_loopnzq,
+        gen_op_loopzq,
+        gen_op_jnz_ecxq,
     },
+#endif
 };
  
 static GenOpFunc *gen_setcc_slow[8] = {
@@ -1014,7 +1237,7 @@ static GenOpFunc *gen_setcc_slow[8] = {
     gen_op_setle_T0_cc,
 };
  
-static GenOpFunc *gen_setcc_sub[3][8] = {
+static GenOpFunc *gen_setcc_sub[4][8] = {
     [OT_BYTE] = {
         NULL,
         gen_op_setb_T0_subb,
@@ -1045,6 +1268,18 @@ static GenOpFunc *gen_setcc_sub[3][8] = {
         gen_op_setl_T0_subl,
         gen_op_setle_T0_subl,
     },
+#ifdef TARGET_X86_64
+    [OT_QUAD] = {
+        NULL,
+        gen_op_setb_T0_subq,
+        gen_op_setz_T0_subq,
+        gen_op_setbe_T0_subq,
+        gen_op_sets_T0_subq,
+        NULL,
+        gen_op_setl_T0_subq,
+        gen_op_setle_T0_subq,
+    },
+#endif
 };
  
 static GenOpFunc *gen_op_fp_arith_ST0_FT0[8] = {
@@ -1183,8 +1418,9 @@ static void gen_shifti(DisasContext *s1, int op, int ot, int d, int c)
  
 static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_ptr)
 {
+    target_long disp;
     int havesib;
-    int base, disp;
+    int base;
     int index;
     int scale;
     int opreg;
@@ -1208,16 +1444,20 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
             havesib = 1;
             code = ldub_code(s->pc++);
             scale = (code >> 6) & 3;
-            index = (code >> 3) & 7;
-            base = code & 7;
+            index = ((code >> 3) & 7) | REX_X(s);
+            base = (code & 7);
         }
+        base |= REX_B(s);
  
         switch (mod) {
         case 0:
-            if (base == 5) {
+            if ((base & 7) == 5) {
                 base = -1;
-                disp = ldl_code(s->pc);
+                disp = (int32_t)ldl_code(s->pc);
                 s->pc += 4;
+                if (CODE64(s) && !havesib) {
+                    disp += s->pc + s->rip_offset;
+                }
             } else {
                 disp = 0;
             }
@@ -1236,15 +1476,45 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
             /* for correct popl handling with esp */
             if (base == 4 && s->popl_esp_hack)
                 disp += s->popl_esp_hack;
-            gen_op_movl_A0_reg[base]();
-            if (disp != 0)
-                gen_op_addl_A0_im(disp);
+#ifdef TARGET_X86_64
+            if (s->aflag == 2) {
+                gen_op_movq_A0_reg[base]();
+                if (disp != 0) {
+                    if ((int32_t)disp == disp)
+                        gen_op_addq_A0_im(disp);
+                    else
+                        gen_op_addq_A0_im64(disp >> 32, disp);
+                }
+            } else 
+#endif
+            {
+                gen_op_movl_A0_reg[base]();
+                if (disp != 0)
+                    gen_op_addl_A0_im(disp);
+            }
         } else {
-            gen_op_movl_A0_im(disp);
+#ifdef TARGET_X86_64
+            if (s->aflag == 2) {
+                if ((int32_t)disp == disp)
+                    gen_op_movq_A0_im(disp);
+                else
+                    gen_op_movq_A0_im64(disp >> 32, disp);
+            } else 
+#endif
+            {
+                gen_op_movl_A0_im(disp);
+            }
         }
         /* XXX: index == 4 is always invalid */
         if (havesib && (index != 4 || scale != 0)) {
-            gen_op_addl_A0_reg_sN[scale][index]();
+#ifdef TARGET_X86_64
+            if (s->aflag == 2) {
+                gen_op_addq_A0_reg_sN[scale][index]();
+            } else 
+#endif
+            {
+                gen_op_addl_A0_reg_sN[scale][index]();
+            }
         }
         if (must_add_seg) {
             if (override < 0) {
@@ -1253,7 +1523,14 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
                 else
                     override = R_DS;
             }
-            gen_op_addl_A0_seg(offsetof(CPUX86State,segs[override].base));
+#ifdef TARGET_X86_64
+            if (s->aflag == 2) {
+                gen_op_addq_A0_seg(offsetof(CPUX86State,segs[override].base));
+            } else 
+#endif
+            {
+                gen_op_addl_A0_seg(offsetof(CPUX86State,segs[override].base));
+            }
         }
     } else {
         switch (mod) {
@@ -1336,7 +1613,7 @@ static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_s
     int mod, rm, opreg, disp;
  
     mod = (modrm >> 6) & 3;
-    rm = modrm & 7;
+    rm = (modrm & 7) | REX_B(s);
     if (mod == 3) {
         if (is_store) {
             if (reg != OR_TMP0)
@@ -1383,11 +1660,22 @@ static inline uint32_t insn_get(DisasContext *s, int ot)
     return ret;
 }
  
-static inline void gen_jcc(DisasContext *s, int b, int val, int next_eip)
+static inline int insn_const_size(unsigned int ot)
+{
+    if (ot <= OT_LONG)
+        return 1 << ot;
+    else
+        return 4;
+}
+
+static inline void gen_jcc(DisasContext *s, int b, 
+                           target_ulong val, target_ulong next_eip)
 {
     TranslationBlock *tb;
     int inv, jcc_op;
-    GenOpFunc3 *func;
+    GenOpFunc1 *func;
+    target_ulong tmp;
+    int l1, l2;
  
     inv = b & 1;
     jcc_op = (b >> 1) & 7;
@@ -1398,6 +1686,7 @@ static inline void gen_jcc(DisasContext *s, int b, int val, int next_eip)
         case CC_OP_SUBB:
         case CC_OP_SUBW:
         case CC_OP_SUBL:
+        case CC_OP_SUBQ:
             func = gen_jcc_sub[s->cc_op - CC_OP_SUBB][jcc_op];
             break;
  
@@ -1405,33 +1694,48 @@ static inline void gen_jcc(DisasContext *s, int b, int val, int next_eip)
         case CC_OP_ADDB:
         case CC_OP_ADDW:
         case CC_OP_ADDL:
+        case CC_OP_ADDQ:
+
         case CC_OP_ADCB:
         case CC_OP_ADCW:
         case CC_OP_ADCL:
+        case CC_OP_ADCQ:
+
         case CC_OP_SBBB:
         case CC_OP_SBBW:
         case CC_OP_SBBL:
+        case CC_OP_SBBQ:
+
         case CC_OP_LOGICB:
         case CC_OP_LOGICW:
         case CC_OP_LOGICL:
+        case CC_OP_LOGICQ:
+
         case CC_OP_INCB:
         case CC_OP_INCW:
         case CC_OP_INCL:
+        case CC_OP_INCQ:
+
         case CC_OP_DECB:
         case CC_OP_DECW:
         case CC_OP_DECL:
+        case CC_OP_DECQ:
+
         case CC_OP_SHLB:
         case CC_OP_SHLW:
         case CC_OP_SHLL:
+        case CC_OP_SHLQ:
+
         case CC_OP_SARB:
         case CC_OP_SARW:
         case CC_OP_SARL:
+        case CC_OP_SARQ:
             switch(jcc_op) {
             case JCC_Z:
-                func = gen_jcc_sub[(s->cc_op - CC_OP_ADDB) % 3][jcc_op];
+                func = gen_jcc_sub[(s->cc_op - CC_OP_ADDB) % 4][jcc_op];
                 break;
             case JCC_S:
-                func = gen_jcc_sub[(s->cc_op - CC_OP_ADDB) % 3][jcc_op];
+                func = gen_jcc_sub[(s->cc_op - CC_OP_ADDB) % 4][jcc_op];
                 break;
             default:
                 func = NULL;
@@ -1448,27 +1752,51 @@ static inline void gen_jcc(DisasContext *s, int b, int val, int next_eip)
  
         if (!func) {
             gen_setcc_slow[jcc_op]();
-            func = gen_op_jcc;
+            func = gen_op_jnz_T0_label;
         }
  
-        tb = s->tb;
-        if (!inv) {
-            func((long)tb, val, next_eip);
-        } else {
-            func((long)tb, next_eip, val);
+        if (inv) {
+            tmp = val;
+            val = next_eip;
+            next_eip = tmp;
         }
+        tb = s->tb;
+
+        l1 = gen_new_label();
+        func(l1);
+
+        gen_op_goto_tb0();
+        gen_jmp_im(next_eip);
+        gen_op_movl_T0_im((long)tb + 0);
+        gen_op_exit_tb();
+
+        gen_set_label(l1);
+        gen_op_goto_tb1();
+        gen_jmp_im(val);
+        gen_op_movl_T0_im((long)tb + 1);
+        gen_op_exit_tb();
+
         s->is_jmp = 3;
     } else {
+
         if (s->cc_op != CC_OP_DYNAMIC) {
             gen_op_set_cc_op(s->cc_op);
             s->cc_op = CC_OP_DYNAMIC;
         }
         gen_setcc_slow[jcc_op]();
-        if (!inv) {
-            gen_op_jcc_im(val, next_eip);
-        } else {
-            gen_op_jcc_im(next_eip, val);
+        if (inv) {
+            tmp = val;
+            val = next_eip;
+            next_eip = tmp;
         }
+        l1 = gen_new_label();
+        l2 = gen_new_label();
+        gen_op_jnz_T0_label(l1);
+        gen_jmp_im(next_eip);
+        gen_op_jmp_label(l2);
+        gen_set_label(l1);
+        gen_jmp_im(val);
+        gen_set_label(l2);
         gen_eob(s);
     }
 }
@@ -1485,6 +1813,7 @@ static void gen_setcc(DisasContext *s, int b)
     case CC_OP_SUBB:
     case CC_OP_SUBW:
     case CC_OP_SUBL:
+    case CC_OP_SUBQ:
         func = gen_setcc_sub[s->cc_op - CC_OP_SUBB][jcc_op];
         if (!func)
             goto slow_jcc;
@@ -1494,24 +1823,33 @@ static void gen_setcc(DisasContext *s, int b)
     case CC_OP_ADDB:
     case CC_OP_ADDW:
     case CC_OP_ADDL:
+    case CC_OP_ADDQ:
+
     case CC_OP_LOGICB:
     case CC_OP_LOGICW:
     case CC_OP_LOGICL:
+    case CC_OP_LOGICQ:
+
     case CC_OP_INCB:
     case CC_OP_INCW:
     case CC_OP_INCL:
+    case CC_OP_INCQ:
+
     case CC_OP_DECB:
     case CC_OP_DECW:
     case CC_OP_DECL:
+    case CC_OP_DECQ:
+
     case CC_OP_SHLB:
     case CC_OP_SHLW:
     case CC_OP_SHLL:
+    case CC_OP_SHLQ:
         switch(jcc_op) {
         case JCC_Z:
-            func = gen_setcc_sub[(s->cc_op - CC_OP_ADDB) % 3][jcc_op];
+            func = gen_setcc_sub[(s->cc_op - CC_OP_ADDB) % 4][jcc_op];
             break;
         case JCC_S:
-            func = gen_setcc_sub[(s->cc_op - CC_OP_ADDB) % 3][jcc_op];
+            func = gen_setcc_sub[(s->cc_op - CC_OP_ADDB) % 4][jcc_op];
             break;
         default:
             goto slow_jcc;
@@ -1532,13 +1870,13 @@ static void gen_setcc(DisasContext *s, int b)
  
 /* move T0 to seg_reg and compute if the CPU state may change. Never
    call this function with seg_reg == R_CS */
-static void gen_movl_seg_T0(DisasContext *s, int seg_reg, unsigned int cur_eip)
+static void gen_movl_seg_T0(DisasContext *s, int seg_reg, target_ulong cur_eip)
 {
     if (s->pe && !s->vm86) {
         /* XXX: optimize by finding processor state dynamically */
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        gen_op_jmp_im(cur_eip);
+        gen_jmp_im(cur_eip);
         gen_op_movl_seg_T0(seg_reg);
         /* abort translation because the addseg value may change or
            because ss32 may change. For R_SS, translation must always
@@ -1555,6 +1893,14 @@ static void gen_movl_seg_T0(DisasContext *s, int seg_reg, unsigned int cur_eip)
  
 static inline void gen_stack_update(DisasContext *s, int addend)
 {
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        if (addend == 8)
+            gen_op_addq_ESP_8();
+        else 
+            gen_op_addq_ESP_im(addend);
+    } else
+#endif
     if (s->ss32) {
         if (addend == 2)
             gen_op_addl_ESP_2();
@@ -1575,70 +1921,108 @@ static inline void gen_stack_update(DisasContext *s, int addend)
 /* generate a push. It depends on ss32, addseg and dflag */
 static void gen_push_T0(DisasContext *s)
 {
-    gen_op_movl_A0_reg[R_ESP]();
-    if (!s->dflag)
-        gen_op_subl_A0_2();
-    else
-        gen_op_subl_A0_4();
-    if (s->ss32) {
-        if (s->addseg) {
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        /* XXX: check 16 bit behaviour */
+        gen_op_movq_A0_reg[R_ESP]();
+        gen_op_subq_A0_8();
+        gen_op_st_T0_A0[OT_QUAD + s->mem_index]();
+        gen_op_movq_ESP_A0();
+    } else 
+#endif
+    {
+        gen_op_movl_A0_reg[R_ESP]();
+        if (!s->dflag)
+            gen_op_subl_A0_2();
+        else
+            gen_op_subl_A0_4();
+        if (s->ss32) {
+            if (s->addseg) {
+                gen_op_movl_T1_A0();
+                gen_op_addl_A0_SS();
+            }
+        } else {
+            gen_op_andl_A0_ffff();
             gen_op_movl_T1_A0();
             gen_op_addl_A0_SS();
         }
-    } else {
-        gen_op_andl_A0_ffff();
-        gen_op_movl_T1_A0();
-        gen_op_addl_A0_SS();
+        gen_op_st_T0_A0[s->dflag + 1 + s->mem_index]();
+        if (s->ss32 && !s->addseg)
+            gen_op_movl_ESP_A0();
+        else
+            gen_op_mov_reg_T1[s->ss32 + 1][R_ESP]();
     }
-    gen_op_st_T0_A0[s->dflag + 1 + s->mem_index]();
-    if (s->ss32 && !s->addseg)
-        gen_op_movl_ESP_A0();
-    else
-        gen_op_mov_reg_T1[s->ss32 + 1][R_ESP]();
 }
  
 /* generate a push. It depends on ss32, addseg and dflag */
 /* slower version for T1, only used for call Ev */
 static void gen_push_T1(DisasContext *s)
 {
-    gen_op_movl_A0_reg[R_ESP]();
-    if (!s->dflag)
-        gen_op_subl_A0_2();
-    else
-        gen_op_subl_A0_4();
-    if (s->ss32) {
-        if (s->addseg) {
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        /* XXX: check 16 bit behaviour */
+        gen_op_movq_A0_reg[R_ESP]();
+        gen_op_subq_A0_8();
+        gen_op_st_T1_A0[OT_QUAD + s->mem_index]();
+        gen_op_movq_ESP_A0();
+    } else 
+#endif
+    {
+        gen_op_movl_A0_reg[R_ESP]();
+        if (!s->dflag)
+            gen_op_subl_A0_2();
+        else
+            gen_op_subl_A0_4();
+        if (s->ss32) {
+            if (s->addseg) {
+                gen_op_addl_A0_SS();
+            }
+        } else {
+            gen_op_andl_A0_ffff();
             gen_op_addl_A0_SS();
         }
-    } else {
-        gen_op_andl_A0_ffff();
-        gen_op_addl_A0_SS();
+        gen_op_st_T1_A0[s->dflag + 1 + s->mem_index]();
+        
+        if (s->ss32 && !s->addseg)
+            gen_op_movl_ESP_A0();
+        else
+            gen_stack_update(s, (-2) << s->dflag);
     }
-    gen_op_st_T1_A0[s->dflag + 1 + s->mem_index]();
-    
-    if (s->ss32 && !s->addseg)
-        gen_op_movl_ESP_A0();
-    else
-        gen_stack_update(s, (-2) << s->dflag);
 }
  
 /* two step pop is necessary for precise exceptions */
 static void gen_pop_T0(DisasContext *s)
 {
-    gen_op_movl_A0_reg[R_ESP]();
-    if (s->ss32) {
-        if (s->addseg)
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        /* XXX: check 16 bit behaviour */
+        gen_op_movq_A0_reg[R_ESP]();
+        gen_op_ld_T0_A0[OT_QUAD + s->mem_index]();
+    } else 
+#endif
+    {
+        gen_op_movl_A0_reg[R_ESP]();
+        if (s->ss32) {
+            if (s->addseg)
+                gen_op_addl_A0_SS();
+        } else {
+            gen_op_andl_A0_ffff();
             gen_op_addl_A0_SS();
-    } else {
-        gen_op_andl_A0_ffff();
-        gen_op_addl_A0_SS();
+        }
+        gen_op_ld_T0_A0[s->dflag + 1 + s->mem_index]();
     }
-    gen_op_ld_T0_A0[s->dflag + 1 + s->mem_index]();
 }
  
 static void gen_pop_update(DisasContext *s)
 {
-    gen_stack_update(s, 2 << s->dflag);
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        gen_stack_update(s, 8);
+    } else
+#endif
+    {
+        gen_stack_update(s, 2 << s->dflag);
+    }
 }
  
 static void gen_stack_A0(DisasContext *s)
@@ -1718,11 +2102,11 @@ static void gen_enter(DisasContext *s, int esp_addend, int level)
     gen_op_mov_reg_T1[ot][R_ESP]();
 }
  
-static void gen_exception(DisasContext *s, int trapno, unsigned int cur_eip)
+static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
 {
     if (s->cc_op != CC_OP_DYNAMIC)
         gen_op_set_cc_op(s->cc_op);
-    gen_op_jmp_im(cur_eip);
+    gen_jmp_im(cur_eip);
     gen_op_raise_exception(trapno);
     s->is_jmp = 3;
 }
@@ -1730,20 +2114,20 @@ static void gen_exception(DisasContext *s, int trapno, unsigned int cur_eip)
 /* an interrupt is different from an exception because of the
    priviledge checks */
 static void gen_interrupt(DisasContext *s, int intno, 
-                          unsigned int cur_eip, unsigned int next_eip)
+                          target_ulong cur_eip, target_ulong next_eip)
 {
     if (s->cc_op != CC_OP_DYNAMIC)
         gen_op_set_cc_op(s->cc_op);
-    gen_op_jmp_im(cur_eip);
+    gen_jmp_im(cur_eip);
     gen_op_raise_interrupt(intno, next_eip);
     s->is_jmp = 3;
 }
  
-static void gen_debug(DisasContext *s, unsigned int cur_eip)
+static void gen_debug(DisasContext *s, target_ulong cur_eip)
 {
     if (s->cc_op != CC_OP_DYNAMIC)
         gen_op_set_cc_op(s->cc_op);
-    gen_op_jmp_im(cur_eip);
+    gen_jmp_im(cur_eip);
     gen_op_debug();
     s->is_jmp = 3;
 }
@@ -1770,80 +2154,186 @@ static void gen_eob(DisasContext *s)
  
 /* generate a jump to eip. No segment change must happen before as a
    direct call to the next block may occur */
-static void gen_jmp(DisasContext *s, unsigned int eip)
+static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num)
 {
     TranslationBlock *tb = s->tb;
  
     if (s->jmp_opt) {
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        gen_op_jmp((long)tb, eip);
+        if (tb_num)
+            gen_op_goto_tb1();
+        else
+            gen_op_goto_tb0();
+        gen_jmp_im(eip);
+        gen_op_movl_T0_im((long)tb + tb_num);
+        gen_op_exit_tb();
         s->is_jmp = 3;
     } else {
-        gen_op_jmp_im(eip);
+        gen_jmp_im(eip);
         gen_eob(s);
     }
 }
  
+static void gen_jmp(DisasContext *s, target_ulong eip)
+{
+    gen_jmp_tb(s, eip, 0);
+}
+
+static void gen_movtl_T0_im(target_ulong val)
+{
+#ifdef TARGET_X86_64    
+    if ((int32_t)val == val) {
+        gen_op_movl_T0_im(val);
+    } else {
+        gen_op_movq_T0_im64(val >> 32, val);
+    }
+#else
+    gen_op_movl_T0_im(val);
+#endif
+}
+
+static GenOpFunc1 *gen_ldo_env_A0[3] = {
+    gen_op_ldo_raw_env_A0,
+#ifndef CONFIG_USER_ONLY
+    gen_op_ldo_kernel_env_A0,
+    gen_op_ldo_user_env_A0,
+#endif
+};
+
+static GenOpFunc1 *gen_sto_env_A0[3] = {
+    gen_op_sto_raw_env_A0,
+#ifndef CONFIG_USER_ONLY
+    gen_op_sto_kernel_env_A0,
+    gen_op_sto_user_env_A0,
+#endif
+};
+
 /* convert one instruction. s->is_jmp is set if the translation must
    be stopped. Return the next pc value */
-static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
+static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 {
     int b, prefixes, aflag, dflag;
     int shift, ot;
     int modrm, reg, rm, mod, reg_addr, op, opreg, offset_addr, val;
-    unsigned int next_eip;
+    target_ulong next_eip, tval;
+    int rex_w, rex_r;
  
     s->pc = pc_start;
     prefixes = 0;
     aflag = s->code32;
     dflag = s->code32;
     s->override = -1;
+    rex_w = -1;
+    rex_r = 0;
+#ifdef TARGET_X86_64
+    s->rex_x = 0;
+    s->rex_b = 0;
+    x86_64_hregs = 0; 
+#endif
+    s->rip_offset = 0; /* for relative ip address */
  next_byte:
     b = ldub_code(s->pc);
     s->pc++;
     /* check prefixes */
-    switch (b) {
-    case 0xf3:
-        prefixes |= PREFIX_REPZ;
-        goto next_byte;
-    case 0xf2:
-        prefixes |= PREFIX_REPNZ;
-        goto next_byte;
-    case 0xf0:
-        prefixes |= PREFIX_LOCK;
-        goto next_byte;
-    case 0x2e:
-        s->override = R_CS;
-        goto next_byte;
-    case 0x36:
-        s->override = R_SS;
-        goto next_byte;
-    case 0x3e:
-        s->override = R_DS;
-        goto next_byte;
-    case 0x26:
-        s->override = R_ES;
-        goto next_byte;
-    case 0x64:
-        s->override = R_FS;
-        goto next_byte;
-    case 0x65:
-        s->override = R_GS;
-        goto next_byte;
-    case 0x66:
-        prefixes |= PREFIX_DATA;
-        goto next_byte;
-    case 0x67:
-        prefixes |= PREFIX_ADR;
-        goto next_byte;
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        switch (b) {
+        case 0xf3:
+            prefixes |= PREFIX_REPZ;
+            goto next_byte;
+        case 0xf2:
+            prefixes |= PREFIX_REPNZ;
+            goto next_byte;
+        case 0xf0:
+            prefixes |= PREFIX_LOCK;
+            goto next_byte;
+        case 0x2e:
+            s->override = R_CS;
+            goto next_byte;
+        case 0x36:
+            s->override = R_SS;
+            goto next_byte;
+        case 0x3e:
+            s->override = R_DS;
+            goto next_byte;
+        case 0x26:
+            s->override = R_ES;
+            goto next_byte;
+        case 0x64:
+            s->override = R_FS;
+            goto next_byte;
+        case 0x65:
+            s->override = R_GS;
+            goto next_byte;
+        case 0x66:
+            prefixes |= PREFIX_DATA;
+            goto next_byte;
+        case 0x67:
+            prefixes |= PREFIX_ADR;
+            goto next_byte;
+        case 0x40 ... 0x4f:
+            /* REX prefix */
+            rex_w = (b >> 3) & 1;
+            rex_r = (b & 0x4) << 1;
+            s->rex_x = (b & 0x2) << 2;
+            REX_B(s) = (b & 0x1) << 3;
+            x86_64_hregs = 1; /* select uniform byte register addressing */
+            goto next_byte;
+        }
+        if (rex_w == 1) {
+            /* 0x66 is ignored if rex.w is set */
+            dflag = 2;
+        } else {
+            if (prefixes & PREFIX_DATA)
+                dflag ^= 1;
+        }
+        if (!(prefixes & PREFIX_ADR))
+            aflag = 2;
+    } else 
+#endif
+    {
+        switch (b) {
+        case 0xf3:
+            prefixes |= PREFIX_REPZ;
+            goto next_byte;
+        case 0xf2:
+            prefixes |= PREFIX_REPNZ;
+            goto next_byte;
+        case 0xf0:
+            prefixes |= PREFIX_LOCK;
+            goto next_byte;
+        case 0x2e:
+            s->override = R_CS;
+            goto next_byte;
+        case 0x36:
+            s->override = R_SS;
+            goto next_byte;
+        case 0x3e:
+            s->override = R_DS;
+            goto next_byte;
+        case 0x26:
+            s->override = R_ES;
+            goto next_byte;
+        case 0x64:
+            s->override = R_FS;
+            goto next_byte;
+        case 0x65:
+            s->override = R_GS;
+            goto next_byte;
+        case 0x66:
+            prefixes |= PREFIX_DATA;
+            goto next_byte;
+        case 0x67:
+            prefixes |= PREFIX_ADR;
+            goto next_byte;
+        }
+        if (prefixes & PREFIX_DATA)
+            dflag ^= 1;
+        if (prefixes & PREFIX_ADR)
+            aflag ^= 1;
     }
  
-    if (prefixes & PREFIX_DATA)
-        dflag ^= 1;
-    if (prefixes & PREFIX_ADR)
-        aflag ^= 1;
-
     s->prefix = prefixes;
     s->aflag = aflag;
     s->dflag = dflag;
@@ -1879,14 +2369,14 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             if ((b & 1) == 0)
                 ot = OT_BYTE;
             else
-                ot = dflag ? OT_LONG : OT_WORD;
+                ot = dflag + OT_WORD;
  
             switch(f) {
             case 0: /* OP Ev, Gv */
                 modrm = ldub_code(s->pc++);
-                reg = ((modrm >> 3) & 7);
+                reg = ((modrm >> 3) & 7) | rex_r;
                 mod = (modrm >> 6) & 3;
-                rm = modrm & 7;
+                rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
                     gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                     opreg = OR_TMP0;
@@ -1907,8 +2397,8 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             case 1: /* OP Gv, Ev */
                 modrm = ldub_code(s->pc++);
                 mod = (modrm >> 6) & 3;
-                reg = ((modrm >> 3) & 7);
-                rm = modrm & 7;
+                reg = ((modrm >> 3) & 7) | rex_r;
+                rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
                     gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                     gen_op_ld_T1_A0[ot + s->mem_index]();
@@ -1938,18 +2428,22 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             if ((b & 1) == 0)
                 ot = OT_BYTE;
             else
-                ot = dflag ? OT_LONG : OT_WORD;
+                ot = dflag + OT_WORD;
  
             modrm = ldub_code(s->pc++);
             mod = (modrm >> 6) & 3;
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
             op = (modrm >> 3) & 7;
  
             if (mod != 3) {
+                if (b == 0x83)
+                    s->rip_offset = 1;
+                else
+                    s->rip_offset = insn_const_size(ot);
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                 opreg = OR_TMP0;
             } else {
-                opreg = rm + OR_EAX;
+                opreg = rm;
             }
  
             switch(b) {
@@ -1983,13 +2477,15 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
  
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
+        rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
         if (mod != 3) {
+            if (op == 0)
+                s->rip_offset = insn_const_size(ot);
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T0_A0[ot + s->mem_index]();
         } else {
@@ -2036,6 +2532,12 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 gen_op_mull_EAX_T0();
                 s->cc_op = CC_OP_MULL;
                 break;
+#ifdef TARGET_X86_64
+            case OT_QUAD:
+                gen_op_mulq_EAX_T0();
+                s->cc_op = CC_OP_MULQ;
+                break;
+#endif
             }
             break;
         case 5: /* imul */
@@ -2053,34 +2555,58 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 gen_op_imull_EAX_T0();
                 s->cc_op = CC_OP_MULL;
                 break;
+#ifdef TARGET_X86_64
+            case OT_QUAD:
+                gen_op_imulq_EAX_T0();
+                s->cc_op = CC_OP_MULQ;
+                break;
+#endif
             }
             break;
         case 6: /* div */
             switch(ot) {
             case OT_BYTE:
-                gen_op_divb_AL_T0(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_divb_AL_T0();
                 break;
             case OT_WORD:
-                gen_op_divw_AX_T0(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_divw_AX_T0();
                 break;
             default:
             case OT_LONG:
-                gen_op_divl_EAX_T0(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_divl_EAX_T0();
+                break;
+#ifdef TARGET_X86_64
+            case OT_QUAD:
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_divq_EAX_T0();
                 break;
+#endif
             }
             break;
         case 7: /* idiv */
             switch(ot) {
             case OT_BYTE:
-                gen_op_idivb_AL_T0(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_idivb_AL_T0();
                 break;
             case OT_WORD:
-                gen_op_idivw_AX_T0(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_idivw_AX_T0();
                 break;
             default:
             case OT_LONG:
-                gen_op_idivl_EAX_T0(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_idivl_EAX_T0();
+                break;
+#ifdef TARGET_X86_64
+            case OT_QUAD:
+                gen_jmp_im(pc_start - s->cs_base);
+                gen_op_idivq_EAX_T0();
                 break;
+#endif
             }
             break;
         default:
@@ -2093,15 +2619,24 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
  
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
+        rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
         if (op >= 2 && b == 0xfe) {
             goto illegal_op;
         }
+        if (CODE64(s)) {
+            if (op >= 2 && op <= 5) {
+                /* operand size for jumps is 64 bit */
+                ot = OT_QUAD;
+            } else if (op == 6) {
+                /* default push size is 64 bit */
+                ot = dflag ? OT_QUAD : OT_WORD;
+            }
+        }
         if (mod != 3) {
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
             if (op >= 2 && op != 3 && op != 5)
@@ -2143,7 +2678,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             if (s->pe && !s->vm86) {
                 if (s->cc_op != CC_OP_DYNAMIC)
                     gen_op_set_cc_op(s->cc_op);
-                gen_op_jmp_im(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
                 gen_op_lcall_protected_T0_T1(dflag, s->pc - s->cs_base);
             } else {
                 gen_op_lcall_real_T0_T1(dflag, s->pc - s->cs_base);
@@ -2164,7 +2699,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             if (s->pe && !s->vm86) {
                 if (s->cc_op != CC_OP_DYNAMIC)
                     gen_op_set_cc_op(s->cc_op);
-                gen_op_jmp_im(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
                 gen_op_ljmp_protected_T0_T1(s->pc - s->cs_base);
             } else {
                 gen_op_movl_seg_T0_vm(offsetof(CPUX86State,segs[R_CS]));
@@ -2186,15 +2721,15 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
  
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
-        reg = (modrm >> 3) & 7;
+        rm = (modrm & 7) | REX_B(s);
+        reg = ((modrm >> 3) & 7) | rex_r;
  
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
-        gen_op_mov_TN_reg[ot][1][reg + OR_EAX]();
+        gen_op_mov_TN_reg[ot][1][reg]();
         gen_op_testl_T0_T1_cc();
         s->cc_op = CC_OP_LOGICB + ot;
         break;
@@ -2204,7 +2739,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         val = insn_get(s, ot);
  
         gen_op_mov_TN_reg[ot][0][OR_EAX]();
@@ -2214,13 +2749,23 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         break;
  
     case 0x98: /* CWDE/CBW */
-        if (dflag)
+#ifdef TARGET_X86_64
+        if (dflag == 2) {
+            gen_op_movslq_RAX_EAX();
+        } else
+#endif
+        if (dflag == 1)
             gen_op_movswl_EAX_AX();
         else
             gen_op_movsbw_AX_AL();
         break;
     case 0x99: /* CDQ/CWD */
-        if (dflag)
+#ifdef TARGET_X86_64
+        if (dflag == 2) {
+            gen_op_movsqo_RDX_RAX();
+        } else
+#endif
+        if (dflag == 1)
             gen_op_movslq_EDX_EAX();
         else
             gen_op_movswl_DX_AX();
@@ -2228,9 +2773,13 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     case 0x1af: /* imul Gv, Ev */
     case 0x69: /* imul Gv, Ev, I */
     case 0x6b:
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = ((modrm >> 3) & 7) + OR_EAX;
+        reg = ((modrm >> 3) & 7) | rex_r;
+        if (b == 0x69)
+            s->rip_offset = insn_const_size(ot);
+        else if (b == 0x6b)
+            s->rip_offset = 1;
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
         if (b == 0x69) {
             val = insn_get(s, ot);
@@ -2242,6 +2791,11 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             gen_op_mov_TN_reg[ot][1][reg]();
         }
  
+#ifdef TARGET_X86_64
+        if (ot == OT_QUAD) {
+            gen_op_imulq_T0_T1();
+        } else
+#endif
         if (ot == OT_LONG) {
             gen_op_imull_T0_T1();
         } else {
@@ -2255,12 +2809,12 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
             gen_op_mov_TN_reg[ot][0][reg]();
             gen_op_mov_TN_reg[ot][1][rm]();
             gen_op_addl_T0_T1();
@@ -2282,13 +2836,13 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         gen_op_mov_TN_reg[ot][1][reg]();
         if (mod == 3) {
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
             gen_op_mov_TN_reg[ot][0][rm]();
             gen_op_cmpxchg_T0_T1_EAX_cc[ot]();
             gen_op_mov_reg_T0[ot][rm]();
@@ -2314,25 +2868,37 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         /**************************/
         /* push/pop */
     case 0x50 ... 0x57: /* push */
-        gen_op_mov_TN_reg[OT_LONG][0][b & 7]();
+        gen_op_mov_TN_reg[OT_LONG][0][(b & 7) | REX_B(s)]();
         gen_push_T0(s);
         break;
     case 0x58 ... 0x5f: /* pop */
-        ot = dflag ? OT_LONG : OT_WORD;
+        if (CODE64(s)) {
+            ot = dflag ? OT_QUAD : OT_WORD;
+        } else {
+            ot = dflag + OT_WORD;
+        }
         gen_pop_T0(s);
         /* NOTE: order is important for pop %sp */
         gen_pop_update(s);
-        gen_op_mov_reg_T0[ot][b & 7]();
+        gen_op_mov_reg_T0[ot][(b & 7) | REX_B(s)]();
         break;
     case 0x60: /* pusha */
+        if (CODE64(s))
+            goto illegal_op;
         gen_pusha(s);
         break;
     case 0x61: /* popa */
+        if (CODE64(s))
+            goto illegal_op;
         gen_popa(s);
         break;
     case 0x68: /* push Iv */
     case 0x6a:
-        ot = dflag ? OT_LONG : OT_WORD;
+        if (CODE64(s)) {
+            ot = dflag ? OT_QUAD : OT_WORD;
+        } else {
+            ot = dflag + OT_WORD;
+        }
         if (b == 0x68)
             val = insn_get(s, ot);
         else
@@ -2341,18 +2907,22 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         gen_push_T0(s);
         break;
     case 0x8f: /* pop Ev */
-        ot = dflag ? OT_LONG : OT_WORD;
+        if (CODE64(s)) {
+            ot = dflag ? OT_QUAD : OT_WORD;
+        } else {
+            ot = dflag + OT_WORD;
+        }
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
         gen_pop_T0(s);
         if (mod == 3) {
             /* NOTE: order is important for pop %sp */
             gen_pop_update(s);
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
             gen_op_mov_reg_T0[ot][rm]();
         } else {
             /* NOTE: order is important too for MMU exceptions */
-            s->popl_esp_hack = 2 << dflag;
+            s->popl_esp_hack = 1 << ot;
             gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
             s->popl_esp_hack = 0;
             gen_pop_update(s);
@@ -2360,6 +2930,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         break;
     case 0xc8: /* enter */
         {
+            /* XXX: long mode support */
             int level;
             val = lduw_code(s->pc);
             s->pc += 2;
@@ -2369,7 +2940,11 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         break;
     case 0xc9: /* leave */
         /* XXX: exception not precise (ESP is updated before potential exception) */
-        if (s->ss32) {
+        /* XXX: may be invalid for 16 bit in long mode */
+        if (CODE64(s)) {
+            gen_op_mov_TN_reg[OT_QUAD][0][R_EBP]();
+            gen_op_mov_reg_T0[OT_QUAD][R_ESP]();
+        } else if (s->ss32) {
             gen_op_mov_TN_reg[OT_LONG][0][R_EBP]();
             gen_op_mov_reg_T0[OT_LONG][R_ESP]();
         } else {
@@ -2377,7 +2952,11 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             gen_op_mov_reg_T0[OT_WORD][R_ESP]();
         }
         gen_pop_T0(s);
-        ot = dflag ? OT_LONG : OT_WORD;
+        if (CODE64(s)) {
+            ot = dflag ? OT_QUAD : OT_WORD;
+        } else {
+            ot = dflag + OT_WORD;
+        }
         gen_op_mov_reg_T0[ot][R_EBP]();
         gen_pop_update(s);
         break;
@@ -2385,6 +2964,8 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     case 0x0e: /* push cs */
     case 0x16: /* push ss */
     case 0x1e: /* push ds */
+        if (CODE64(s))
+            goto illegal_op;
         gen_op_movl_T0_seg(b >> 3);
         gen_push_T0(s);
         break;
@@ -2396,6 +2977,8 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     case 0x07: /* pop es */
     case 0x17: /* pop ss */
     case 0x1f: /* pop ds */
+        if (CODE64(s))
+            goto illegal_op;
         reg = b >> 3;
         gen_pop_T0(s);
         gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
@@ -2409,7 +2992,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             s->tf = 0;
         }
         if (s->is_jmp) {
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
         break;
@@ -2419,7 +3002,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         gen_movl_seg_T0(s, (b >> 3) & 7, pc_start - s->cs_base);
         gen_pop_update(s);
         if (s->is_jmp) {
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
         break;
@@ -2431,38 +3014,40 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
  
         /* generate a generic store */
-        gen_ldst_modrm(s, modrm, ot, OR_EAX + reg, 1);
+        gen_ldst_modrm(s, modrm, ot, reg, 1);
         break;
     case 0xc6:
     case 0xc7: /* mov Ev, Iv */
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
-        if (mod != 3)
+        if (mod != 3) {
+            s->rip_offset = insn_const_size(ot);
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        }
         val = insn_get(s, ot);
         gen_op_movl_T0_im(val);
         if (mod != 3)
             gen_op_st_T0_A0[ot + s->mem_index]();
         else
-            gen_op_mov_reg_T0[ot][modrm & 7]();
+            gen_op_mov_reg_T0[ot][(modrm & 7) | REX_B(s)]();
         break;
     case 0x8a:
     case 0x8b: /* mov Ev, Gv */
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = OT_WORD + dflag;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
  
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_reg_T0[ot][reg]();
@@ -2483,7 +3068,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             s->tf = 0;
         }
         if (s->is_jmp) {
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
         break;
@@ -2494,9 +3079,10 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if (reg >= 6)
             goto illegal_op;
         gen_op_movl_T0_seg(reg);
-        ot = OT_WORD;
-        if (mod == 3 && dflag)
-            ot = OT_LONG;
+        if (mod == 3)
+            ot = OT_WORD + dflag;
+        else
+            ot = OT_WORD;
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
         break;
  
@@ -2511,9 +3097,9 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             /* ot is the size of source */
             ot = (b & 1) + OT_BYTE;
             modrm = ldub_code(s->pc++);
-            reg = ((modrm >> 3) & 7) + OR_EAX;
+            reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
  
             if (mod == 3) {
                 gen_op_mov_TN_reg[ot][0][rm]();
@@ -2546,12 +3132,12 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         break;
  
     case 0x8d: /* lea */
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         /* we must ensure that no segment is added */
         s->override = -1;
         val = s->addseg;
@@ -2565,42 +3151,67 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     case 0xa1:
     case 0xa2: /* mov Ov, EAX */
     case 0xa3:
-        if ((b & 1) == 0)
-            ot = OT_BYTE;
-        else
-            ot = dflag ? OT_LONG : OT_WORD;
-        if (s->aflag)
-            offset_addr = insn_get(s, OT_LONG);
-        else
-            offset_addr = insn_get(s, OT_WORD);
-        gen_op_movl_A0_im(offset_addr);
-        /* handle override */
         {
-            int override, must_add_seg;
-            must_add_seg = s->addseg;
-            if (s->override >= 0) {
-                override = s->override;
-                must_add_seg = 1;
-            } else {
-                override = R_DS;
+            target_ulong offset_addr;
+
+            if ((b & 1) == 0)
+                ot = OT_BYTE;
+            else
+                ot = dflag + OT_WORD;
+#ifdef TARGET_X86_64
+            if (CODE64(s)) {
+                offset_addr = ldq_code(s->pc);
+                s->pc += 8;
+                if (offset_addr == (int32_t)offset_addr)
+                    gen_op_movq_A0_im(offset_addr);
+                else
+                    gen_op_movq_A0_im64(offset_addr >> 32, offset_addr);
+            } else 
+#endif
+            {
+                if (s->aflag) {
+                    offset_addr = insn_get(s, OT_LONG);
+                } else {
+                    offset_addr = insn_get(s, OT_WORD);
+                }
+                gen_op_movl_A0_im(offset_addr);
+            }
+            /* handle override */
+            {
+                int override, must_add_seg;
+                must_add_seg = s->addseg;
+                if (s->override >= 0) {
+                    override = s->override;
+                    must_add_seg = 1;
+                } else {
+                    override = R_DS;
+                }
+                if (must_add_seg) {
+                    gen_op_addl_A0_seg(offsetof(CPUX86State,segs[override].base));
+                }
             }
-            if (must_add_seg) {
-                gen_op_addl_A0_seg(offsetof(CPUX86State,segs[override].base));
+            if ((b & 2) == 0) {
+                gen_op_ld_T0_A0[ot + s->mem_index]();
+                gen_op_mov_reg_T0[ot][R_EAX]();
+            } else {
+                gen_op_mov_TN_reg[ot][0][R_EAX]();
+                gen_op_st_T0_A0[ot + s->mem_index]();
             }
         }
-        if ((b & 2) == 0) {
-            gen_op_ld_T0_A0[ot + s->mem_index]();
-            gen_op_mov_reg_T0[ot][R_EAX]();
-        } else {
-            gen_op_mov_TN_reg[ot][0][R_EAX]();
-            gen_op_st_T0_A0[ot + s->mem_index]();
-        }
         break;
     case 0xd7: /* xlat */
-        gen_op_movl_A0_reg[R_EBX]();
-        gen_op_addl_A0_AL();
-        if (s->aflag == 0)
-            gen_op_andl_A0_ffff();
+#ifdef TARGET_X86_64
+        if (CODE64(s)) {
+            gen_op_movq_A0_reg[R_EBX]();
+            gen_op_addq_A0_AL();
+        } else 
+#endif
+        {
+            gen_op_movl_A0_reg[R_EBX]();
+            gen_op_addl_A0_AL();
+            if (s->aflag == 0)
+                gen_op_andl_A0_ffff();
+        }
         /* handle override */
         {
             int override, must_add_seg;
@@ -2622,19 +3233,32 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     case 0xb0 ... 0xb7: /* mov R, Ib */
         val = insn_get(s, OT_BYTE);
         gen_op_movl_T0_im(val);
-        gen_op_mov_reg_T0[OT_BYTE][b & 7]();
+        gen_op_mov_reg_T0[OT_BYTE][(b & 7) | REX_B(s)]();
         break;
     case 0xb8 ... 0xbf: /* mov R, Iv */
-        ot = dflag ? OT_LONG : OT_WORD;
-        val = insn_get(s, ot);
-        reg = OR_EAX + (b & 7);
-        gen_op_movl_T0_im(val);
-        gen_op_mov_reg_T0[ot][reg]();
+#ifdef TARGET_X86_64
+        if (dflag == 2) {
+            uint64_t tmp;
+            /* 64 bit case */
+            tmp = ldq_code(s->pc);
+            s->pc += 8;
+            reg = (b & 7) | REX_B(s);
+            gen_movtl_T0_im(tmp);
+            gen_op_mov_reg_T0[OT_QUAD][reg]();
+        } else 
+#endif
+        {
+            ot = dflag ? OT_LONG : OT_WORD;
+            val = insn_get(s, ot);
+            reg = (b & 7) | REX_B(s);
+            gen_op_movl_T0_im(val);
+            gen_op_mov_reg_T0[ot][reg]();
+        }
         break;
  
     case 0x91 ... 0x97: /* xchg R, EAX */
-        ot = dflag ? OT_LONG : OT_WORD;
-        reg = b & 7;
+        ot = dflag + OT_WORD;
+        reg = (b & 7) | REX_B(s);
         rm = R_EAX;
         goto do_xchg_reg;
     case 0x86:
@@ -2642,12 +3266,12 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
         do_xchg_reg:
             gen_op_mov_TN_reg[ot][0][reg]();
             gen_op_mov_TN_reg[ot][1][rm]();
@@ -2667,9 +3291,13 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         }
         break;
     case 0xc4: /* les Gv */
+        if (CODE64(s))
+            goto illegal_op;
         op = R_ES;
         goto do_lxx;
     case 0xc5: /* lds Gv */
+        if (CODE64(s))
+            goto illegal_op;
         op = R_DS;
         goto do_lxx;
     case 0x1b2: /* lss Gv */
@@ -2683,7 +3311,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     do_lxx:
         ot = dflag ? OT_LONG : OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
@@ -2696,7 +3324,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         /* then put the data */
         gen_op_mov_reg_T1[ot][reg]();
         if (s->is_jmp) {
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
         break;
@@ -2712,18 +3340,20 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             if ((b & 1) == 0)
                 ot = OT_BYTE;
             else
-                ot = dflag ? OT_LONG : OT_WORD;
+                ot = dflag + OT_WORD;
  
             modrm = ldub_code(s->pc++);
             mod = (modrm >> 6) & 3;
-            rm = modrm & 7;
             op = (modrm >> 3) & 7;
  
             if (mod != 3) {
+                if (shift == 2) {
+                    s->rip_offset = 1;
+                }
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                 opreg = OR_TMP0;
             } else {
-                opreg = rm + OR_EAX;
+                opreg = (modrm & 7) | REX_B(s);
             }
  
             /* simpler op */
@@ -2764,11 +3394,11 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         op = 1;
         shift = 0;
     do_shiftd:
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
-        reg = (modrm >> 3) & 7;
+        rm = (modrm & 7) | REX_B(s);
+        reg = ((modrm >> 3) & 7) | rex_r;
  
         if (mod != 3) {
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
@@ -2780,7 +3410,10 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
  
         if (shift) {
             val = ldub_code(s->pc++);
-            val &= 0x1f;
+            if (ot == OT_QUAD)
+                val &= 0x3f;
+            else
+                val &= 0x1f;
             if (val) {
                 if (mod == 3)
                     gen_op_shiftd_T0_T1_im_cc[ot][op](val);
@@ -2970,7 +3603,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                     /* check exceptions (FreeBSD FPU probe) */
                     if (s->cc_op != CC_OP_DYNAMIC)
                         gen_op_set_cc_op(s->cc_op);
-                    gen_op_jmp_im(pc_start - s->cs_base);
+                    gen_jmp_im(pc_start - s->cs_base);
                     gen_op_fwait();
                     break;
                 default:
@@ -3257,7 +3890,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
  
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
             gen_repz_movs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
@@ -3271,7 +3904,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
  
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
             gen_repz_stos(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
@@ -3284,7 +3917,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
             gen_repz_lods(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
         } else {
@@ -3296,7 +3929,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-                ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         if (prefixes & PREFIX_REPNZ) {
             gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1);
         } else if (prefixes & PREFIX_REPZ) {
@@ -3312,7 +3945,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if ((b & 1) == 0)
             ot = OT_BYTE;
         else
-            ot = dflag ? OT_LONG : OT_WORD;
+            ot = dflag + OT_WORD;
         if (prefixes & PREFIX_REPNZ) {
             gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1);
         } else if (prefixes & PREFIX_REPZ) {
@@ -3427,7 +4060,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         if (s->pe && !s->vm86) {
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_op_jmp_im(pc_start - s->cs_base);
+            gen_jmp_im(pc_start - s->cs_base);
             gen_op_lret_protected(s->dflag, val);
         } else {
             gen_stack_A0(s);
@@ -3465,7 +4098,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         } else {
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_op_jmp_im(pc_start - s->cs_base);
+            gen_jmp_im(pc_start - s->cs_base);
             gen_op_iret_protected(s->dflag, s->pc - s->cs_base);
             s->cc_op = CC_OP_EFLAGS;
         }
@@ -3473,72 +4106,79 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         break;
     case 0xe8: /* call im */
         {
-            unsigned int next_eip;
-            ot = dflag ? OT_LONG : OT_WORD;
-            val = insn_get(s, ot);
+            if (dflag)
+                tval = (int32_t)insn_get(s, OT_LONG);
+            else
+                tval = (int16_t)insn_get(s, OT_WORD);
             next_eip = s->pc - s->cs_base;
-            val += next_eip;
+            tval += next_eip;
             if (s->dflag == 0)
-                val &= 0xffff;
-            gen_op_movl_T0_im(next_eip);
+                tval &= 0xffff;
+            gen_movtl_T0_im(next_eip);
             gen_push_T0(s);
-            gen_jmp(s, val);
+            gen_jmp(s, tval);
         }
         break;
     case 0x9a: /* lcall im */
         {
             unsigned int selector, offset;
-
+            
+            if (CODE64(s))
+                goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
             offset = insn_get(s, ot);
             selector = insn_get(s, OT_WORD);
  
             gen_op_movl_T0_im(selector);
-            gen_op_movl_T1_im(offset);
+            gen_op_movl_T1_imu(offset);
         }
         goto do_lcall;
     case 0xe9: /* jmp */
-        ot = dflag ? OT_LONG : OT_WORD;
-        val = insn_get(s, ot);
-        val += s->pc - s->cs_base;
+        if (dflag)
+            tval = (int32_t)insn_get(s, OT_LONG);
+        else
+            tval = (int16_t)insn_get(s, OT_WORD);
+        tval += s->pc - s->cs_base;
         if (s->dflag == 0)
-            val = val & 0xffff;
-        gen_jmp(s, val);
+            tval &= 0xffff;
+        gen_jmp(s, tval);
         break;
     case 0xea: /* ljmp im */
         {
             unsigned int selector, offset;
  
+            if (CODE64(s))
+                goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
             offset = insn_get(s, ot);
             selector = insn_get(s, OT_WORD);
  
             gen_op_movl_T0_im(selector);
-            gen_op_movl_T1_im(offset);
+            gen_op_movl_T1_imu(offset);
         }
         goto do_ljmp;
     case 0xeb: /* jmp Jb */
-        val = (int8_t)insn_get(s, OT_BYTE);
-        val += s->pc - s->cs_base;
+        tval = (int8_t)insn_get(s, OT_BYTE);
+        tval += s->pc - s->cs_base;
         if (s->dflag == 0)
-            val = val & 0xffff;
-        gen_jmp(s, val);
+            tval &= 0xffff;
+        gen_jmp(s, tval);
         break;
     case 0x70 ... 0x7f: /* jcc Jb */
-        val = (int8_t)insn_get(s, OT_BYTE);
+        tval = (int8_t)insn_get(s, OT_BYTE);
         goto do_jcc;
     case 0x180 ... 0x18f: /* jcc Jv */
         if (dflag) {
-            val = insn_get(s, OT_LONG);
+            tval = (int32_t)insn_get(s, OT_LONG);
         } else {
-            val = (int16_t)insn_get(s, OT_WORD); 
+            tval = (int16_t)insn_get(s, OT_WORD); 
         }
     do_jcc:
         next_eip = s->pc - s->cs_base;
-        val += next_eip;
+        tval += next_eip;
         if (s->dflag == 0)
-            val &= 0xffff;
-        gen_jcc(s, b, val, next_eip);
+            tval &= 0xffff;
+        gen_jcc(s, b, tval, next_eip);
         break;
  
     case 0x190 ... 0x19f: /* setcc Gv */
@@ -3547,16 +4187,16 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         gen_ldst_modrm(s, modrm, OT_BYTE, OR_TMP0, 1);
         break;
     case 0x140 ... 0x14f: /* cmov Gv, Ev */
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         gen_setcc(s, b);
         if (mod != 3) {
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T1_A0[ot + s->mem_index]();
         } else {
-            rm = modrm & 7;
+            rm = (modrm & 7) | REX_B(s);
             gen_op_mov_TN_reg[ot][1][rm]();
         }
         gen_op_cmov_reg_T1_T0[ot - OT_WORD][reg]();
@@ -3603,11 +4243,13 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             gen_pop_update(s);
             s->cc_op = CC_OP_EFLAGS;
             /* abort translation because TF flag may change */
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
         break;
     case 0x9e: /* sahf */
+        if (CODE64(s))
+            goto illegal_op;
         gen_op_mov_TN_reg[OT_BYTE][0][R_AH]();
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
@@ -3615,6 +4257,8 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         s->cc_op = CC_OP_EFLAGS;
         break;
     case 0x9f: /* lahf */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_movl_T0_eflags();
@@ -3648,12 +4292,13 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         /************************/
         /* bit operations */
     case 0x1ba: /* bt/bts/btr/btc Gv, im */
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        op = (modrm >> 3) & 7;
+        op = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
+        rm = (modrm & 7) | REX_B(s);
         if (mod != 3) {
+            s->rip_offset = 1;
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T0_A0[ot + s->mem_index]();
         } else {
@@ -3687,19 +4332,16 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
     case 0x1bb: /* btc */
         op = 3;
     do_btx:
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
+        rm = (modrm & 7) | REX_B(s);
         gen_op_mov_TN_reg[OT_LONG][1][reg]();
         if (mod != 3) {
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
             /* specific case: we need to add a displacement */
-            if (ot == OT_WORD)
-                gen_op_add_bitw_A0_T1();
-            else
-                gen_op_add_bitl_A0_T1();
+            gen_op_add_bit_A0_T1[ot - OT_WORD]();
             gen_op_ld_T0_A0[ot + s->mem_index]();
         } else {
             gen_op_mov_TN_reg[ot][0][rm]();
@@ -3716,9 +4358,9 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         break;
     case 0x1bc: /* bsf */
     case 0x1bd: /* bsr */
-        ot = dflag ? OT_LONG : OT_WORD;
+        ot = dflag + OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
         /* NOTE: in order to handle the 0 case, we must load the
            result. It could be optimized with a generated jump */
@@ -3730,35 +4372,47 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         /************************/
         /* bcd */
     case 0x27: /* daa */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_daa();
         s->cc_op = CC_OP_EFLAGS;
         break;
     case 0x2f: /* das */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_das();
         s->cc_op = CC_OP_EFLAGS;
         break;
     case 0x37: /* aaa */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_aaa();
         s->cc_op = CC_OP_EFLAGS;
         break;
     case 0x3f: /* aas */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_aas();
         s->cc_op = CC_OP_EFLAGS;
         break;
     case 0xd4: /* aam */
+        if (CODE64(s))
+            goto illegal_op;
         val = ldub_code(s->pc++);
         gen_op_aam(val);
         s->cc_op = CC_OP_LOGICB;
         break;
     case 0xd5: /* aad */
+        if (CODE64(s))
+            goto illegal_op;
         val = ldub_code(s->pc++);
         gen_op_aad(val);
         s->cc_op = CC_OP_LOGICB;
@@ -3766,6 +4420,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         /************************/
         /* misc */
     case 0x90: /* nop */
+        /* XXX: xchg + rex handling */
         /* XXX: correct lock test for all insn */
         if (prefixes & PREFIX_LOCK)
             goto illegal_op;
@@ -3777,7 +4432,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         } else {
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_op_jmp_im(pc_start - s->cs_base);
+            gen_jmp_im(pc_start - s->cs_base);
             gen_op_fwait();
         }
         break;
@@ -3793,12 +4448,19 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         }
         break;
     case 0xce: /* into */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_into(s->pc - s->cs_base);
         break;
     case 0xf1: /* icebp (undocumented, exits to external debugger) */
+#if 0
         gen_debug(s, pc_start - s->cs_base);
+#else
+        /* test ! */
+        cpu_set_log(CPU_LOG_TB_IN_ASM | CPU_LOG_PCALL);
+#endif
         break;
     case 0xfa: /* cli */
         if (!s->vm86) {
@@ -3826,7 +4488,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
                     gen_op_set_inhibit_irq();
                 /* give a chance to handle pending irqs */
-                gen_op_jmp_im(s->pc - s->cs_base);
+                gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
             } else {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
@@ -3840,6 +4502,8 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         }
         break;
     case 0x62: /* bound */
+        if (CODE64(s))
+            goto illegal_op;
         ot = dflag ? OT_LONG : OT_WORD;
         modrm = ldub_code(s->pc++);
         reg = (modrm >> 3) & 7;
@@ -3848,18 +4512,30 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             goto illegal_op;
         gen_op_mov_TN_reg[ot][0][reg]();
         gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_jmp_im(pc_start - s->cs_base);
         if (ot == OT_WORD)
-            gen_op_boundw(pc_start - s->cs_base);
+            gen_op_boundw();
         else
-            gen_op_boundl(pc_start - s->cs_base);
+            gen_op_boundl();
         break;
     case 0x1c8 ... 0x1cf: /* bswap reg */
-        reg = b & 7;
-        gen_op_mov_TN_reg[OT_LONG][0][reg]();
-        gen_op_bswapl_T0();
-        gen_op_mov_reg_T0[OT_LONG][reg]();
+        reg = (b & 7) | REX_B(s);
+#ifdef TARGET_X86_64
+        if (dflag == 2) {
+            gen_op_mov_TN_reg[OT_QUAD][0][reg]();
+            gen_op_bswapq_T0();
+            gen_op_mov_reg_T0[OT_QUAD][reg]();
+        } else 
+#endif
+        {
+            gen_op_mov_TN_reg[OT_LONG][0][reg]();
+            gen_op_bswapl_T0();
+            gen_op_mov_reg_T0[OT_LONG][reg]();
+        }
         break;
     case 0xd6: /* salc */
+        if (CODE64(s))
+            goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_op_salc();
@@ -3871,13 +4547,32 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         /* FALL THRU */
     case 0xe2: /* loop */
     case 0xe3: /* jecxz */
-        val = (int8_t)insn_get(s, OT_BYTE);
-        next_eip = s->pc - s->cs_base;
-        val += next_eip;
-        if (s->dflag == 0)
-            val &= 0xffff;
-        gen_op_loop[s->aflag][b & 3](val, next_eip);
-        gen_eob(s);
+        {
+            int l1, l2;
+
+            tval = (int8_t)insn_get(s, OT_BYTE);
+            next_eip = s->pc - s->cs_base;
+            tval += next_eip;
+            if (s->dflag == 0)
+                tval &= 0xffff;
+            
+            l1 = gen_new_label();
+            l2 = gen_new_label();
+            b &= 3;
+            if (b == 3) {
+                gen_op_jz_ecx[s->aflag](l1);
+            } else {
+                gen_op_dec_ECX[s->aflag]();
+                gen_op_loop[s->aflag][b](l1);
+            }
+
+            gen_jmp_im(next_eip);
+            gen_op_jmp_label(l2);
+            gen_set_label(l1);
+            gen_jmp_im(tval);
+            gen_set_label(l2);
+            gen_eob(s);
+        }
         break;
     case 0x130: /* wrmsr */
     case 0x132: /* rdmsr */
@@ -3894,6 +4589,8 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         gen_op_rdtsc();
         break;
     case 0x134: /* sysenter */
+        if (CODE64(s))
+            goto illegal_op;
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -3901,12 +4598,14 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 gen_op_set_cc_op(s->cc_op);
                 s->cc_op = CC_OP_DYNAMIC;
             }
-            gen_op_jmp_im(pc_start - s->cs_base);
+            gen_jmp_im(pc_start - s->cs_base);
             gen_op_sysenter();
             gen_eob(s);
         }
         break;
     case 0x135: /* sysexit */
+        if (CODE64(s))
+            goto illegal_op;
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -3914,11 +4613,36 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 gen_op_set_cc_op(s->cc_op);
                 s->cc_op = CC_OP_DYNAMIC;
             }
-            gen_op_jmp_im(pc_start - s->cs_base);
+            gen_jmp_im(pc_start - s->cs_base);
             gen_op_sysexit();
             gen_eob(s);
         }
         break;
+#ifdef TARGET_X86_64
+    case 0x105: /* syscall */
+        /* XXX: is it usable in real mode ? */
+        if (s->cc_op != CC_OP_DYNAMIC) {
+            gen_op_set_cc_op(s->cc_op);
+            s->cc_op = CC_OP_DYNAMIC;
+        }
+        gen_jmp_im(pc_start - s->cs_base);
+        gen_op_syscall();
+        gen_eob(s);
+        break;
+    case 0x107: /* sysret */
+        if (!s->pe) {
+            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+        } else {
+            if (s->cc_op != CC_OP_DYNAMIC) {
+                gen_op_set_cc_op(s->cc_op);
+                s->cc_op = CC_OP_DYNAMIC;
+            }
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_op_sysret(s->dflag);
+            gen_eob(s);
+        }
+        break;
+#endif
     case 0x1a2: /* cpuid */
         gen_op_cpuid();
         break;
@@ -3928,7 +4652,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         } else {
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_op_hlt();
             s->is_jmp = 3;
         }
@@ -3954,7 +4678,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
-                gen_op_jmp_im(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
                 gen_op_lldt_T0();
             }
             break;
@@ -3974,7 +4698,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
-                gen_op_jmp_im(pc_start - s->cs_base);
+                gen_jmp_im(pc_start - s->cs_base);
                 gen_op_ltr_T0();
             }
             break;
@@ -4010,14 +4734,19 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             else
                 gen_op_movl_T0_env(offsetof(CPUX86State,idt.limit));
             gen_op_st_T0_A0[OT_WORD + s->mem_index]();
-            gen_op_addl_A0_im(2);
+#ifdef TARGET_X86_64
+            if (CODE64(s)) 
+                gen_op_addq_A0_im(2);
+            else
+#endif
+                gen_op_addl_A0_im(2);
             if (op == 0)
-                gen_op_movl_T0_env(offsetof(CPUX86State,gdt.base));
+                gen_op_movtl_T0_env(offsetof(CPUX86State,gdt.base));
             else
-                gen_op_movl_T0_env(offsetof(CPUX86State,idt.base));
+                gen_op_movtl_T0_env(offsetof(CPUX86State,idt.base));
             if (!s->dflag)
                 gen_op_andl_T0_im(0xffffff);
-            gen_op_st_T0_A0[OT_LONG + s->mem_index]();
+            gen_op_st_T0_A0[CODE64(s) + OT_LONG + s->mem_index]();
             break;
         case 2: /* lgdt */
         case 3: /* lidt */
@@ -4028,15 +4757,20 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             } else {
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_T1_A0[OT_WORD + s->mem_index]();
-                gen_op_addl_A0_im(2);
-                gen_op_ld_T0_A0[OT_LONG + s->mem_index]();
+#ifdef TARGET_X86_64
+                if (CODE64(s))
+                    gen_op_addq_A0_im(2);
+                else
+#endif
+                    gen_op_addl_A0_im(2);
+                gen_op_ld_T0_A0[CODE64(s) + OT_LONG + s->mem_index]();
                 if (!s->dflag)
                     gen_op_andl_T0_im(0xffffff);
                 if (op == 2) {
-                    gen_op_movl_env_T0(offsetof(CPUX86State,gdt.base));
+                    gen_op_movtl_env_T0(offsetof(CPUX86State,gdt.base));
                     gen_op_movl_env_T1(offsetof(CPUX86State,gdt.limit));
                 } else {
-                    gen_op_movl_env_T0(offsetof(CPUX86State,idt.base));
+                    gen_op_movtl_env_T0(offsetof(CPUX86State,idt.base));
                     gen_op_movl_env_T1(offsetof(CPUX86State,idt.limit));
                 }
             }
@@ -4051,7 +4785,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             } else {
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_op_lmsw_T0();
-                gen_op_jmp_im(s->pc - s->cs_base);
+                gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
             }
             break;
@@ -4059,12 +4793,25 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
-                if (mod == 3)
-                    goto illegal_op;
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
-                gen_op_invlpg_A0();
-                gen_op_jmp_im(s->pc - s->cs_base);
-                gen_eob(s);
+                if (mod == 3) {
+#ifdef TARGET_X86_64
+                    if (CODE64(s) && (modrm & 7) == 0) {
+                        /* swapgs */
+                        gen_op_movtl_T0_env(offsetof(CPUX86State,segs[R_GS].base));
+                        gen_op_movtl_T1_env(offsetof(CPUX86State,kernelgsbase));
+                        gen_op_movtl_env_T1(offsetof(CPUX86State,segs[R_GS].base));
+                        gen_op_movtl_env_T0(offsetof(CPUX86State,kernelgsbase));
+                    } else 
+#endif
+                    {
+                        goto illegal_op;
+                    }
+                } else {
+                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_op_invlpg_A0();
+                    gen_jmp_im(s->pc - s->cs_base);
+                    gen_eob(s);
+                }
             }
             break;
         default:
@@ -4079,30 +4826,87 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             /* nothing to do */
         }
         break;
-    case 0x63: /* arpl */
-        if (!s->pe || s->vm86)
-            goto illegal_op;
-        ot = dflag ? OT_LONG : OT_WORD;
+    case 0x1ae: /* sfence */
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
-        rm = modrm & 7;
-        if (mod != 3) {
+        op = (modrm >> 3) & 7;
+        switch(op) {
+        case 0: /* fxsave */
+            if (mod == 3 || !(s->cpuid_features & CPUID_FXSR))
+                goto illegal_op;
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
-            gen_op_ld_T0_A0[ot + s->mem_index]();
-        } else {
-            gen_op_mov_TN_reg[ot][0][rm]();
+            gen_op_fxsave_A0((s->dflag == 2));
+            break;
+        case 1: /* fxrstor */
+            if (mod == 3 || !(s->cpuid_features & CPUID_FXSR))
+                goto illegal_op;
+            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_op_fxrstor_A0((s->dflag == 2));
+            break;
+        case 5: /* lfence */
+        case 6: /* mfence */
+        case 7: /* sfence */
+            if ((modrm & 0xc7) != 0xc0 || !(s->cpuid_features & CPUID_SSE))
+                goto illegal_op;
+            break;
+        default:
+            goto illegal_op;
         }
-        if (s->cc_op != CC_OP_DYNAMIC)
-            gen_op_set_cc_op(s->cc_op);
-        gen_op_arpl();
-        s->cc_op = CC_OP_EFLAGS;
-        if (mod != 3) {
-            gen_op_st_T0_A0[ot + s->mem_index]();
-        } else {
-            gen_op_mov_reg_T0[ot][rm]();
+        break;
+    case 0x63: /* arpl or movslS (x86_64) */
+#ifdef TARGET_X86_64
+        if (CODE64(s)) {
+            int d_ot;
+            /* d_ot is the size of destination */
+            d_ot = dflag + OT_WORD;
+
+            modrm = ldub_code(s->pc++);
+            reg = ((modrm >> 3) & 7) | rex_r;
+            mod = (modrm >> 6) & 3;
+            rm = (modrm & 7) | REX_B(s);
+            
+            if (mod == 3) {
+                gen_op_mov_TN_reg[OT_LONG][0][rm]();
+                /* sign extend */
+                if (d_ot == OT_QUAD)
+                    gen_op_movslq_T0_T0();
+                gen_op_mov_reg_T0[d_ot][reg]();
+            } else {
+                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                if (d_ot == OT_QUAD) {
+                    gen_op_lds_T0_A0[OT_LONG + s->mem_index]();
+                } else {
+                    gen_op_ld_T0_A0[OT_LONG + s->mem_index]();
+                }
+                gen_op_mov_reg_T0[d_ot][reg]();
+            }
+        } else 
+#endif
+        {
+            if (!s->pe || s->vm86)
+                goto illegal_op;
+            ot = dflag ? OT_LONG : OT_WORD;
+            modrm = ldub_code(s->pc++);
+            reg = (modrm >> 3) & 7;
+            mod = (modrm >> 6) & 3;
+            rm = modrm & 7;
+            if (mod != 3) {
+                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_op_ld_T0_A0[ot + s->mem_index]();
+            } else {
+                gen_op_mov_TN_reg[ot][0][rm]();
+            }
+            if (s->cc_op != CC_OP_DYNAMIC)
+                gen_op_set_cc_op(s->cc_op);
+            gen_op_arpl();
+            s->cc_op = CC_OP_EFLAGS;
+            if (mod != 3) {
+                gen_op_st_T0_A0[ot + s->mem_index]();
+            } else {
+                gen_op_mov_reg_T0[ot][rm]();
+            }
+            gen_op_arpl_update();
         }
-        gen_op_arpl_update();
         break;
     case 0x102: /* lar */
     case 0x103: /* lsl */
@@ -4110,7 +4914,7 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             goto illegal_op;
         ot = dflag ? OT_LONG : OT_WORD;
         modrm = ldub_code(s->pc++);
-        reg = (modrm >> 3) & 7;
+        reg = ((modrm >> 3) & 7) | rex_r;
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_TN_reg[ot][1][reg]();
         if (s->cc_op != CC_OP_DYNAMIC)
@@ -4148,23 +4952,28 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             modrm = ldub_code(s->pc++);
             if ((modrm & 0xc0) != 0xc0)
                 goto illegal_op;
-            rm = modrm & 7;
-            reg = (modrm >> 3) & 7;
+            rm = (modrm & 7) | REX_B(s);
+            reg = ((modrm >> 3) & 7) | rex_r;
+            if (CODE64(s))
+                ot = OT_QUAD;
+            else
+                ot = OT_LONG;
             switch(reg) {
             case 0:
             case 2:
             case 3:
             case 4:
                 if (b & 2) {
-                    gen_op_mov_TN_reg[OT_LONG][0][rm]();
+                    gen_op_mov_TN_reg[ot][0][rm]();
                     gen_op_movl_crN_T0(reg);
-                    gen_op_jmp_im(s->pc - s->cs_base);
+                    gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
                 } else {
-                    gen_op_movl_T0_env(offsetof(CPUX86State,cr[reg]));
-                    gen_op_mov_reg_T0[OT_LONG][rm]();
+                    gen_op_movtl_T0_env(offsetof(CPUX86State,cr[reg]));
+                    gen_op_mov_reg_T0[ot][rm]();
                 }
                 break;
+                /* XXX: add CR8 for x86_64 */
             default:
                 goto illegal_op;
             }
@@ -4178,19 +4987,23 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
             modrm = ldub_code(s->pc++);
             if ((modrm & 0xc0) != 0xc0)
                 goto illegal_op;
-            rm = modrm & 7;
-            reg = (modrm >> 3) & 7;
+            rm = (modrm & 7) | REX_B(s);
+            reg = ((modrm >> 3) & 7) | rex_r;
+            if (CODE64(s))
+                ot = OT_QUAD;
+            else
+                ot = OT_LONG;
             /* XXX: do it dynamically with CR4.DE bit */
-            if (reg == 4 || reg == 5)
+            if (reg == 4 || reg == 5 || reg >= 8)
                 goto illegal_op;
             if (b & 2) {
-                gen_op_mov_TN_reg[OT_LONG][0][rm]();
+                gen_op_mov_TN_reg[ot][0][rm]();
                 gen_op_movl_drN_T0(reg);
-                gen_op_jmp_im(s->pc - s->cs_base);
+                gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
             } else {
-                gen_op_movl_T0_env(offsetof(CPUX86State,dr[reg]));
-                gen_op_mov_reg_T0[OT_LONG][rm]();
+                gen_op_movtl_T0_env(offsetof(CPUX86State,dr[reg]));
+                gen_op_mov_reg_T0[ot][rm]();
             }
         }
         break;
@@ -4200,10 +5013,69 @@ static uint8_t *disas_insn(DisasContext *s, uint8_t *pc_start)
         } else {
             gen_op_clts();
             /* abort block because static cpu state changed */
-            gen_op_jmp_im(s->pc - s->cs_base);
+            gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
         break;
+    /* SSE support */
+    case 0x16f:
+        if (prefixes & PREFIX_DATA) {
+            /* movdqa xmm1, xmm2/mem128 */
+            if (!(s->cpuid_features & CPUID_SSE))
+                goto illegal_op;
+            modrm = ldub_code(s->pc++);
+            reg = ((modrm >> 3) & 7) | rex_r;
+            mod = (modrm >> 6) & 3;
+            if (mod != 3) {
+                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_ldo_env_A0[s->mem_index >> 2](offsetof(CPUX86State,xmm_regs[reg]));
+            } else {
+                rm = (modrm & 7) | REX_B(s);
+                gen_op_movo(offsetof(CPUX86State,xmm_regs[reg]),
+                            offsetof(CPUX86State,xmm_regs[rm]));
+            }
+        } else {
+            goto illegal_op;
+        }
+        break;
+    case 0x1e7:
+        if (prefixes & PREFIX_DATA) {
+            /* movntdq mem128, xmm1 */
+            if (!(s->cpuid_features & CPUID_SSE))
+                goto illegal_op;
+            modrm = ldub_code(s->pc++);
+            reg = ((modrm >> 3) & 7) | rex_r;
+            mod = (modrm >> 6) & 3;
+            if (mod != 3) {
+                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_sto_env_A0[s->mem_index >> 2](offsetof(CPUX86State,xmm_regs[reg]));
+            } else {
+                goto illegal_op;
+            }
+        } else {
+            goto illegal_op;
+        }
+        break;
+    case 0x17f:
+        if (prefixes & PREFIX_DATA) {
+            /* movdqa xmm2/mem128, xmm1 */
+            if (!(s->cpuid_features & CPUID_SSE))
+                goto illegal_op;
+            modrm = ldub_code(s->pc++);
+            reg = ((modrm >> 3) & 7) | rex_r;
+            mod = (modrm >> 6) & 3;
+            if (mod != 3) {
+                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_sto_env_A0[s->mem_index >> 2](offsetof(CPUX86State,xmm_regs[reg]));
+            } else {
+                rm = (modrm & 7) | REX_B(s);
+                gen_op_movo(offsetof(CPUX86State,xmm_regs[rm]),
+                            offsetof(CPUX86State,xmm_regs[reg]));
+            }
+        } else {
+            goto illegal_op;
+        }
+        break;
     default:
         goto illegal_op;
     }
@@ -4301,26 +5173,51 @@ static uint16_t opc_read_flags[NB_OPS] = {
     [INDEX_op_salc] = CC_C,
  
     /* needed for correct flag optimisation before string ops */
+    [INDEX_op_jnz_ecxw] = CC_OSZAPC,
+    [INDEX_op_jnz_ecxl] = CC_OSZAPC,
     [INDEX_op_jz_ecxw] = CC_OSZAPC,
     [INDEX_op_jz_ecxl] = CC_OSZAPC,
-    [INDEX_op_jz_ecxw_im] = CC_OSZAPC,
-    [INDEX_op_jz_ecxl_im] = CC_OSZAPC,
+
+#ifdef TARGET_X86_64
+    [INDEX_op_jb_subq] = CC_C,
+    [INDEX_op_jz_subq] = CC_Z,
+    [INDEX_op_jbe_subq] = CC_Z | CC_C,
+    [INDEX_op_js_subq] = CC_S,
+    [INDEX_op_jl_subq] = CC_O | CC_S,
+    [INDEX_op_jle_subq] = CC_O | CC_S | CC_Z,
+
+    [INDEX_op_loopnzq] = CC_Z,
+    [INDEX_op_loopzq] = CC_Z,
+
+    [INDEX_op_setb_T0_subq] = CC_C,
+    [INDEX_op_setz_T0_subq] = CC_Z,
+    [INDEX_op_setbe_T0_subq] = CC_Z | CC_C,
+    [INDEX_op_sets_T0_subq] = CC_S,
+    [INDEX_op_setl_T0_subq] = CC_O | CC_S,
+    [INDEX_op_setle_T0_subq] = CC_O | CC_S | CC_Z,
+
+    [INDEX_op_jnz_ecxq] = CC_OSZAPC,
+    [INDEX_op_jz_ecxq] = CC_OSZAPC,
+#endif
  
 #define DEF_READF(SUFFIX)\
     [INDEX_op_adcb ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_adcw ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_adcl ## SUFFIX ## _T0_T1_cc] = CC_C,\
+    X86_64_DEF([INDEX_op_adcq ## SUFFIX ## _T0_T1_cc] = CC_C,)\
     [INDEX_op_sbbb ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_sbbw ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_sbbl ## SUFFIX ## _T0_T1_cc] = CC_C,\
+    X86_64_DEF([INDEX_op_sbbq ## SUFFIX ## _T0_T1_cc] = CC_C,)\
 \
     [INDEX_op_rclb ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_rclw ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_rcll ## SUFFIX ## _T0_T1_cc] = CC_C,\
+    X86_64_DEF([INDEX_op_rclq ## SUFFIX ## _T0_T1_cc] = CC_C,)\
     [INDEX_op_rcrb ## SUFFIX ## _T0_T1_cc] = CC_C,\
     [INDEX_op_rcrw ## SUFFIX ## _T0_T1_cc] = CC_C,\
-    [INDEX_op_rcrl ## SUFFIX ## _T0_T1_cc] = CC_C,
-
+    [INDEX_op_rcrl ## SUFFIX ## _T0_T1_cc] = CC_C,\
+    X86_64_DEF([INDEX_op_rcrq ## SUFFIX ## _T0_T1_cc] = CC_C,)
  
     DEF_READF( )
     DEF_READF(_raw)
@@ -4341,14 +5238,17 @@ static uint16_t opc_write_flags[NB_OPS] = {
     [INDEX_op_testl_T0_T1_cc] = CC_OSZAPC,
  
     [INDEX_op_mulb_AL_T0] = CC_OSZAPC,
-    [INDEX_op_imulb_AL_T0] = CC_OSZAPC,
     [INDEX_op_mulw_AX_T0] = CC_OSZAPC,
-    [INDEX_op_imulw_AX_T0] = CC_OSZAPC,
     [INDEX_op_mull_EAX_T0] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_mulq_EAX_T0] = CC_OSZAPC,)
+    [INDEX_op_imulb_AL_T0] = CC_OSZAPC,
+    [INDEX_op_imulw_AX_T0] = CC_OSZAPC,
     [INDEX_op_imull_EAX_T0] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_imulq_EAX_T0] = CC_OSZAPC,)
     [INDEX_op_imulw_T0_T1] = CC_OSZAPC,
     [INDEX_op_imull_T0_T1] = CC_OSZAPC,
-    
+    X86_64_DEF([INDEX_op_imulq_T0_T1] = CC_OSZAPC,)
+
     /* bcd */
     [INDEX_op_aam] = CC_OSZAPC,
     [INDEX_op_aad] = CC_OSZAPC,
@@ -4370,21 +5270,28 @@ static uint16_t opc_write_flags[NB_OPS] = {
  
     [INDEX_op_btw_T0_T1_cc] = CC_OSZAPC,
     [INDEX_op_btl_T0_T1_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_btq_T0_T1_cc] = CC_OSZAPC,)
     [INDEX_op_btsw_T0_T1_cc] = CC_OSZAPC,
     [INDEX_op_btsl_T0_T1_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_btsq_T0_T1_cc] = CC_OSZAPC,)
     [INDEX_op_btrw_T0_T1_cc] = CC_OSZAPC,
     [INDEX_op_btrl_T0_T1_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_btrq_T0_T1_cc] = CC_OSZAPC,)
     [INDEX_op_btcw_T0_T1_cc] = CC_OSZAPC,
     [INDEX_op_btcl_T0_T1_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_btcq_T0_T1_cc] = CC_OSZAPC,)
  
     [INDEX_op_bsfw_T0_cc] = CC_OSZAPC,
     [INDEX_op_bsfl_T0_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_bsfq_T0_cc] = CC_OSZAPC,)
     [INDEX_op_bsrw_T0_cc] = CC_OSZAPC,
     [INDEX_op_bsrl_T0_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_bsrq_T0_cc] = CC_OSZAPC,)
  
     [INDEX_op_cmpxchgb_T0_T1_EAX_cc] = CC_OSZAPC,
     [INDEX_op_cmpxchgw_T0_T1_EAX_cc] = CC_OSZAPC,
     [INDEX_op_cmpxchgl_T0_T1_EAX_cc] = CC_OSZAPC,
+    X86_64_DEF([INDEX_op_cmpxchgq_T0_T1_EAX_cc] = CC_OSZAPC,)
  
     [INDEX_op_cmpxchg8b] = CC_Z,
     [INDEX_op_lar] = CC_Z,
@@ -4396,49 +5303,63 @@ static uint16_t opc_write_flags[NB_OPS] = {
     [INDEX_op_adcb ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_adcw ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_adcl ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_adcq ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,)\
     [INDEX_op_sbbb ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_sbbw ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_sbbl ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_sbbq ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,)\
 \
     [INDEX_op_rolb ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rolw ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_roll ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
+    X86_64_DEF([INDEX_op_rolq ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,)\
     [INDEX_op_rorb ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rorw ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rorl ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
+    X86_64_DEF([INDEX_op_rorq ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,)\
 \
     [INDEX_op_rclb ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rclw ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rcll ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
+    X86_64_DEF([INDEX_op_rclq ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,)\
     [INDEX_op_rcrb ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rcrw ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
     [INDEX_op_rcrl ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,\
+    X86_64_DEF([INDEX_op_rcrq ## SUFFIX ## _T0_T1_cc] = CC_O | CC_C,)\
 \
     [INDEX_op_shlb ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_shlw ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_shll ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_shlq ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,)\
 \
     [INDEX_op_shrb ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_shrw ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_shrl ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_shrq ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,)\
 \
     [INDEX_op_sarb ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_sarw ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
     [INDEX_op_sarl ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_sarq ## SUFFIX ## _T0_T1_cc] = CC_OSZAPC,)\
 \
     [INDEX_op_shldw ## SUFFIX ## _T0_T1_ECX_cc] = CC_OSZAPC,\
     [INDEX_op_shldl ## SUFFIX ## _T0_T1_ECX_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_shldq ## SUFFIX ## _T0_T1_ECX_cc] = CC_OSZAPC,)\
     [INDEX_op_shldw ## SUFFIX ## _T0_T1_im_cc] = CC_OSZAPC,\
     [INDEX_op_shldl ## SUFFIX ## _T0_T1_im_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_shldq ## SUFFIX ## _T0_T1_im_cc] = CC_OSZAPC,)\
 \
     [INDEX_op_shrdw ## SUFFIX ## _T0_T1_ECX_cc] = CC_OSZAPC,\
     [INDEX_op_shrdl ## SUFFIX ## _T0_T1_ECX_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_shrdq ## SUFFIX ## _T0_T1_ECX_cc] = CC_OSZAPC,)\
     [INDEX_op_shrdw ## SUFFIX ## _T0_T1_im_cc] = CC_OSZAPC,\
     [INDEX_op_shrdl ## SUFFIX ## _T0_T1_im_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_shrdq ## SUFFIX ## _T0_T1_im_cc] = CC_OSZAPC,)\
 \
     [INDEX_op_cmpxchgb ## SUFFIX ## _T0_T1_EAX_cc] = CC_OSZAPC,\
     [INDEX_op_cmpxchgw ## SUFFIX ## _T0_T1_EAX_cc] = CC_OSZAPC,\
-    [INDEX_op_cmpxchgl ## SUFFIX ## _T0_T1_EAX_cc] = CC_OSZAPC,
+    [INDEX_op_cmpxchgl ## SUFFIX ## _T0_T1_EAX_cc] = CC_OSZAPC,\
+    X86_64_DEF([INDEX_op_cmpxchgq ## SUFFIX ## _T0_T1_EAX_cc] = CC_OSZAPC,)
  
  
     DEF_WRITEF( )
@@ -4462,23 +5383,28 @@ static uint16_t opc_simpler[NB_OPS] = {
     [INDEX_op_shlb_T0_T1_cc] = INDEX_op_shlb_T0_T1,
     [INDEX_op_shlw_T0_T1_cc] = INDEX_op_shlw_T0_T1,
     [INDEX_op_shll_T0_T1_cc] = INDEX_op_shll_T0_T1,
+    X86_64_DEF([INDEX_op_shlq_T0_T1_cc] = INDEX_op_shlq_T0_T1,)
  
     [INDEX_op_shrb_T0_T1_cc] = INDEX_op_shrb_T0_T1,
     [INDEX_op_shrw_T0_T1_cc] = INDEX_op_shrw_T0_T1,
     [INDEX_op_shrl_T0_T1_cc] = INDEX_op_shrl_T0_T1,
+    X86_64_DEF([INDEX_op_shrq_T0_T1_cc] = INDEX_op_shrq_T0_T1,)
  
     [INDEX_op_sarb_T0_T1_cc] = INDEX_op_sarb_T0_T1,
     [INDEX_op_sarw_T0_T1_cc] = INDEX_op_sarw_T0_T1,
     [INDEX_op_sarl_T0_T1_cc] = INDEX_op_sarl_T0_T1,
+    X86_64_DEF([INDEX_op_sarq_T0_T1_cc] = INDEX_op_sarq_T0_T1,)
  
 #define DEF_SIMPLER(SUFFIX)\
     [INDEX_op_rolb ## SUFFIX ## _T0_T1_cc] = INDEX_op_rolb ## SUFFIX ## _T0_T1,\
     [INDEX_op_rolw ## SUFFIX ## _T0_T1_cc] = INDEX_op_rolw ## SUFFIX ## _T0_T1,\
     [INDEX_op_roll ## SUFFIX ## _T0_T1_cc] = INDEX_op_roll ## SUFFIX ## _T0_T1,\
+    X86_64_DEF([INDEX_op_rolq ## SUFFIX ## _T0_T1_cc] = INDEX_op_rolq ## SUFFIX ## _T0_T1,)\
 \
     [INDEX_op_rorb ## SUFFIX ## _T0_T1_cc] = INDEX_op_rorb ## SUFFIX ## _T0_T1,\
     [INDEX_op_rorw ## SUFFIX ## _T0_T1_cc] = INDEX_op_rorw ## SUFFIX ## _T0_T1,\
-    [INDEX_op_rorl ## SUFFIX ## _T0_T1_cc] = INDEX_op_rorl ## SUFFIX ## _T0_T1,
+    [INDEX_op_rorl ## SUFFIX ## _T0_T1_cc] = INDEX_op_rorl ## SUFFIX ## _T0_T1,\
+    X86_64_DEF([INDEX_op_rorq ## SUFFIX ## _T0_T1_cc] = INDEX_op_rorq ## SUFFIX ## _T0_T1,)
  
     DEF_SIMPLER( )
     DEF_SIMPLER(_raw)
@@ -4533,15 +5459,15 @@ static inline int gen_intermediate_code_internal(CPUState *env,
                                                  int search_pc)
 {
     DisasContext dc1, *dc = &dc1;
-    uint8_t *pc_ptr;
+    target_ulong pc_ptr;
     uint16_t *gen_opc_end;
     int flags, j, lj, cflags;
-    uint8_t *pc_start;
-    uint8_t *cs_base;
+    target_ulong pc_start;
+    target_ulong cs_base;
  
     /* generate intermediate code */
-    pc_start = (uint8_t *)tb->pc;
-    cs_base = (uint8_t *)tb->cs_base;
+    pc_start = tb->pc;
+    cs_base = tb->cs_base;
     flags = tb->flags;
     cflags = tb->cflags;
  
@@ -4563,10 +5489,15 @@ static inline int gen_intermediate_code_internal(CPUState *env,
     dc->mem_index = 0;
     if (flags & HF_SOFTMMU_MASK) {
         if (dc->cpl == 3)
-            dc->mem_index = 6;
+            dc->mem_index = 2 * 4;
         else
-            dc->mem_index = 3;
+            dc->mem_index = 1 * 4;
     }
+    dc->cpuid_features = env->cpuid_features;
+#ifdef TARGET_X86_64
+    dc->lma = (flags >> HF_LMA_SHIFT) & 1;
+    dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
+#endif
     dc->flags = flags;
     dc->jmp_opt = !(dc->tf || env->singlestep_enabled ||
                     (flags & HF_INHIBIT_IRQ_MASK)
@@ -4583,6 +5514,7 @@ static inline int gen_intermediate_code_internal(CPUState *env,
     gen_opc_ptr = gen_opc_buf;
     gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
     gen_opparam_ptr = gen_opparam_buf;
+    nb_gen_labels = 0;
  
     dc->is_jmp = DISAS_NEXT;
     pc_ptr = pc_start;
@@ -4591,7 +5523,7 @@ static inline int gen_intermediate_code_internal(CPUState *env,
     for(;;) {
         if (env->nb_breakpoints > 0) {
             for(j = 0; j < env->nb_breakpoints; j++) {
-                if (env->breakpoints[j] == (unsigned long)pc_ptr) {
+                if (env->breakpoints[j] == pc_ptr) {
                     gen_debug(dc, pc_ptr - dc->cs_base);
                     break;
                 }
@@ -4604,7 +5536,7 @@ static inline int gen_intermediate_code_internal(CPUState *env,
                 while (lj < j)
                     gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = (uint32_t)pc_ptr;
+            gen_opc_pc[lj] = pc_ptr;
             gen_opc_cc_op[lj] = dc->cc_op;
             gen_opc_instr_start[lj] = 1;
         }
@@ -4620,14 +5552,14 @@ static inline int gen_intermediate_code_internal(CPUState *env,
         if (dc->tf || dc->singlestep_enabled || 
             (flags & HF_INHIBIT_IRQ_MASK) ||
             (cflags & CF_SINGLE_INSN)) {
-            gen_op_jmp_im(pc_ptr - dc->cs_base);
+            gen_jmp_im(pc_ptr - dc->cs_base);
             gen_eob(dc);
             break;
         }
         /* if too long translation, stop generation too */
         if (gen_opc_ptr >= gen_opc_end ||
             (pc_ptr - pc_start) >= (TARGET_PAGE_SIZE - 32)) {
-            gen_op_jmp_im(pc_ptr - dc->cs_base);
+            gen_jmp_im(pc_ptr - dc->cs_base);
             gen_eob(dc);
             break;
         }
@@ -4646,9 +5578,16 @@ static inline int gen_intermediate_code_internal(CPUState *env,
         cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
     }
     if (loglevel & CPU_LOG_TB_IN_ASM) {
+        int disas_flags;
         fprintf(logfile, "----------------\n");
         fprintf(logfile, "IN: %s\n", lookup_symbol(pc_start));
-	disas(logfile, pc_start, pc_ptr - pc_start, 0, !dc->code32);
+#ifdef TARGET_X86_64
+        if (dc->code64)
+            disas_flags = 2;
+        else
+#endif
+            disas_flags = !dc->code32;
+	target_disas(logfile, pc_start, pc_ptr - pc_start, disas_flags);
         fprintf(logfile, "\n");
         if (loglevel & CPU_LOG_TB_OP) {
             fprintf(logfile, "OP:\n");