x86_64 target support

git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@1197 c046a42c-6fe2-441c-8c8c-71466251a162

x86_64 target support
git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@1197 c046a42c-6fe2-441c-8c8c-71466251a162
bellard
1 parent c4687878
Showing 9 changed files with 2124 additions and 687 deletions
target-i386/cpu.h
target-i386/exec.h
target-i386/helper.c
target-i386/helper2.c
target-i386/op.c
target-i386/opreg_template.h
target-i386/ops_mem.h
target-i386/ops_template.h
target-i386/ops_template_mem.h
@@ -20,7 +20,13 @@
 #ifndef CPU_I386_H
 #define CPU_I386_H
  
+#include "config.h"
+
+#ifdef TARGET_X86_64
+#define TARGET_LONG_BITS 64
+#else
 #define TARGET_LONG_BITS 32
+#endif
  
 /* target supports implicit self modifying code */
 #define TARGET_HAS_SMC
@@ -63,6 +69,8 @@
 #define DESC_G_MASK     (1 << 23)
 #define DESC_B_SHIFT    22
 #define DESC_B_MASK     (1 << DESC_B_SHIFT)
+#define DESC_L_SHIFT    21 /* x86_64 only : 64 bit code segment */
+#define DESC_L_MASK     (1 << DESC_L_SHIFT)
 #define DESC_AVL_MASK   (1 << 20)
 #define DESC_P_MASK     (1 << 15)
 #define DESC_DPL_SHIFT  13
@@ -125,6 +133,8 @@
 #define HF_EM_SHIFT         10
 #define HF_TS_SHIFT         11
 #define HF_IOPL_SHIFT       12 /* must be same as eflags */
+#define HF_LMA_SHIFT        14 /* only used on x86_64: long mode active */
+#define HF_CS64_SHIFT       15 /* only used on x86_64: 64 bit code segment  */
 #define HF_VM_SHIFT         17 /* must be same as eflags */
  
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
@@ -138,6 +148,8 @@
 #define HF_MP_MASK           (1 << HF_MP_SHIFT)
 #define HF_EM_MASK           (1 << HF_EM_SHIFT)
 #define HF_TS_MASK           (1 << HF_TS_SHIFT)
+#define HF_LMA_MASK          (1 << HF_LMA_SHIFT)
+#define HF_CS64_MASK         (1 << HF_CS64_SHIFT)
  
 #define CR0_PE_MASK  (1 << 0)
 #define CR0_MP_MASK  (1 << 1)
@@ -156,6 +168,9 @@
 #define CR4_PSE_MASK  (1 << 4)
 #define CR4_PAE_MASK  (1 << 5)
 #define CR4_PGE_MASK  (1 << 7)
+#define CR4_PCE_MASK  (1 << 8)
+#define CR4_OSFXSR_MASK (1 << 9)
+#define CR4_OSXMMEXCPT_MASK  (1 << 10)
  
 #define PG_PRESENT_BIT	0
 #define PG_RW_BIT	1
@@ -193,6 +208,44 @@
 #define MSR_IA32_SYSENTER_ESP           0x175
 #define MSR_IA32_SYSENTER_EIP           0x176
  
+#define MSR_EFER                        0xc0000080
+
+#define MSR_EFER_SCE   (1 << 0)
+#define MSR_EFER_LME   (1 << 8)
+#define MSR_EFER_LMA   (1 << 10)
+#define MSR_EFER_NXE   (1 << 11)
+#define MSR_EFER_FFXSR (1 << 14)
+
+#define MSR_STAR                        0xc0000081
+#define MSR_LSTAR                       0xc0000082
+#define MSR_CSTAR                       0xc0000083
+#define MSR_FMASK                       0xc0000084
+#define MSR_FSBASE                      0xc0000100
+#define MSR_GSBASE                      0xc0000101
+#define MSR_KERNELGSBASE                0xc0000102
+
+/* cpuid_features bits */
+#define CPUID_FP87 (1 << 0)
+#define CPUID_VME  (1 << 1)
+#define CPUID_DE   (1 << 2)
+#define CPUID_PSE  (1 << 3)
+#define CPUID_TSC  (1 << 4)
+#define CPUID_MSR  (1 << 5)
+#define CPUID_PAE  (1 << 6)
+#define CPUID_MCE  (1 << 7)
+#define CPUID_CX8  (1 << 8)
+#define CPUID_APIC (1 << 9)
+#define CPUID_SEP  (1 << 11) /* sysenter/sysexit */
+#define CPUID_MTRR (1 << 12)
+#define CPUID_PGE  (1 << 13)
+#define CPUID_MCA  (1 << 14)
+#define CPUID_CMOV (1 << 15)
+/* ... */
+#define CPUID_MMX  (1 << 23)
+#define CPUID_FXSR (1 << 24)
+#define CPUID_SSE  (1 << 25)
+#define CPUID_SSE2 (1 << 26)
+
 #define EXCP00_DIVZ	0
 #define EXCP01_SSTP	1
 #define EXCP02_NMI	2
@@ -219,42 +272,52 @@ enum {
     CC_OP_MULB, /* modify all flags, C, O = (CC_SRC != 0) */
     CC_OP_MULW,
     CC_OP_MULL,
+    CC_OP_MULQ,
  
     CC_OP_ADDB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_ADDW,
     CC_OP_ADDL,
+    CC_OP_ADDQ,
  
     CC_OP_ADCB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_ADCW,
     CC_OP_ADCL,
+    CC_OP_ADCQ,
  
     CC_OP_SUBB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_SUBW,
     CC_OP_SUBL,
+    CC_OP_SUBQ,
  
     CC_OP_SBBB, /* modify all flags, CC_DST = res, CC_SRC = src1 */
     CC_OP_SBBW,
     CC_OP_SBBL,
+    CC_OP_SBBQ,
  
     CC_OP_LOGICB, /* modify all flags, CC_DST = res */
     CC_OP_LOGICW,
     CC_OP_LOGICL,
+    CC_OP_LOGICQ,
  
     CC_OP_INCB, /* modify all flags except, CC_DST = res, CC_SRC = C */
     CC_OP_INCW,
     CC_OP_INCL,
+    CC_OP_INCQ,
  
     CC_OP_DECB, /* modify all flags except, CC_DST = res, CC_SRC = C  */
     CC_OP_DECW,
     CC_OP_DECL,
+    CC_OP_DECQ,
  
     CC_OP_SHLB, /* modify all flags, CC_DST = res, CC_SRC.msb = C */
     CC_OP_SHLW,
     CC_OP_SHLL,
+    CC_OP_SHLQ,
  
     CC_OP_SARB, /* modify all flags, CC_DST = res, CC_SRC.lsb = C */
     CC_OP_SARW,
     CC_OP_SARL,
+    CC_OP_SARQ,
  
     CC_OP_NB,
 };
@@ -271,22 +334,42 @@ typedef double CPU86_LDouble;
  
 typedef struct SegmentCache {
     uint32_t selector;
-    uint8_t *base;
+    target_ulong base;
     uint32_t limit;
     uint32_t flags;
 } SegmentCache;
  
+typedef struct {
+    union {
+        uint8_t b[16];
+        uint16_t w[8];
+        uint32_t l[4];
+        uint64_t q[2];
+    } u;
+} XMMReg;
+
+#ifdef TARGET_X86_64
+#define CPU_NB_REGS 16
+#else
+#define CPU_NB_REGS 8
+#endif
+
 typedef struct CPUX86State {
+#if TARGET_LONG_BITS > HOST_LONG_BITS
+    /* temporaries if we cannot store them in host registers */
+    target_ulong t0, t1, t2;
+#endif
+
     /* standard registers */
-    uint32_t regs[8];
-    uint32_t eip;
-    uint32_t eflags; /* eflags register. During CPU emulation, CC
+    target_ulong regs[CPU_NB_REGS];
+    target_ulong eip;
+    target_ulong eflags; /* eflags register. During CPU emulation, CC
                         flags and DF are set to zero because they are
                         stored elsewhere */
  
     /* emulator internal eflags handling */
-    uint32_t cc_src;
-    uint32_t cc_dst;
+    target_ulong cc_src;
+    target_ulong cc_dst;
     uint32_t cc_op;
     int32_t df; /* D flag : 1 if D = 0, -1 if D = 1 */
     uint32_t hflags; /* hidden flags, see HF_xxx constants */
@@ -314,10 +397,21 @@ typedef struct CPUX86State {
     SegmentCache gdt; /* only base and limit are used */
     SegmentCache idt; /* only base and limit are used */
  
+    XMMReg xmm_regs[CPU_NB_REGS];
+    XMMReg xmm_t0;
+
     /* sysenter registers */
     uint32_t sysenter_cs;
     uint32_t sysenter_esp;
     uint32_t sysenter_eip;
+#ifdef TARGET_X86_64
+    target_ulong efer;
+    target_ulong star;
+    target_ulong lstar;
+    target_ulong cstar;
+    target_ulong fmask;
+    target_ulong kernelgsbase;
+#endif
  
     /* temporary data for USE_CODE_COPY mode */
 #ifdef USE_CODE_COPY
@@ -333,8 +427,8 @@ typedef struct CPUX86State {
     int exception_is_int;
     int exception_next_eip;
     struct TranslationBlock *current_tb; /* currently executing TB */
-    uint32_t cr[5]; /* NOTE: cr1 is unused */
-    uint32_t dr[8]; /* debug registers */
+    target_ulong cr[5]; /* NOTE: cr1 is unused */
+    target_ulong dr[8]; /* debug registers */
     int interrupt_request; 
     int user_mode_only; /* user mode only simulation */
  
@@ -346,18 +440,28 @@ typedef struct CPUX86State {
        context) */
     unsigned long mem_write_pc; /* host pc at which the memory was
                                    written */
-    unsigned long mem_write_vaddr; /* target virtual addr at which the
-                                      memory was written */
+    target_ulong mem_write_vaddr; /* target virtual addr at which the
+                                     memory was written */
     /* 0 = kernel, 1 = user */
     CPUTLBEntry tlb_read[2][CPU_TLB_SIZE];
     CPUTLBEntry tlb_write[2][CPU_TLB_SIZE];
  
     /* from this point: preserved by CPU reset */
     /* ice debug support */
-    uint32_t breakpoints[MAX_BREAKPOINTS];
+    target_ulong breakpoints[MAX_BREAKPOINTS];
     int nb_breakpoints;
     int singlestep_enabled;
  
+    /* processor features (e.g. for CPUID insn) */
+    uint32_t cpuid_vendor1;
+    uint32_t cpuid_vendor2;
+    uint32_t cpuid_vendor3;
+    uint32_t cpuid_version;
+    uint32_t cpuid_features;
+
+    /* in order to simplify APIC support, we leave this pointer to the
+       user */
+    struct APICState *apic_state;
     /* user data */
     void *opaque;
 } CPUX86State;
@@ -382,7 +486,7 @@ void cpu_set_ferr(CPUX86State *s);
    cache: it synchronizes the hflags with the segment cache values */
 static inline void cpu_x86_load_seg_cache(CPUX86State *env, 
                                           int seg_reg, unsigned int selector,
-                                          uint8_t *base, unsigned int limit, 
+                                          uint32_t base, unsigned int limit, 
                                           unsigned int flags)
 {
     SegmentCache *sc;
@@ -395,27 +499,45 @@ static inline void cpu_x86_load_seg_cache(CPUX86State *env,
     sc->flags = flags;
  
     /* update the hidden flags */
-    new_hflags = (env->segs[R_CS].flags & DESC_B_MASK)
-        >> (DESC_B_SHIFT - HF_CS32_SHIFT);
-    new_hflags |= (env->segs[R_SS].flags & DESC_B_MASK)
-        >> (DESC_B_SHIFT - HF_SS32_SHIFT);
-    if (!(env->cr[0] & CR0_PE_MASK) || 
-        (env->eflags & VM_MASK) ||
-        !(new_hflags & HF_CS32_MASK)) {
-        /* XXX: try to avoid this test. The problem comes from the
-           fact that is real mode or vm86 mode we only modify the
-           'base' and 'selector' fields of the segment cache to go
-           faster. A solution may be to force addseg to one in
-           translate-i386.c. */
-        new_hflags |= HF_ADDSEG_MASK;
-    } else {
-        new_hflags |= (((unsigned long)env->segs[R_DS].base | 
-                        (unsigned long)env->segs[R_ES].base |
-                        (unsigned long)env->segs[R_SS].base) != 0) << 
-            HF_ADDSEG_SHIFT;
+    {
+        if (seg_reg == R_CS) {
+#ifdef TARGET_X86_64
+            if ((env->hflags & HF_LMA_MASK) && (flags & DESC_L_MASK)) {
+                /* long mode */
+                env->hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
+                env->hflags &= ~(HF_ADDSEG_MASK);
+            } else 
+#endif
+            {
+                /* legacy / compatibility case */
+                new_hflags = (env->segs[R_CS].flags & DESC_B_MASK)
+                    >> (DESC_B_SHIFT - HF_CS32_SHIFT);
+                env->hflags = (env->hflags & ~(HF_CS32_MASK | HF_CS64_MASK)) |
+                    new_hflags;
+            }
+        }
+        new_hflags = (env->segs[R_SS].flags & DESC_B_MASK)
+            >> (DESC_B_SHIFT - HF_SS32_SHIFT);
+        if (env->hflags & HF_CS64_MASK) {
+            /* zero base assumed for DS, ES and SS in long mode */
+        } else if (!(env->cr[0] & CR0_PE_MASK) || 
+            (env->eflags & VM_MASK) ||
+            !(new_hflags & HF_CS32_MASK)) {
+            /* XXX: try to avoid this test. The problem comes from the
+               fact that is real mode or vm86 mode we only modify the
+               'base' and 'selector' fields of the segment cache to go
+               faster. A solution may be to force addseg to one in
+               translate-i386.c. */
+            new_hflags |= HF_ADDSEG_MASK;
+        } else {
+            new_hflags |= (((unsigned long)env->segs[R_DS].base | 
+                            (unsigned long)env->segs[R_ES].base |
+                            (unsigned long)env->segs[R_SS].base) != 0) << 
+                HF_ADDSEG_SHIFT;
+        }
+        env->hflags = (env->hflags & 
+                       ~(HF_SS32_MASK | HF_ADDSEG_MASK)) | new_hflags;
     }
-    env->hflags = (env->hflags & 
-                   ~(HF_CS32_MASK | HF_SS32_MASK | HF_ADDSEG_MASK)) | new_hflags;
 }
  
 /* wrapper, just in case memory mappings must be changed */
@@ -448,6 +570,9 @@ void cpu_x86_set_a20(CPUX86State *env, int a20_state);
  
 uint64_t cpu_get_tsc(CPUX86State *env);
  
+void cpu_set_apic_base(CPUX86State *env, uint64_t val);
+uint64_t cpu_get_apic_base(CPUX86State *env);
+
 /* will be suppressed */
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
  
@@ -20,14 +20,29 @@
 #include "config.h"
 #include "dyngen-exec.h"
  
+/* XXX: factorize this mess */
+#if defined(__alpha__) || defined (__ia64__) || defined(__x86_64__)
+#define HOST_LONG_BITS 64
+#else
+#define HOST_LONG_BITS 32
+#endif
+
+#ifdef TARGET_X86_64
+#define TARGET_LONG_BITS 64
+#else
+#define TARGET_LONG_BITS 32
+#endif
+
 /* at least 4 register variables are defined */
 register struct CPUX86State *env asm(AREG0);
+
+/* XXX: use 64 bit regs if HOST_LONG_BITS == 64 */
+#if TARGET_LONG_BITS == 32
+
 register uint32_t T0 asm(AREG1);
 register uint32_t T1 asm(AREG2);
 register uint32_t T2 asm(AREG3);
  
-#define A0 T2
-
 /* if more registers are available, we define some registers too */
 #ifdef AREG4
 register uint32_t EAX asm(AREG4);
@@ -69,6 +84,17 @@ register uint32_t EDI asm(AREG11);
 #define reg_EDI
 #endif
  
+#else
+
+/* no registers can be used */
+#define T0 (env->t0)
+#define T1 (env->t1)
+#define T2 (env->t2)
+
+#endif
+
+#define A0 T2
+
 extern FILE *logfile;
 extern int loglevel;
  
@@ -136,26 +162,24 @@ void helper_movl_crN_T0(int reg);
 void helper_movl_drN_T0(int reg);
 void helper_invlpg(unsigned int addr);
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
-void cpu_x86_update_cr3(CPUX86State *env, uint32_t new_cr3);
+void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
 void cpu_x86_flush_tlb(CPUX86State *env, uint32_t addr);
-int cpu_x86_handle_mmu_fault(CPUX86State *env, uint32_t addr, 
+int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr, 
                              int is_write, int is_user, int is_softmmu);
-void tlb_fill(unsigned long addr, int is_write, int is_user, 
+void tlb_fill(target_ulong addr, int is_write, int is_user, 
               void *retaddr);
 void __hidden cpu_lock(void);
 void __hidden cpu_unlock(void);
 void do_interrupt(int intno, int is_int, int error_code, 
-                  unsigned int next_eip, int is_hw);
+                  target_ulong next_eip, int is_hw);
 void do_interrupt_user(int intno, int is_int, int error_code, 
-                       unsigned int next_eip);
+                       target_ulong next_eip);
 void raise_interrupt(int intno, int is_int, int error_code, 
                      unsigned int next_eip);
 void raise_exception_err(int exception_index, int error_code);
 void raise_exception(int exception_index);
 void __hidden cpu_loop_exit(void);
-void helper_fsave(uint8_t *ptr, int data32);
-void helper_frstor(uint8_t *ptr, int data32);
  
 void OPPROTO op_movl_eflags_T0(void);
 void OPPROTO op_movl_T0_eflags(void);
@@ -163,13 +187,20 @@ void raise_interrupt(int intno, int is_int, int error_code,
                      unsigned int next_eip);
 void raise_exception_err(int exception_index, int error_code);
 void raise_exception(int exception_index);
-void helper_divl_EAX_T0(uint32_t eip);
-void helper_idivl_EAX_T0(uint32_t eip);
+void helper_divl_EAX_T0(void);
+void helper_idivl_EAX_T0(void);
+void helper_mulq_EAX_T0(void);
+void helper_imulq_EAX_T0(void);
+void helper_imulq_T0_T1(void);
+void helper_divq_EAX_T0(void);
+void helper_idivq_EAX_T0(void);
 void helper_cmpxchg8b(void);
 void helper_cpuid(void);
 void helper_enter_level(int level, int data32);
 void helper_sysenter(void);
 void helper_sysexit(void);
+void helper_syscall(void);
+void helper_sysret(int dflag);
 void helper_rdtsc(void);
 void helper_rdmsr(void);
 void helper_wrmsr(void);
@@ -252,7 +283,7 @@ void check_iol_DX(void);
 #define stl(p, v) stl_data(p, v)
 #define stq(p, v) stq_data(p, v)
  
-static inline double ldfq(void *ptr)
+static inline double ldfq(target_ulong ptr)
 {
     union {
         double d;
@@ -262,7 +293,7 @@ static inline double ldfq(void *ptr)
     return u.d;
 }
  
-static inline void stfq(void *ptr, double v)
+static inline void stfq(target_ulong ptr, double v)
 {
     union {
         double d;
@@ -272,7 +303,7 @@ static inline void stfq(void *ptr, double v)
     stq(ptr, u.i);
 }
  
-static inline float ldfl(void *ptr)
+static inline float ldfl(target_ulong ptr)
 {
     union {
         float f;
@@ -282,7 +313,7 @@ static inline float ldfl(void *ptr)
     return u.f;
 }
  
-static inline void stfl(void *ptr, float v)
+static inline void stfl(target_ulong ptr, float v)
 {
     union {
         float f;
@@ -411,7 +442,7 @@ static inline void fpop(void)
 }
  
 #ifndef USE_X86LDOUBLE
-static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
+static inline CPU86_LDouble helper_fldt(target_ulong ptr)
 {
     CPU86_LDoubleU temp;
     int upper, e;
@@ -451,12 +482,12 @@ static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
  
 #ifdef CONFIG_USER_ONLY
  
-static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
+static inline CPU86_LDouble helper_fldt(target_ulong ptr)
 {
     return *(CPU86_LDouble *)ptr;
 }
  
-static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
+static inline void helper_fstt(CPU86_LDouble f, target_ulong ptr)
 {
     *(CPU86_LDouble *)ptr = f;
 }
@@ -465,7 +496,7 @@ static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
  
 /* we use memory access macros */
  
-static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
+static inline CPU86_LDouble helper_fldt(target_ulong ptr)
 {
     CPU86_LDoubleU temp;
  
@@ -474,7 +505,7 @@ static inline CPU86_LDouble helper_fldt(uint8_t *ptr)
     return temp.d;
 }
  
-static inline void helper_fstt(CPU86_LDouble f, uint8_t *ptr)
+static inline void helper_fstt(CPU86_LDouble f, target_ulong ptr)
 {
     CPU86_LDoubleU temp;
  
@@ -522,10 +553,12 @@ void helper_fscale(void);
 void helper_fsin(void);
 void helper_fcos(void);
 void helper_fxam_ST0(void);
-void helper_fstenv(uint8_t *ptr, int data32);
-void helper_fldenv(uint8_t *ptr, int data32);
-void helper_fsave(uint8_t *ptr, int data32);
-void helper_frstor(uint8_t *ptr, int data32);
+void helper_fstenv(target_ulong ptr, int data32);
+void helper_fldenv(target_ulong ptr, int data32);
+void helper_fsave(target_ulong ptr, int data32);
+void helper_frstor(target_ulong ptr, int data32);
+void helper_fxsave(target_ulong ptr, int data64);
+void helper_fxrstor(target_ulong ptr, int data64);
 void restore_native_fp_state(CPUState *env);
 void save_native_fp_state(CPUState *env);
  
@@ -119,7 +119,7 @@ static inline int load_segment(uint32_t *e1_ptr, uint32_t *e2_ptr,
 {
     SegmentCache *dt;
     int index;
-    uint8_t *ptr;
+    target_ulong ptr;
  
     if (selector & 0x4)
         dt = &env->ldt;
@@ -143,9 +143,9 @@ static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
     return limit;
 }
  
-static inline uint8_t *get_seg_base(uint32_t e1, uint32_t e2)
+static inline uint32_t get_seg_base(uint32_t e1, uint32_t e2)
 {
-    return (uint8_t *)((e1 >> 16) | ((e2 & 0xff) << 16) | (e2 & 0xff000000));
+    return ((e1 >> 16) | ((e2 & 0xff) << 16) | (e2 & 0xff000000));
 }
  
 static inline void load_seg_cache_raw_dt(SegmentCache *sc, uint32_t e1, uint32_t e2)
@@ -160,7 +160,7 @@ static inline void load_seg_vm(int seg, int selector)
 {
     selector &= 0xffff;
     cpu_x86_load_seg_cache(env, seg, selector, 
-                           (uint8_t *)(selector << 4), 0xffff, 0);
+                           (selector << 4), 0xffff, 0);
 }
  
 static inline void get_ss_esp_from_tss(uint32_t *ss_ptr, 
@@ -258,13 +258,13 @@ static void switch_tss(int tss_selector,
                        uint32_t next_eip)
 {
     int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;
-    uint8_t *tss_base;
+    target_ulong tss_base;
     uint32_t new_regs[8], new_segs[6];
     uint32_t new_eflags, new_eip, new_cr3, new_ldt, new_trap;
     uint32_t old_eflags, eflags_mask;
     SegmentCache *dt;
     int index;
-    uint8_t *ptr;
+    target_ulong ptr;
  
     type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
 #ifdef DEBUG_PCALL
@@ -345,7 +345,7 @@ static void switch_tss(int tss_selector,
  
     /* clear busy bit (it is restartable) */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_IRET) {
-        uint8_t *ptr;
+        target_ulong ptr;
         uint32_t e2;
         ptr = env->gdt.base + (env->tr.selector & ~7);
         e2 = ldl_kernel(ptr + 4);
@@ -397,7 +397,7 @@ static void switch_tss(int tss_selector,
  
     /* set busy bit */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_CALL) {
-        uint8_t *ptr;
+        target_ulong ptr;
         uint32_t e2;
         ptr = env->gdt.base + (tss_selector & ~7);
         e2 = ldl_kernel(ptr + 4);
@@ -445,11 +445,11 @@ static void switch_tss(int tss_selector,
         cpu_x86_set_cpl(env, new_segs[R_CS] & 3);
         /* first just selectors as the rest may trigger exceptions */
         for(i = 0; i < 6; i++)
-            cpu_x86_load_seg_cache(env, i, new_segs[i], NULL, 0, 0);
+            cpu_x86_load_seg_cache(env, i, new_segs[i], 0, 0, 0);
     }
  
     env->ldt.selector = new_ldt & ~4;
-    env->ldt.base = NULL;
+    env->ldt.base = 0;
     env->ldt.limit = 0;
     env->ldt.flags = 0;
  
@@ -573,7 +573,7 @@ static inline unsigned int get_sp_mask(unsigned int e2)
  
 #define POPL(ssp, sp, sp_mask, val)\
 {\
-    val = ldl_kernel((ssp) + (sp & (sp_mask)));\
+    val = (uint32_t)ldl_kernel((ssp) + (sp & (sp_mask)));\
     sp += 4;\
 }
  
@@ -582,7 +582,7 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
                                    unsigned int next_eip, int is_hw)
 {
     SegmentCache *dt;
-    uint8_t *ptr, *ssp;
+    target_ulong ptr, ssp;
     int type, dpl, selector, ss_dpl, cpl, sp_mask;
     int has_error_code, new_stack, shift;
     uint32_t e1, e2, offset, ss, esp, ss_e1, ss_e2;
@@ -703,7 +703,7 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
         raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         new_stack = 0; /* avoid warning */
         sp_mask = 0; /* avoid warning */
-        ssp = NULL; /* avoid warning */
+        ssp = 0; /* avoid warning */
         esp = 0; /* avoid warning */
     }
  
@@ -754,10 +754,10 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
  
     if (new_stack) {
         if (env->eflags & VM_MASK) {
-            cpu_x86_load_seg_cache(env, R_ES, 0, NULL, 0, 0);
-            cpu_x86_load_seg_cache(env, R_DS, 0, NULL, 0, 0);
-            cpu_x86_load_seg_cache(env, R_FS, 0, NULL, 0, 0);
-            cpu_x86_load_seg_cache(env, R_GS, 0, NULL, 0, 0);
+            cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0, 0);
+            cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0, 0);
+            cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0, 0);
+            cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0, 0);
         }
         ss = (ss & ~3) | dpl;
         cpu_x86_load_seg_cache(env, R_SS, ss, 
@@ -780,12 +780,264 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
     env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);
 }
  
+#ifdef TARGET_X86_64
+
+#define PUSHQ(sp, val)\
+{\
+    sp -= 8;\
+    stq_kernel(sp, (val));\
+}
+
+#define POPQ(sp, val)\
+{\
+    val = ldq_kernel(sp);\
+    sp += 8;\
+}
+
+static inline target_ulong get_rsp_from_tss(int level)
+{
+    int index;
+    
+#if 0
+    printf("TR: base=" TARGET_FMT_lx " limit=%x\n", 
+           env->tr.base, env->tr.limit);
+#endif
+
+    if (!(env->tr.flags & DESC_P_MASK))
+        cpu_abort(env, "invalid tss");
+    index = 8 * level + 4;
+    if ((index + 7) > env->tr.limit)
+        raise_exception_err(EXCP0A_TSS, env->tr.selector & 0xfffc);
+    return ldq_kernel(env->tr.base + index);
+}
+
+/* 64 bit interrupt */
+static void do_interrupt64(int intno, int is_int, int error_code,
+                           target_ulong next_eip, int is_hw)
+{
+    SegmentCache *dt;
+    target_ulong ptr;
+    int type, dpl, selector, cpl, ist;
+    int has_error_code, new_stack;
+    uint32_t e1, e2, e3, ss;
+    target_ulong old_eip, esp, offset;
+
+    has_error_code = 0;
+    if (!is_int && !is_hw) {
+        switch(intno) {
+        case 8:
+        case 10:
+        case 11:
+        case 12:
+        case 13:
+        case 14:
+        case 17:
+            has_error_code = 1;
+            break;
+        }
+    }
+    if (is_int)
+        old_eip = next_eip;
+    else
+        old_eip = env->eip;
+
+    dt = &env->idt;
+    if (intno * 16 + 15 > dt->limit)
+        raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
+    ptr = dt->base + intno * 16;
+    e1 = ldl_kernel(ptr);
+    e2 = ldl_kernel(ptr + 4);
+    e3 = ldl_kernel(ptr + 8);
+    /* check gate type */
+    type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
+    switch(type) {
+    case 14: /* 386 interrupt gate */
+    case 15: /* 386 trap gate */
+        break;
+    default:
+        raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
+        break;
+    }
+    dpl = (e2 >> DESC_DPL_SHIFT) & 3;
+    cpl = env->hflags & HF_CPL_MASK;
+    /* check privledge if software int */
+    if (is_int && dpl < cpl)
+        raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
+    /* check valid bit */
+    if (!(e2 & DESC_P_MASK))
+        raise_exception_err(EXCP0B_NOSEG, intno * 16 + 2);
+    selector = e1 >> 16;
+    offset = ((target_ulong)e3 << 32) | (e2 & 0xffff0000) | (e1 & 0x0000ffff);
+    ist = e2 & 7;
+    if ((selector & 0xfffc) == 0)
+        raise_exception_err(EXCP0D_GPF, 0);
+
+    if (load_segment(&e1, &e2, selector) != 0)
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK)))
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    dpl = (e2 >> DESC_DPL_SHIFT) & 3;
+    if (dpl > cpl)
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    if (!(e2 & DESC_P_MASK))
+        raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
+    if (!(e2 & DESC_L_MASK) || (e2 & DESC_B_MASK))
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+    if ((!(e2 & DESC_C_MASK) && dpl < cpl) || ist != 0) {
+        /* to inner priviledge */
+        if (ist != 0)
+            esp = get_rsp_from_tss(ist + 3);
+        else
+            esp = get_rsp_from_tss(dpl);
+        ss = 0;
+        new_stack = 1;
+    } else if ((e2 & DESC_C_MASK) || dpl == cpl) {
+        /* to same priviledge */
+        if (env->eflags & VM_MASK)
+            raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+        new_stack = 0;
+        esp = ESP & ~0xf; /* align stack */
+        dpl = cpl;
+    } else {
+        raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
+        new_stack = 0; /* avoid warning */
+        esp = 0; /* avoid warning */
+    }
+
+    PUSHQ(esp, env->segs[R_SS].selector);
+    PUSHQ(esp, ESP);
+    PUSHQ(esp, compute_eflags());
+    PUSHQ(esp, env->segs[R_CS].selector);
+    PUSHQ(esp, old_eip);
+    if (has_error_code) {
+        PUSHQ(esp, error_code);
+    }
+    
+    if (new_stack) {
+        ss = 0 | dpl;
+        cpu_x86_load_seg_cache(env, R_SS, ss, 0, 0, 0);
+    }
+    ESP = esp;
+
+    selector = (selector & ~3) | dpl;
+    cpu_x86_load_seg_cache(env, R_CS, selector, 
+                   get_seg_base(e1, e2),
+                   get_seg_limit(e1, e2),
+                   e2);
+    cpu_x86_set_cpl(env, dpl);
+    env->eip = offset;
+
+    /* interrupt gate clear IF mask */
+    if ((type & 1) == 0) {
+        env->eflags &= ~IF_MASK;
+    }
+    env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);
+}
+
+void helper_syscall(void)
+{
+    int selector;
+
+    if (!(env->efer & MSR_EFER_SCE)) {
+        raise_exception_err(EXCP06_ILLOP, 0);
+    }
+    selector = (env->star >> 32) & 0xffff;
+    if (env->hflags & HF_LMA_MASK) {
+        ECX = env->eip;
+        env->regs[11] = compute_eflags();
+
+        cpu_x86_set_cpl(env, 0);
+        cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc, 
+                           0, 0xffffffff, 
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK | DESC_L_MASK);
+        cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_W_MASK | DESC_A_MASK);
+        env->eflags &= ~env->fmask;
+        if (env->hflags & HF_CS64_MASK)
+            env->eip = env->lstar;
+        else
+            env->eip = env->cstar;
+    } else {
+        ECX = (uint32_t)env->eip;
+        
+        cpu_x86_set_cpl(env, 0);
+        cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc, 
+                           0, 0xffffffff, 
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
+        cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK |
+                               DESC_W_MASK | DESC_A_MASK);
+        env->eflags &= ~(IF_MASK | RF_MASK | VM_MASK);
+        env->eip = (uint32_t)env->star;
+    }
+}
+
+void helper_sysret(int dflag)
+{
+    int cpl, selector;
+
+    cpl = env->hflags & HF_CPL_MASK;
+    if (!(env->cr[0] & CR0_PE_MASK) || cpl != 0) {
+        raise_exception_err(EXCP0D_GPF, 0);
+    }
+    selector = (env->star >> 48) & 0xffff;
+    if (env->hflags & HF_LMA_MASK) {
+        if (dflag == 2) {
+            cpu_x86_load_seg_cache(env, R_CS, (selector + 16) | 3, 
+                                   0, 0xffffffff, 
+                                   DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                                   DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                                   DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK | 
+                                   DESC_L_MASK);
+            env->eip = ECX;
+        } else {
+            cpu_x86_load_seg_cache(env, R_CS, selector | 3, 
+                                   0, 0xffffffff, 
+                                   DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                                   DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                                   DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
+            env->eip = (uint32_t)ECX;
+        }
+        cpu_x86_load_seg_cache(env, R_SS, selector + 8, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                               DESC_W_MASK | DESC_A_MASK);
+        load_eflags((uint32_t)(env->regs[11]), 0xffffffff);
+        cpu_x86_set_cpl(env, 3);
+    } else {
+        cpu_x86_load_seg_cache(env, R_CS, selector | 3, 
+                               0, 0xffffffff, 
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                               DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
+        env->eip = (uint32_t)ECX;
+        cpu_x86_load_seg_cache(env, R_SS, selector + 8, 
+                               0, 0xffffffff,
+                               DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                               DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
+                               DESC_W_MASK | DESC_A_MASK);
+        env->eflags |= IF_MASK;
+        cpu_x86_set_cpl(env, 3);
+    }
+}
+#endif
+
 /* real mode interrupt */
 static void do_interrupt_real(int intno, int is_int, int error_code,
                               unsigned int next_eip)
 {
     SegmentCache *dt;
-    uint8_t *ptr, *ssp;
+    target_ulong ptr, ssp;
     int selector;
     uint32_t offset, esp;
     uint32_t old_cs, old_eip;
@@ -813,16 +1065,16 @@ static void do_interrupt_real(int intno, int is_int, int error_code,
     ESP = (ESP & ~0xffff) | (esp & 0xffff);
     env->eip = offset;
     env->segs[R_CS].selector = selector;
-    env->segs[R_CS].base = (uint8_t *)(selector << 4);
+    env->segs[R_CS].base = (selector << 4);
     env->eflags &= ~(IF_MASK | TF_MASK | AC_MASK | RF_MASK);
 }
  
 /* fake user mode interrupt */
 void do_interrupt_user(int intno, int is_int, int error_code, 
-                       unsigned int next_eip)
+                       target_ulong next_eip)
 {
     SegmentCache *dt;
-    uint8_t *ptr;
+    target_ulong ptr;
     int dpl, cpl;
     uint32_t e2;
  
@@ -849,26 +1101,26 @@ void do_interrupt_user(int intno, int is_int, int error_code,
  * instruction. It is only relevant if is_int is TRUE.  
  */
 void do_interrupt(int intno, int is_int, int error_code, 
-                  unsigned int next_eip, int is_hw)
+                  target_ulong next_eip, int is_hw)
 {
 #ifdef DEBUG_PCALL
     if (loglevel & (CPU_LOG_PCALL | CPU_LOG_INT)) {
         if ((env->cr[0] & CR0_PE_MASK)) {
             static int count;
-            fprintf(logfile, "%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:%08x pc=%08x SP=%04x:%08x",
+            fprintf(logfile, "%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:" TARGET_FMT_lx " pc=" TARGET_FMT_lx " SP=%04x:" TARGET_FMT_lx,
                     count, intno, error_code, is_int,
                     env->hflags & HF_CPL_MASK,
                     env->segs[R_CS].selector, EIP,
                     (int)env->segs[R_CS].base + EIP,
                     env->segs[R_SS].selector, ESP);
             if (intno == 0x0e) {
-                fprintf(logfile, " CR2=%08x", env->cr[2]);
+                fprintf(logfile, " CR2=" TARGET_FMT_lx, env->cr[2]);
             } else {
-                fprintf(logfile, " EAX=%08x", EAX);
+                fprintf(logfile, " EAX=" TARGET_FMT_lx, EAX);
             }
             fprintf(logfile, "\n");
-#if 0
             cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
+#if 0
             {
                 int i;
                 uint8_t *ptr;
@@ -885,7 +1137,14 @@ void do_interrupt(int intno, int is_int, int error_code,
     }
 #endif
     if (env->cr[0] & CR0_PE_MASK) {
-        do_interrupt_protected(intno, is_int, error_code, next_eip, is_hw);
+#if TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            do_interrupt64(intno, is_int, error_code, next_eip, is_hw);
+        } else
+#endif
+        {
+            do_interrupt_protected(intno, is_int, error_code, next_eip, is_hw);
+        }
     } else {
         do_interrupt_real(intno, is_int, error_code, next_eip);
     }
@@ -932,20 +1191,20 @@ void raise_exception(int exception_index)
 #ifdef BUGGY_GCC_DIV64
 /* gcc 2.95.4 on PowerPC does not seem to like using __udivdi3, so we
    call it from another function */
-uint32_t div64(uint32_t *q_ptr, uint64_t num, uint32_t den)
+uint32_t div32(uint32_t *q_ptr, uint64_t num, uint32_t den)
 {
     *q_ptr = num / den;
     return num % den;
 }
  
-int32_t idiv64(int32_t *q_ptr, int64_t num, int32_t den)
+int32_t idiv32(int32_t *q_ptr, int64_t num, int32_t den)
 {
     *q_ptr = num / den;
     return num % den;
 }
 #endif
  
-void helper_divl_EAX_T0(uint32_t eip)
+void helper_divl_EAX_T0(void)
 {
     unsigned int den, q, r;
     uint64_t num;
@@ -953,20 +1212,19 @@ void helper_divl_EAX_T0(uint32_t eip)
     num = EAX | ((uint64_t)EDX << 32);
     den = T0;
     if (den == 0) {
-        EIP = eip;
         raise_exception(EXCP00_DIVZ);
     }
 #ifdef BUGGY_GCC_DIV64
-    r = div64(&q, num, den);
+    r = div32(&q, num, den);
 #else
     q = (num / den);
     r = (num % den);
 #endif
-    EAX = q;
-    EDX = r;
+    EAX = (uint32_t)q;
+    EDX = (uint32_t)r;
 }
  
-void helper_idivl_EAX_T0(uint32_t eip)
+void helper_idivl_EAX_T0(void)
 {
     int den, q, r;
     int64_t num;
@@ -974,17 +1232,16 @@ void helper_idivl_EAX_T0(uint32_t eip)
     num = EAX | ((uint64_t)EDX << 32);
     den = T0;
     if (den == 0) {
-        EIP = eip;
         raise_exception(EXCP00_DIVZ);
     }
 #ifdef BUGGY_GCC_DIV64
-    r = idiv64(&q, num, den);
+    r = idiv32(&q, num, den);
 #else
     q = (num / den);
     r = (num % den);
 #endif
-    EAX = q;
-    EDX = r;
+    EAX = (uint32_t)q;
+    EDX = (uint32_t)r;
 }
  
 void helper_cmpxchg8b(void)
@@ -993,9 +1250,9 @@ void helper_cmpxchg8b(void)
     int eflags;
  
     eflags = cc_table[CC_OP].compute_all();
-    d = ldq((uint8_t *)A0);
+    d = ldq(A0);
     if (d == (((uint64_t)EDX << 32) | EAX)) {
-        stq((uint8_t *)A0, ((uint64_t)ECX << 32) | EBX);
+        stq(A0, ((uint64_t)ECX << 32) | EBX);
         eflags |= CC_Z;
     } else {
         EDX = d >> 32;
@@ -1005,58 +1262,20 @@ void helper_cmpxchg8b(void)
     CC_SRC = eflags;
 }
  
-#define CPUID_FP87 (1 << 0)
-#define CPUID_VME  (1 << 1)
-#define CPUID_DE   (1 << 2)
-#define CPUID_PSE  (1 << 3)
-#define CPUID_TSC  (1 << 4)
-#define CPUID_MSR  (1 << 5)
-#define CPUID_PAE  (1 << 6)
-#define CPUID_MCE  (1 << 7)
-#define CPUID_CX8  (1 << 8)
-#define CPUID_APIC (1 << 9)
-#define CPUID_SEP  (1 << 11) /* sysenter/sysexit */
-#define CPUID_MTRR (1 << 12)
-#define CPUID_PGE  (1 << 13)
-#define CPUID_MCA  (1 << 14)
-#define CPUID_CMOV (1 << 15)
-/* ... */
-#define CPUID_MMX  (1 << 23)
-#define CPUID_FXSR (1 << 24)
-#define CPUID_SSE  (1 << 25)
-#define CPUID_SSE2 (1 << 26)
-
 void helper_cpuid(void)
 {
-    switch(EAX) {
+    switch((uint32_t)EAX) {
     case 0:
         EAX = 2; /* max EAX index supported */
-        EBX = 0x756e6547;
-        ECX = 0x6c65746e;
-        EDX = 0x49656e69;
+        EBX = env->cpuid_vendor1;
+        EDX = env->cpuid_vendor2;
+        ECX = env->cpuid_vendor3;
         break;
     case 1:
-        {
-            int family, model, stepping;
-            /* EAX = 1 info */
-#if 0
-            /* pentium 75-200 */
-            family = 5;
-            model = 2;
-            stepping = 11;
-#else
-            /* pentium pro */
-            family = 6;
-            model = 1;
-            stepping = 3;
-#endif
-            EAX = (family << 8) | (model << 4) | stepping;
-            EBX = 0;
-            ECX = 0;
-            EDX = CPUID_FP87 | CPUID_DE | CPUID_PSE |
-                CPUID_TSC | CPUID_MSR | CPUID_MCE |
-                CPUID_CX8 | CPUID_PGE | CPUID_CMOV;
-        }
+        EAX = env->cpuid_version;
+        EBX = 0;
+        ECX = 0;
+        EDX = env->cpuid_features;
         break;
     default:
         /* cache info: needed for Pentium Pro compatibility */
@@ -1065,12 +1284,34 @@ void helper_cpuid(void)
         ECX = 0;
         EDX = 0;
         break;
+#ifdef TARGET_X86_64
+    case 0x80000000:
+        EAX = 0x80000008;
+        EBX = env->cpuid_vendor1;
+        EDX = env->cpuid_vendor2;
+        ECX = env->cpuid_vendor3;
+        break;
+    case 0x80000001:
+        EAX = env->cpuid_features;
+        EBX = 0;
+        ECX = 0;
+        /* long mode + syscall/sysret features */
+        EDX = (env->cpuid_features & 0x0183F3FF) | (1 << 29) | (1 << 11);
+        break;
+    case 0x80000008:
+        /* virtual & phys address size in low 2 bytes. */
+        EAX = 0x00003028;
+        EBX = 0;
+        ECX = 0;
+        EDX = 0;
+        break;
+#endif
     }
 }
  
 void helper_enter_level(int level, int data32)
 {
-    uint8_t *ssp;
+    target_ulong ssp;
     uint32_t esp_mask, esp, ebp;
  
     esp_mask = get_sp_mask(env->segs[R_SS].flags);
@@ -1105,20 +1346,26 @@ void helper_lldt_T0(void)
     int selector;
     SegmentCache *dt;
     uint32_t e1, e2;
-    int index;
-    uint8_t *ptr;
+    int index, entry_limit;
+    target_ulong ptr;
  
     selector = T0 & 0xffff;
     if ((selector & 0xfffc) == 0) {
         /* XXX: NULL selector case: invalid LDT */
-        env->ldt.base = NULL;
+        env->ldt.base = 0;
         env->ldt.limit = 0;
     } else {
         if (selector & 0x4)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         dt = &env->gdt;
         index = selector & ~7;
-        if ((index + 7) > dt->limit)
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK)
+            entry_limit = 15;
+        else
+#endif            
+            entry_limit = 7;
+        if ((index + entry_limit) > dt->limit)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         ptr = dt->base + index;
         e1 = ldl_kernel(ptr);
@@ -1127,7 +1374,17 @@ void helper_lldt_T0(void)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         if (!(e2 & DESC_P_MASK))
             raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
-        load_seg_cache_raw_dt(&env->ldt, e1, e2);
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            uint32_t e3;
+            e3 = ldl_kernel(ptr + 8);
+            load_seg_cache_raw_dt(&env->ldt, e1, e2);
+            env->ldt.base |= (target_ulong)e3 << 32;
+        } else
+#endif
+        {
+            load_seg_cache_raw_dt(&env->ldt, e1, e2);
+        }
     }
     env->ldt.selector = selector;
 }
@@ -1137,13 +1394,13 @@ void helper_ltr_T0(void)
     int selector;
     SegmentCache *dt;
     uint32_t e1, e2;
-    int index, type;
-    uint8_t *ptr;
+    int index, type, entry_limit;
+    target_ulong ptr;
  
     selector = T0 & 0xffff;
     if ((selector & 0xfffc) == 0) {
-        /* NULL selector case: invalid LDT */
-        env->tr.base = NULL;
+        /* NULL selector case: invalid TR */
+        env->tr.base = 0;
         env->tr.limit = 0;
         env->tr.flags = 0;
     } else {
@@ -1151,7 +1408,13 @@ void helper_ltr_T0(void)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         dt = &env->gdt;
         index = selector & ~7;
-        if ((index + 7) > dt->limit)
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK)
+            entry_limit = 15;
+        else
+#endif            
+            entry_limit = 7;
+        if ((index + entry_limit) > dt->limit)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         ptr = dt->base + index;
         e1 = ldl_kernel(ptr);
@@ -1162,7 +1425,17 @@ void helper_ltr_T0(void)
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         if (!(e2 & DESC_P_MASK))
             raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
-        load_seg_cache_raw_dt(&env->tr, e1, e2);
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            uint32_t e3;
+            e3 = ldl_kernel(ptr + 8);
+            load_seg_cache_raw_dt(&env->tr, e1, e2);
+            env->tr.base |= (target_ulong)e3 << 32;
+        } else 
+#endif
+        {
+            load_seg_cache_raw_dt(&env->tr, e1, e2);
+        }
         e2 |= DESC_TSS_BUSY_MASK;
         stl_kernel(ptr + 4, e2);
     }
@@ -1176,14 +1449,14 @@ void load_seg(int seg_reg, int selector)
     int cpl, dpl, rpl;
     SegmentCache *dt;
     int index;
-    uint8_t *ptr;
+    target_ulong ptr;
  
     selector &= 0xffff;
     if ((selector & 0xfffc) == 0) {
         /* null selector case */
         if (seg_reg == R_SS)
             raise_exception_err(EXCP0D_GPF, 0);
-        cpu_x86_load_seg_cache(env, seg_reg, selector, NULL, 0, 0);
+        cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
     } else {
  
         if (selector & 0x4)
@@ -1196,7 +1469,7 @@ void load_seg(int seg_reg, int selector)
         ptr = dt->base + index;
         e1 = ldl_kernel(ptr);
         e2 = ldl_kernel(ptr + 4);
-
+        
         if (!(e2 & DESC_S_MASK))
             raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
         rpl = selector & 3;
@@ -1247,9 +1520,10 @@ void load_seg(int seg_reg, int selector)
 /* protected mode jump */
 void helper_ljmp_protected_T0_T1(int next_eip)
 {
-    int new_cs, new_eip, gate_cs, type;
+    int new_cs, gate_cs, type;
     uint32_t e1, e2, cpl, dpl, rpl, limit;
-
+    target_ulong new_eip;
+    
     new_cs = T0;
     new_eip = T1;
     if ((new_cs & 0xfffc) == 0)
@@ -1312,7 +1586,7 @@ void helper_ljmp_protected_T0_T1(int next_eip)
             if (((e2 & (DESC_S_MASK | DESC_CS_MASK)) != 
                  (DESC_S_MASK | DESC_CS_MASK)))
                 raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
-            if (((e2 & DESC_C_MASK) && (dpl > cpl)) ||
+            if (((e2 & DESC_C_MASK) && (dpl > cpl)) || 
                 (!(e2 & DESC_C_MASK) && (dpl != cpl)))
                 raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
             if (!(e2 & DESC_P_MASK))
@@ -1336,7 +1610,7 @@ void helper_lcall_real_T0_T1(int shift, int next_eip)
 {
     int new_cs, new_eip;
     uint32_t esp, esp_mask;
-    uint8_t *ssp;
+    target_ulong ssp;
  
     new_cs = T0;
     new_eip = T1;
@@ -1354,7 +1628,7 @@ void helper_lcall_real_T0_T1(int shift, int next_eip)
     ESP = (ESP & ~esp_mask) | (esp & esp_mask);
     env->eip = new_eip;
     env->segs[R_CS].selector = new_cs;
-    env->segs[R_CS].base = (uint8_t *)(new_cs << 4);
+    env->segs[R_CS].base = (new_cs << 4);
 }
  
 /* protected mode call */
@@ -1364,7 +1638,7 @@ void helper_lcall_protected_T0_T1(int shift, int next_eip)
     uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
     uint32_t ss, ss_e1, ss_e2, sp, type, ss_dpl, sp_mask;
     uint32_t val, limit, old_sp_mask;
-    uint8_t *ssp, *old_ssp;
+    target_ulong ssp, old_ssp;
  
     new_cs = T0;
     new_eip = T1;
@@ -1471,7 +1745,7 @@ void helper_lcall_protected_T0_T1(int shift, int next_eip)
             get_ss_esp_from_tss(&ss, &sp, dpl);
 #ifdef DEBUG_PCALL
             if (loglevel & CPU_LOG_PCALL)
-                fprintf(logfile, "new ss:esp=%04x:%08x param_count=%d ESP=%x\n", 
+                fprintf(logfile, "new ss:esp=%04x:%08x param_count=%d ESP=" TARGET_FMT_lx "\n", 
                         ss, sp, param_count, ESP);
 #endif
             if ((ss & 0xfffc) == 0)
@@ -1555,7 +1829,7 @@ void helper_lcall_protected_T0_T1(int shift, int next_eip)
 void helper_iret_real(int shift)
 {
     uint32_t sp, new_cs, new_eip, new_eflags, sp_mask;
-    uint8_t *ssp;
+    target_ulong ssp;
     int eflags_mask;
  
     sp_mask = 0xffff; /* XXXX: use SS segment size ? */
@@ -1595,7 +1869,7 @@ static inline void validate_seg(int seg_reg, int cpl)
     if (!(e2 & DESC_CS_MASK) || !(e2 & DESC_C_MASK)) {
         /* data or non conforming code segment */
         if (dpl < cpl) {
-            cpu_x86_load_seg_cache(env, seg_reg, 0, NULL, 0, 0);
+            cpu_x86_load_seg_cache(env, seg_reg, 0, 0, 0, 0);
         }
     }
 }
@@ -1603,16 +1877,31 @@ static inline void validate_seg(int seg_reg, int cpl)
 /* protected mode iret */
 static inline void helper_ret_protected(int shift, int is_iret, int addend)
 {
-    uint32_t sp, new_cs, new_eip, new_eflags, new_esp, new_ss, sp_mask;
+    uint32_t new_cs, new_eflags, new_ss;
     uint32_t new_es, new_ds, new_fs, new_gs;
     uint32_t e1, e2, ss_e1, ss_e2;
     int cpl, dpl, rpl, eflags_mask, iopl;
-    uint8_t *ssp;
+    target_ulong ssp, sp, new_eip, new_esp, sp_mask;
  
-    sp_mask = get_sp_mask(env->segs[R_SS].flags);
+#ifdef TARGET_X86_64
+    if (shift == 2)
+        sp_mask = -1;
+    else
+#endif
+        sp_mask = get_sp_mask(env->segs[R_SS].flags);
     sp = ESP;
     ssp = env->segs[R_SS].base;
     new_eflags = 0; /* avoid warning */
+#ifdef TARGET_X86_64
+    if (shift == 2) {
+        POPQ(sp, new_eip);
+        POPQ(sp, new_cs);
+        new_cs &= 0xffff;
+        if (is_iret) {
+            POPQ(sp, new_eflags);
+        }
+    } else
+#endif
     if (shift == 1) {
         /* 32 bits */
         POPL(ssp, sp, sp_mask, new_eip);
@@ -1632,7 +1921,7 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
     }
 #ifdef DEBUG_PCALL
     if (loglevel & CPU_LOG_PCALL) {
-        fprintf(logfile, "lret new %04x:%08x s=%d addend=0x%x\n",
+        fprintf(logfile, "lret new %04x:" TARGET_FMT_lx " s=%d addend=0x%x\n",
                 new_cs, new_eip, shift, addend);
         cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
     }
@@ -1660,7 +1949,7 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
         raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
  
     sp += addend;
-    if (rpl == cpl) {
+    if (rpl == cpl && !(env->hflags & HF_CS64_MASK)) {
         /* return to same priledge level */
         cpu_x86_load_seg_cache(env, R_CS, new_cs, 
                        get_seg_base(e1, e2),
@@ -1668,6 +1957,13 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
                        e2);
     } else {
         /* return to different priviledge level */
+#ifdef TARGET_X86_64
+        if (shift == 2) {
+            POPQ(sp, new_esp);
+            POPQ(sp, new_ss);
+            new_ss &= 0xffff;
+        } else
+#endif
         if (shift == 1) {
             /* 32 bits */
             POPL(ssp, sp, sp_mask, new_esp);
@@ -1680,36 +1976,49 @@ static inline void helper_ret_protected(int shift, int is_iret, int addend)
         }
 #ifdef DEBUG_PCALL
         if (loglevel & CPU_LOG_PCALL) {
-            fprintf(logfile, "new ss:esp=%04x:%08x\n",
+            fprintf(logfile, "new ss:esp=%04x:" TARGET_FMT_lx "\n",
                     new_ss, new_esp);
         }
 #endif
-        
-        if ((new_ss & 3) != rpl)
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        if (load_segment(&ss_e1, &ss_e2, new_ss) != 0)
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        if (!(ss_e2 & DESC_S_MASK) ||
-            (ss_e2 & DESC_CS_MASK) ||
-            !(ss_e2 & DESC_W_MASK))
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
-        if (dpl != rpl)
-            raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
-        if (!(ss_e2 & DESC_P_MASK))
-            raise_exception_err(EXCP0B_NOSEG, new_ss & 0xfffc);
+        if ((env->hflags & HF_LMA_MASK) && (new_ss & 0xfffc) == 0) {
+            /* NULL ss is allowed in long mode */
+            cpu_x86_load_seg_cache(env, R_SS, new_ss, 
+                                   0, 0xffffffff,
+                                   DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
+                                   DESC_S_MASK | (rpl << DESC_DPL_SHIFT) |
+                                   DESC_W_MASK | DESC_A_MASK);
+        } else {
+            if ((new_ss & 3) != rpl)
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            if (load_segment(&ss_e1, &ss_e2, new_ss) != 0)
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            if (!(ss_e2 & DESC_S_MASK) ||
+                (ss_e2 & DESC_CS_MASK) ||
+                !(ss_e2 & DESC_W_MASK))
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
+            if (dpl != rpl)
+                raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
+            if (!(ss_e2 & DESC_P_MASK))
+                raise_exception_err(EXCP0B_NOSEG, new_ss & 0xfffc);
+            cpu_x86_load_seg_cache(env, R_SS, new_ss, 
+                                   get_seg_base(ss_e1, ss_e2),
+                                   get_seg_limit(ss_e1, ss_e2),
+                                   ss_e2);
+        }
  
         cpu_x86_load_seg_cache(env, R_CS, new_cs, 
                        get_seg_base(e1, e2),
                        get_seg_limit(e1, e2),
                        e2);
-        cpu_x86_load_seg_cache(env, R_SS, new_ss, 
-                       get_seg_base(ss_e1, ss_e2),
-                       get_seg_limit(ss_e1, ss_e2),
-                       ss_e2);
         cpu_x86_set_cpl(env, rpl);
         sp = new_esp;
-        sp_mask = get_sp_mask(ss_e2);
+#ifdef TARGET_X86_64
+        if (shift == 2)
+            sp_mask = -1;
+        else
+#endif
+            sp_mask = get_sp_mask(ss_e2);
  
         /* validate data segments */
         validate_seg(R_ES, cpl);
@@ -1765,6 +2074,10 @@ void helper_iret_protected(int shift, int next_eip)
  
     /* specific case for TSS */
     if (env->eflags & NT_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK)
+            raise_exception_err(EXCP0D_GPF, 0);
+#endif
         tss_selector = lduw_kernel(env->tr.base + 0);
         if (tss_selector & 4)
             raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
@@ -1793,12 +2106,12 @@ void helper_sysenter(void)
     env->eflags &= ~(VM_MASK | IF_MASK | RF_MASK);
     cpu_x86_set_cpl(env, 0);
     cpu_x86_load_seg_cache(env, R_CS, env->sysenter_cs & 0xfffc, 
-                           NULL, 0xffffffff, 
+                           0, 0xffffffff, 
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK |
                            DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_SS, (env->sysenter_cs + 8) & 0xfffc, 
-                           NULL, 0xffffffff,
+                           0, 0xffffffff,
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK |
                            DESC_W_MASK | DESC_A_MASK);
@@ -1816,12 +2129,12 @@ void helper_sysexit(void)
     }
     cpu_x86_set_cpl(env, 3);
     cpu_x86_load_seg_cache(env, R_CS, ((env->sysenter_cs + 16) & 0xfffc) | 3, 
-                           NULL, 0xffffffff, 
+                           0, 0xffffffff, 
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
                            DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_SS, ((env->sysenter_cs + 24) & 0xfffc) | 3, 
-                           NULL, 0xffffffff,
+                           0, 0xffffffff,
                            DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                            DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
                            DESC_W_MASK | DESC_A_MASK);
@@ -1863,22 +2176,67 @@ void helper_rdtsc(void)
     uint64_t val;
  
     val = cpu_get_tsc(env);
-    EAX = val;
-    EDX = val >> 32;
+    EAX = (uint32_t)(val);
+    EDX = (uint32_t)(val >> 32);
+}
+
+#if defined(CONFIG_USER_ONLY) 
+void helper_wrmsr(void)
+{
 }
  
+void helper_rdmsr(void)
+{
+}
+#else
 void helper_wrmsr(void)
 {
-    switch(ECX) {
+    uint64_t val;
+
+    val = ((uint32_t)EAX) | ((uint64_t)((uint32_t)EDX) << 32);
+
+    switch((uint32_t)ECX) {
     case MSR_IA32_SYSENTER_CS:
-        env->sysenter_cs = EAX & 0xffff;
+        env->sysenter_cs = val & 0xffff;
         break;
     case MSR_IA32_SYSENTER_ESP:
-        env->sysenter_esp = EAX;
+        env->sysenter_esp = val;
         break;
     case MSR_IA32_SYSENTER_EIP:
-        env->sysenter_eip = EAX;
+        env->sysenter_eip = val;
+        break;
+    case MSR_IA32_APICBASE:
+        cpu_set_apic_base(env, val);
+        break;
+#ifdef TARGET_X86_64
+    case MSR_EFER:
+#define MSR_EFER_UPDATE_MASK (MSR_EFER_SCE | MSR_EFER_LME | \
+                              MSR_EFER_NXE | MSR_EFER_FFXSR)
+        env->efer = (env->efer & ~MSR_EFER_UPDATE_MASK) | 
+            (val & MSR_EFER_UPDATE_MASK);
         break;
+    case MSR_STAR:
+        env->star = val;
+        break;
+    case MSR_LSTAR:
+        env->lstar = val;
+        break;
+    case MSR_CSTAR:
+        env->cstar = val;
+        break;
+    case MSR_FMASK:
+        env->fmask = val;
+        break;
+    case MSR_FSBASE:
+        env->segs[R_FS].base = val;
+        break;
+    case MSR_GSBASE:
+        env->segs[R_GS].base = val;
+        break;
+    case MSR_KERNELGSBASE:
+        env->kernelgsbase = val;
+        break;
+#endif
     default:
         /* XXX: exception ? */
         break; 
@@ -1887,24 +2245,55 @@ void helper_wrmsr(void)
  
 void helper_rdmsr(void)
 {
-    switch(ECX) {
+    uint64_t val;
+    switch((uint32_t)ECX) {
     case MSR_IA32_SYSENTER_CS:
-        EAX = env->sysenter_cs;
-        EDX = 0;
+        val = env->sysenter_cs;
         break;
     case MSR_IA32_SYSENTER_ESP:
-        EAX = env->sysenter_esp;
-        EDX = 0;
+        val = env->sysenter_esp;
         break;
     case MSR_IA32_SYSENTER_EIP:
-        EAX = env->sysenter_eip;
-        EDX = 0;
+        val = env->sysenter_eip;
+        break;
+    case MSR_IA32_APICBASE:
+        val = cpu_get_apic_base(env);
+        break;
+#ifdef TARGET_X86_64
+    case MSR_EFER:
+        val = env->efer;
+        break;
+    case MSR_STAR:
+        val = env->star;
+        break;
+    case MSR_LSTAR:
+        val = env->lstar;
+        break;
+    case MSR_CSTAR:
+        val = env->cstar;
+        break;
+    case MSR_FMASK:
+        val = env->fmask;
+        break;
+    case MSR_FSBASE:
+        val = env->segs[R_FS].base;
+        break;
+    case MSR_GSBASE:
+        val = env->segs[R_GS].base;
         break;
+    case MSR_KERNELGSBASE:
+        val = env->kernelgsbase;
+        break;
+#endif
     default:
         /* XXX: exception ? */
+        val = 0;
         break; 
     }
+    EAX = (uint32_t)(val);
+    EDX = (uint32_t)(val >> 32);
 }
+#endif
  
 void helper_lsl(void)
 {
@@ -2055,14 +2444,14 @@ void helper_fldt_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = helper_fldt((uint8_t *)A0);
+    env->fpregs[new_fpstt] = helper_fldt(A0);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
  
 void helper_fstt_ST0_A0(void)
 {
-    helper_fstt(ST0, (uint8_t *)A0);
+    helper_fstt(ST0, A0);
 }
  
 void fpu_set_exception(int mask)
@@ -2102,11 +2491,11 @@ void helper_fbld_ST0_A0(void)
  
     val = 0;
     for(i = 8; i >= 0; i--) {
-        v = ldub((uint8_t *)A0 + i);
+        v = ldub(A0 + i);
         val = (val * 100) + ((v >> 4) * 10) + (v & 0xf);
     }
     tmp = val;
-    if (ldub((uint8_t *)A0 + 9) & 0x80)
+    if (ldub(A0 + 9) & 0x80)
         tmp = -tmp;
     fpush();
     ST0 = tmp;
@@ -2116,12 +2505,12 @@ void helper_fbst_ST0_A0(void)
 {
     CPU86_LDouble tmp;
     int v;
-    uint8_t *mem_ref, *mem_end;
+    target_ulong mem_ref, mem_end;
     int64_t val;
  
     tmp = rint(ST0);
     val = (int64_t)tmp;
-    mem_ref = (uint8_t *)A0;
+    mem_ref = A0;
     mem_end = mem_ref + 9;
     if (val < 0) {
         stb(mem_end, 0x80);
@@ -2402,7 +2791,7 @@ void helper_fxam_ST0(void)
     }
 }
  
-void helper_fstenv(uint8_t *ptr, int data32)
+void helper_fstenv(target_ulong ptr, int data32)
 {
     int fpus, fptag, exp, i;
     uint64_t mant;
@@ -2452,7 +2841,7 @@ void helper_fstenv(uint8_t *ptr, int data32)
     }
 }
  
-void helper_fldenv(uint8_t *ptr, int data32)
+void helper_fldenv(target_ulong ptr, int data32)
 {
     int i, fpus, fptag;
  
@@ -2474,7 +2863,7 @@ void helper_fldenv(uint8_t *ptr, int data32)
     }
 }
  
-void helper_fsave(uint8_t *ptr, int data32)
+void helper_fsave(target_ulong ptr, int data32)
 {
     CPU86_LDouble tmp;
     int i;
@@ -2502,7 +2891,7 @@ void helper_fsave(uint8_t *ptr, int data32)
     env->fptags[7] = 1;
 }
  
-void helper_frstor(uint8_t *ptr, int data32)
+void helper_frstor(target_ulong ptr, int data32)
 {
     CPU86_LDouble tmp;
     int i;
@@ -2517,7 +2906,78 @@ void helper_frstor(uint8_t *ptr, int data32)
     }
 }
  
-/* XXX: merge with helper_fstt ? */
+void helper_fxsave(target_ulong ptr, int data64)
+{
+    int fpus, fptag, i, nb_xmm_regs;
+    CPU86_LDouble tmp;
+    target_ulong addr;
+
+    fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
+    fptag = 0;
+    for(i = 0; i < 8; i++) {
+        fptag |= ((!env->fptags[(env->fpstt + i) & 7]) << i);
+    }
+    stw(ptr, env->fpuc);
+    stw(ptr + 2, fpus);
+    stw(ptr + 4, fptag);
+
+    addr = ptr + 0x20;
+    for(i = 0;i < 8; i++) {
+        tmp = ST(i);
+        helper_fstt(tmp, addr);
+        addr += 16;
+    }
+    
+    if (env->cr[4] & CR4_OSFXSR_MASK) {
+        /* XXX: finish it, endianness */
+        stl(ptr + 0x18, 0); /* mxcsr */
+        stl(ptr + 0x1c, 0); /* mxcsr_mask */
+        nb_xmm_regs = 8 << data64;
+        addr = ptr + 0xa0;
+        for(i = 0; i < nb_xmm_regs; i++) {
+            stq(addr, env->xmm_regs[i].u.q[0]);
+            stq(addr, env->xmm_regs[i].u.q[1]);
+            addr += 16;
+        }
+    }
+}
+
+void helper_fxrstor(target_ulong ptr, int data64)
+{
+    int i, fpus, fptag, nb_xmm_regs;
+    CPU86_LDouble tmp;
+    target_ulong addr;
+
+    env->fpuc = lduw(ptr);
+    fpus = lduw(ptr + 2);
+    fptag = ldub(ptr + 4);
+    env->fpstt = (fpus >> 11) & 7;
+    env->fpus = fpus & ~0x3800;
+    fptag ^= 0xff;
+    for(i = 0;i < 8; i++) {
+        env->fptags[(env->fpstt + i) & 7] = ((fptag >> i) & 1);
+    }
+
+    addr = ptr + 0x20;
+    for(i = 0;i < 8; i++) {
+        tmp = helper_fldt(addr);
+        ST(i) = tmp;
+        addr += 16;
+    }
+
+    if (env->cr[4] & CR4_OSFXSR_MASK) {
+        /* XXX: finish it, endianness */
+        //ldl(ptr + 0x18);
+        //ldl(ptr + 0x1c);
+        nb_xmm_regs = 8 << data64;
+        addr = ptr + 0xa0;
+        for(i = 0; i < nb_xmm_regs; i++) {
+            env->xmm_regs[i].u.q[0] = ldq(addr);
+            env->xmm_regs[i].u.q[1] = ldq(addr);
+            addr += 16;
+        }
+    }
+}
  
 #ifndef USE_X86LDOUBLE
  
@@ -2575,6 +3035,179 @@ CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper)
 }
 #endif
  
+#ifdef TARGET_X86_64
+
+//#define DEBUG_MULDIV
+
+static void add128(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b)
+{
+    *plow += a;
+    /* carry test */
+    if (*plow < a)
+        (*phigh)++;
+    *phigh += b;
+}
+
+static void neg128(uint64_t *plow, uint64_t *phigh)
+{
+    *plow = ~ *plow;
+    *phigh = ~ *phigh;
+    add128(plow, phigh, 1, 0);
+}
+
+static void mul64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b)
+{
+    uint32_t a0, a1, b0, b1;
+    uint64_t v;
+
+    a0 = a;
+    a1 = a >> 32;
+
+    b0 = b;
+    b1 = b >> 32;
+    
+    v = (uint64_t)a0 * (uint64_t)b0;
+    *plow = v;
+    *phigh = 0;
+
+    v = (uint64_t)a0 * (uint64_t)b1;
+    add128(plow, phigh, v << 32, v >> 32);
+    
+    v = (uint64_t)a1 * (uint64_t)b0;
+    add128(plow, phigh, v << 32, v >> 32);
+    
+    v = (uint64_t)a1 * (uint64_t)b1;
+    *phigh += v;
+#ifdef DEBUG_MULDIV
+    printf("mul: 0x%016llx * 0x%016llx = 0x%016llx%016llx\n",
+           a, b, *phigh, *plow);
+#endif
+}
+
+static void imul64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b)
+{
+    int sa, sb;
+    sa = (a < 0);
+    if (sa)
+        a = -a;
+    sb = (b < 0);
+    if (sb)
+        b = -b;
+    mul64(plow, phigh, a, b);
+    if (sa ^ sb) {
+        neg128(plow, phigh);
+    }
+}
+
+static void div64(uint64_t *plow, uint64_t *phigh, uint64_t b)
+{
+    uint64_t q, r, a1, a0;
+    int i, qb;
+
+    a0 = *plow;
+    a1 = *phigh;
+    if (a1 == 0) {
+        q = a0 / b;
+        r = a0 % b;
+        *plow = q;
+        *phigh = r;
+    } else {
+        /* XXX: use a better algorithm */
+        for(i = 0; i < 64; i++) {
+            if (a1 >= b) {
+                a1 -= b;
+                qb = 1;
+            } else {
+                qb = 0;
+            }
+            a1 = (a1 << 1) | (a0 >> 63);
+            a0 = (a0 << 1) | qb;
+        }
+#if defined(DEBUG_MULDIV) || 1
+        printf("div: 0x%016llx%016llx / 0x%016llx: q=0x%016llx r=0x%016llx\n",
+               *phigh, *plow, b, a0, a1);
+#endif
+        *plow = a0;
+        *phigh = a1;
+    }
+}
+
+static void idiv64(uint64_t *plow, uint64_t *phigh, uint64_t b)
+{
+    int sa, sb;
+    sa = ((int64_t)*phigh < 0);
+    if (sa)
+        neg128(plow, phigh);
+    sb = (b < 0);
+    if (sb)
+        b = -b;
+    div64(plow, phigh, b);
+    if (sa ^ sb)
+        *plow = - *plow;
+    if (sb)
+        *phigh = - *phigh;
+}
+
+void helper_mulq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+
+    mul64(&r0, &r1, EAX, T0);
+    EAX = r0;
+    EDX = r1;
+    CC_DST = r0;
+    CC_SRC = r1;
+}
+
+void helper_imulq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+
+    imul64(&r0, &r1, EAX, T0);
+    EAX = r0;
+    EDX = r1;
+    CC_DST = r0;
+    CC_SRC = (r1 != (r0 >> 63));
+}
+
+void helper_imulq_T0_T1(void)
+{
+    uint64_t r0, r1;
+
+    imul64(&r0, &r1, T0, T1);
+    T0 = r0;
+    CC_DST = r0;
+    CC_SRC = ((int64_t)r1 != ((int64_t)r0 >> 63));
+}
+
+void helper_divq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+    if (T0 == 0) {
+        raise_exception(EXCP00_DIVZ);
+    }
+    r0 = EAX;
+    r1 = EDX;
+    div64(&r0, &r1, T0);
+    EAX = r0;
+    EDX = r1;
+}
+
+void helper_idivq_EAX_T0(void)
+{
+    uint64_t r0, r1;
+    if (T0 == 0) {
+        raise_exception(EXCP00_DIVZ);
+    }
+    r0 = EAX;
+    r1 = EDX;
+    idiv64(&r0, &r1, T0);
+    EAX = r0;
+    EDX = r1;
+}
+
+#endif
+
 #if !defined(CONFIG_USER_ONLY) 
  
 #define MMUSUFFIX _mmu
@@ -2598,7 +3231,7 @@ CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper)
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
 /* XXX: fix it to restore all registers */
-void tlb_fill(unsigned long addr, int is_write, int is_user, void *retaddr)
+void tlb_fill(target_ulong addr, int is_write, int is_user, void *retaddr)
 {
     TranslationBlock *tb;
     int ret;
@@ -77,6 +77,41 @@ CPUX86State *cpu_x86_init(void)
         asm volatile ("movl %0, %%fs" : : "r" ((1 << 3) | 7));
     }
 #endif
+    {
+        int family, model, stepping;
+#ifdef TARGET_X86_64
+        env->cpuid_vendor1 = 0x68747541; /* "Auth" */
+        env->cpuid_vendor2 = 0x69746e65; /* "enti" */
+        env->cpuid_vendor3 = 0x444d4163; /* "cAMD" */
+        family = 6;
+        model = 2;
+        stepping = 3;
+#else
+        env->cpuid_vendor1 = 0x756e6547; /* "Genu" */
+        env->cpuid_vendor2 = 0x49656e69; /* "ineI" */
+        env->cpuid_vendor3 = 0x6c65746e; /* "ntel" */
+#if 0
+        /* pentium 75-200 */
+        family = 5;
+        model = 2;
+        stepping = 11;
+#else
+        /* pentium pro */
+        family = 6;
+        model = 1;
+        stepping = 3;
+#endif
+#endif
+        env->cpuid_version = (family << 8) | (model << 4) | stepping;
+        env->cpuid_features = (CPUID_FP87 | CPUID_DE | CPUID_PSE |
+                               CPUID_TSC | CPUID_MSR | CPUID_MCE |
+                               CPUID_CX8 | CPUID_PGE | CPUID_CMOV);
+#ifdef TARGET_X86_64
+        /* currently not enabled for std i386 because not fully tested */
+        env->cpuid_features |= CPUID_APIC | CPUID_FXSR | CPUID_PAE |
+            CPUID_SSE | CPUID_SSE2;
+#endif
+    }
     cpu_single_env = env;
     cpu_reset(env);
     return env;
@@ -107,12 +142,12 @@ void cpu_reset(CPUX86State *env)
     env->tr.limit = 0xffff;
     env->tr.flags = DESC_P_MASK;
  
-    cpu_x86_load_seg_cache(env, R_CS, 0xf000, (uint8_t *)0xffff0000, 0xffff, 0); 
-    cpu_x86_load_seg_cache(env, R_DS, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_ES, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_SS, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_FS, 0, NULL, 0xffff, 0);
-    cpu_x86_load_seg_cache(env, R_GS, 0, NULL, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_CS, 0xf000, 0xffff0000, 0xffff, 0); 
+    cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_SS, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0xffff, 0);
+    cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0xffff, 0);
  
     env->eip = 0xfff0;
     env->regs[R_EDX] = 0x600; /* indicate P6 processor */
@@ -136,36 +171,56 @@ void cpu_x86_close(CPUX86State *env)
 static const char *cc_op_str[] = {
     "DYNAMIC",
     "EFLAGS",
+
     "MULB",
     "MULW",
     "MULL",
+    "MULQ",
+
     "ADDB",
     "ADDW",
     "ADDL",
+    "ADDQ",
+
     "ADCB",
     "ADCW",
     "ADCL",
+    "ADCQ",
+
     "SUBB",
     "SUBW",
     "SUBL",
+    "SUBQ",
+
     "SBBB",
     "SBBW",
     "SBBL",
+    "SBBQ",
+
     "LOGICB",
     "LOGICW",
     "LOGICL",
+    "LOGICQ",
+
     "INCB",
     "INCW",
     "INCL",
+    "INCQ",
+
     "DECB",
     "DECW",
     "DECL",
+    "DECQ",
+
     "SHLB",
     "SHLW",
     "SHLL",
+    "SHLQ",
+
     "SARB",
     "SARW",
     "SARL",
+    "SARQ",
 };
  
 void cpu_dump_state(CPUState *env, FILE *f, 
@@ -177,55 +232,147 @@ void cpu_dump_state(CPUState *env, FILE *f,
     static const char *seg_name[6] = { "ES", "CS", "SS", "DS", "FS", "GS" };
  
     eflags = env->eflags;
-    cpu_fprintf(f, "EAX=%08x EBX=%08x ECX=%08x EDX=%08x\n"
-            "ESI=%08x EDI=%08x EBP=%08x ESP=%08x\n"
-            "EIP=%08x EFL=%08x [%c%c%c%c%c%c%c]    CPL=%d II=%d A20=%d\n",
-            env->regs[R_EAX], env->regs[R_EBX], env->regs[R_ECX], env->regs[R_EDX], 
-            env->regs[R_ESI], env->regs[R_EDI], env->regs[R_EBP], env->regs[R_ESP], 
-            env->eip, eflags,
-            eflags & DF_MASK ? 'D' : '-',
-            eflags & CC_O ? 'O' : '-',
-            eflags & CC_S ? 'S' : '-',
-            eflags & CC_Z ? 'Z' : '-',
-            eflags & CC_A ? 'A' : '-',
-            eflags & CC_P ? 'P' : '-',
-            eflags & CC_C ? 'C' : '-',
-            env->hflags & HF_CPL_MASK, 
-            (env->hflags >> HF_INHIBIT_IRQ_SHIFT) & 1,
-            (env->a20_mask >> 20) & 1);
-    for(i = 0; i < 6; i++) {
-        SegmentCache *sc = &env->segs[i];
-        cpu_fprintf(f, "%s =%04x %08x %08x %08x\n",
-                seg_name[i],
-                sc->selector,
-                (int)sc->base,
-                sc->limit,
-                sc->flags);
+#ifdef TARGET_X86_64
+    if (env->hflags & HF_CS64_MASK) {
+        cpu_fprintf(f, 
+                    "RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n"
+                    "RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n"
+                    "R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n"
+                    "R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n"
+                    "RIP=%016llx RFL=%08x [%c%c%c%c%c%c%c]    CPL=%d II=%d A20=%d\n",
+                    env->regs[R_EAX], 
+                    env->regs[R_EBX], 
+                    env->regs[R_ECX], 
+                    env->regs[R_EDX], 
+                    env->regs[R_ESI], 
+                    env->regs[R_EDI], 
+                    env->regs[R_EBP], 
+                    env->regs[R_ESP], 
+                    env->regs[8], 
+                    env->regs[9], 
+                    env->regs[10], 
+                    env->regs[11], 
+                    env->regs[12], 
+                    env->regs[13], 
+                    env->regs[14], 
+                    env->regs[15], 
+                    env->eip, eflags,
+                    eflags & DF_MASK ? 'D' : '-',
+                    eflags & CC_O ? 'O' : '-',
+                    eflags & CC_S ? 'S' : '-',
+                    eflags & CC_Z ? 'Z' : '-',
+                    eflags & CC_A ? 'A' : '-',
+                    eflags & CC_P ? 'P' : '-',
+                    eflags & CC_C ? 'C' : '-',
+                    env->hflags & HF_CPL_MASK, 
+                    (env->hflags >> HF_INHIBIT_IRQ_SHIFT) & 1,
+                    (env->a20_mask >> 20) & 1);
+    } else 
+#endif
+    {
+        cpu_fprintf(f, "EAX=%08x EBX=%08x ECX=%08x EDX=%08x\n"
+                    "ESI=%08x EDI=%08x EBP=%08x ESP=%08x\n"
+                    "EIP=%08x EFL=%08x [%c%c%c%c%c%c%c]    CPL=%d II=%d A20=%d\n",
+                    (uint32_t)env->regs[R_EAX], 
+                    (uint32_t)env->regs[R_EBX], 
+                    (uint32_t)env->regs[R_ECX], 
+                    (uint32_t)env->regs[R_EDX], 
+                    (uint32_t)env->regs[R_ESI], 
+                    (uint32_t)env->regs[R_EDI], 
+                    (uint32_t)env->regs[R_EBP], 
+                    (uint32_t)env->regs[R_ESP], 
+                    (uint32_t)env->eip, eflags,
+                    eflags & DF_MASK ? 'D' : '-',
+                    eflags & CC_O ? 'O' : '-',
+                    eflags & CC_S ? 'S' : '-',
+                    eflags & CC_Z ? 'Z' : '-',
+                    eflags & CC_A ? 'A' : '-',
+                    eflags & CC_P ? 'P' : '-',
+                    eflags & CC_C ? 'C' : '-',
+                    env->hflags & HF_CPL_MASK, 
+                    (env->hflags >> HF_INHIBIT_IRQ_SHIFT) & 1,
+                    (env->a20_mask >> 20) & 1);
+    }
+
+#ifdef TARGET_X86_64
+    if (env->hflags & HF_LMA_MASK) {
+        for(i = 0; i < 6; i++) {
+            SegmentCache *sc = &env->segs[i];
+            cpu_fprintf(f, "%s =%04x %016llx %08x %08x\n",
+                        seg_name[i],
+                        sc->selector,
+                        sc->base,
+                        sc->limit,
+                        sc->flags);
+        }
+        cpu_fprintf(f, "LDT=%04x %016llx %08x %08x\n",
+                    env->ldt.selector,
+                    env->ldt.base,
+                    env->ldt.limit,
+                    env->ldt.flags);
+        cpu_fprintf(f, "TR =%04x %016llx %08x %08x\n",
+                    env->tr.selector,
+                    env->tr.base,
+                    env->tr.limit,
+                    env->tr.flags);
+        cpu_fprintf(f, "GDT=     %016llx %08x\n",
+                    env->gdt.base, env->gdt.limit);
+        cpu_fprintf(f, "IDT=     %016llx %08x\n",
+                    env->idt.base, env->idt.limit);
+        cpu_fprintf(f, "CR0=%08x CR2=%016llx CR3=%016llx CR4=%08x\n",
+                    (uint32_t)env->cr[0], 
+                    env->cr[2], 
+                    env->cr[3], 
+                    (uint32_t)env->cr[4]);
+    } else
+#endif
+    {
+        for(i = 0; i < 6; i++) {
+            SegmentCache *sc = &env->segs[i];
+            cpu_fprintf(f, "%s =%04x %08x %08x %08x\n",
+                        seg_name[i],
+                        sc->selector,
+                        (uint32_t)sc->base,
+                        sc->limit,
+                        sc->flags);
+        }
+        cpu_fprintf(f, "LDT=%04x %08x %08x %08x\n",
+                    env->ldt.selector,
+                    (uint32_t)env->ldt.base,
+                    env->ldt.limit,
+                    env->ldt.flags);
+        cpu_fprintf(f, "TR =%04x %08x %08x %08x\n",
+                    env->tr.selector,
+                    (uint32_t)env->tr.base,
+                    env->tr.limit,
+                    env->tr.flags);
+        cpu_fprintf(f, "GDT=     %08x %08x\n",
+                    (uint32_t)env->gdt.base, env->gdt.limit);
+        cpu_fprintf(f, "IDT=     %08x %08x\n",
+                    (uint32_t)env->idt.base, env->idt.limit);
+        cpu_fprintf(f, "CR0=%08x CR2=%08x CR3=%08x CR4=%08x\n",
+                    (uint32_t)env->cr[0], 
+                    (uint32_t)env->cr[2], 
+                    (uint32_t)env->cr[3], 
+                    (uint32_t)env->cr[4]);
     }
-    cpu_fprintf(f, "LDT=%04x %08x %08x %08x\n",
-            env->ldt.selector,
-            (int)env->ldt.base,
-            env->ldt.limit,
-            env->ldt.flags);
-    cpu_fprintf(f, "TR =%04x %08x %08x %08x\n",
-            env->tr.selector,
-            (int)env->tr.base,
-            env->tr.limit,
-            env->tr.flags);
-    cpu_fprintf(f, "GDT=     %08x %08x\n",
-            (int)env->gdt.base, env->gdt.limit);
-    cpu_fprintf(f, "IDT=     %08x %08x\n",
-            (int)env->idt.base, env->idt.limit);
-    cpu_fprintf(f, "CR0=%08x CR2=%08x CR3=%08x CR4=%08x\n",
-            env->cr[0], env->cr[2], env->cr[3], env->cr[4]);
-    
     if (flags & X86_DUMP_CCOP) {
         if ((unsigned)env->cc_op < CC_OP_NB)
             snprintf(cc_op_name, sizeof(cc_op_name), "%s", cc_op_str[env->cc_op]);
         else
             snprintf(cc_op_name, sizeof(cc_op_name), "[%d]", env->cc_op);
-        cpu_fprintf(f, "CCS=%08x CCD=%08x CCO=%-8s\n",
-                env->cc_src, env->cc_dst, cc_op_name);
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_CS64_MASK) {
+            cpu_fprintf(f, "CCS=%016llx CCD=%016llx CCO=%-8s\n",
+                        env->cc_src, env->cc_dst, 
+                        cc_op_name);
+        } else 
+#endif
+        {
+            cpu_fprintf(f, "CCS=%08x CCD=%08x CCO=%-8s\n",
+                        (uint32_t)env->cc_src, (uint32_t)env->cc_dst, 
+                        cc_op_name);
+        }
     }
     if (flags & X86_DUMP_FPU) {
         cpu_fprintf(f, "ST0=%f ST1=%f ST2=%f ST3=%f\n", 
@@ -274,6 +421,24 @@ void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0)
         (env->cr[0] & (CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK))) {
         tlb_flush(env, 1);
     }
+
+#ifdef TARGET_X86_64
+    if (!(env->cr[0] & CR0_PG_MASK) && (new_cr0 & CR0_PG_MASK) &&
+        (env->efer & MSR_EFER_LME)) {
+        /* enter in long mode */
+        /* XXX: generate an exception */
+        if (!(env->cr[4] & CR4_PAE_MASK))
+            return;
+        env->efer |= MSR_EFER_LMA;
+        env->hflags |= HF_LMA_MASK;
+    } else if ((env->cr[0] & CR0_PG_MASK) && !(new_cr0 & CR0_PG_MASK) &&
+               (env->efer & MSR_EFER_LMA)) {
+        /* exit long mode */
+        env->efer &= ~MSR_EFER_LMA;
+        env->hflags &= ~(HF_LMA_MASK | HF_CS64_MASK);
+        env->eip &= 0xffffffff;
+    }
+#endif
     env->cr[0] = new_cr0 | CR0_ET_MASK;
  
     /* update PE flag in hidden flags */
@@ -286,12 +451,12 @@ void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0)
         ((new_cr0 << (HF_MP_SHIFT - 1)) & (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK));
 }
  
-void cpu_x86_update_cr3(CPUX86State *env, uint32_t new_cr3)
+void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3)
 {
     env->cr[3] = new_cr3;
     if (env->cr[0] & CR0_PG_MASK) {
 #if defined(DEBUG_MMU)
-        printf("CR3 update: CR3=%08x\n", new_cr3);
+        printf("CR3 update: CR3=" TARGET_FMT_lx "\n", new_cr3);
 #endif
         tlb_flush(env, 0);
     }
@@ -300,7 +465,7 @@ void cpu_x86_update_cr3(CPUX86State *env, uint32_t new_cr3)
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 {
 #if defined(DEBUG_MMU)
-    printf("CR4 update: CR4=%08x\n", env->cr[4]);
+    printf("CR4 update: CR4=%08x\n", (uint32_t)env->cr[4]);
 #endif
     if ((new_cr4 & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK)) !=
         (env->cr[4] & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK))) {
@@ -315,22 +480,51 @@ void cpu_x86_flush_tlb(CPUX86State *env, uint32_t addr)
     tlb_flush_page(env, addr);
 }
  
+static inline uint8_t *get_phys_mem_ptr(target_phys_addr_t addr)
+{
+    /* XXX: incorrect */
+    return phys_ram_base + addr;
+}
+
+/* WARNING: addr must be aligned */
+uint32_t ldl_phys_aligned(target_phys_addr_t addr)
+{
+    uint8_t *ptr;
+    uint32_t val;
+    ptr = get_phys_mem_ptr(addr);
+    if (!ptr)
+        val = 0;
+    else
+        val = ldl_raw(ptr);
+    return val;
+}
+
+void stl_phys_aligned(target_phys_addr_t addr, uint32_t val)
+{
+    uint8_t *ptr;
+    ptr = get_phys_mem_ptr(addr);
+    if (!ptr)
+        return;
+    stl_raw(ptr, val);
+}
+
 /* return value:
    -1 = cannot handle fault 
    0  = nothing more to do 
    1  = generate PF fault
    2  = soft MMU activation required for this block
 */
-int cpu_x86_handle_mmu_fault(CPUX86State *env, uint32_t addr, 
+int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr, 
                              int is_write, int is_user, int is_softmmu)
 {
-    uint8_t *pde_ptr, *pte_ptr;
-    uint32_t pde, pte, virt_addr, ptep;
+    uint32_t pdpe_addr, pde_addr, pte_addr;
+    uint32_t pde, pte, ptep, pdpe;
     int error_code, is_dirty, prot, page_size, ret;
-    unsigned long paddr, vaddr, page_offset;
+    unsigned long paddr, page_offset;
+    target_ulong vaddr, virt_addr;
  
 #if defined(DEBUG_MMU)
-    printf("MMU fault: addr=0x%08x w=%d u=%d eip=%08x\n", 
+    printf("MMU fault: addr=" TARGET_FMT_lx " w=%d u=%d eip=" TARGET_FMT_lx "\n", 
            addr, is_write, is_user, env->eip);
 #endif
     is_write &= 1;
@@ -349,90 +543,166 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, uint32_t addr,
         goto do_mapping;
     }
  
-    /* page directory entry */
-    pde_ptr = phys_ram_base + 
-        (((env->cr[3] & ~0xfff) + ((addr >> 20) & ~3)) & env->a20_mask);
-    pde = ldl_raw(pde_ptr);
-    if (!(pde & PG_PRESENT_MASK)) {
-        error_code = 0;
-        goto do_fault;
-    }
-    /* if PSE bit is set, then we use a 4MB page */
-    if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-        if (is_user) {
-            if (!(pde & PG_USER_MASK))
-                goto do_fault_protect;
-            if (is_write && !(pde & PG_RW_MASK))
-                goto do_fault_protect;
-        } else {
-            if ((env->cr[0] & CR0_WP_MASK) && 
-                is_write && !(pde & PG_RW_MASK)) 
-                goto do_fault_protect;
+
+    if (env->cr[4] & CR4_PAE_MASK) {
+        /* XXX: we only use 32 bit physical addresses */
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            uint32_t pml4e_addr, pml4e;
+            int32_t sext;
+
+            /* XXX: handle user + rw rights */
+            /* XXX: handle NX flag */
+            /* test virtual address sign extension */
+            sext = (int64_t)addr >> 47;
+            if (sext != 0 && sext != -1) {
+                error_code = 0;
+                goto do_fault;
+            }
+            
+            pml4e_addr = ((env->cr[3] & ~0xfff) + (((addr >> 39) & 0x1ff) << 3)) & 
+                env->a20_mask;
+            pml4e = ldl_phys_aligned(pml4e_addr);
+            if (!(pml4e & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
+            if (!(pml4e & PG_ACCESSED_MASK)) {
+                pml4e |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pml4e_addr, pml4e);
+            }
+            
+            pdpe_addr = ((pml4e & ~0xfff) + (((addr >> 30) & 0x1ff) << 3)) & 
+                env->a20_mask;
+            pdpe = ldl_phys_aligned(pdpe_addr);
+            if (!(pdpe & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
+            if (!(pdpe & PG_ACCESSED_MASK)) {
+                pdpe |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pdpe_addr, pdpe);
+            }
+        } else 
+#endif
+        {
+            pdpe_addr = ((env->cr[3] & ~0x1f) + ((addr >> 30) << 3)) & 
+                env->a20_mask;
+            pdpe = ldl_phys_aligned(pdpe_addr);
+            if (!(pdpe & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
         }
-        is_dirty = is_write && !(pde & PG_DIRTY_MASK);
-        if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
-            pde |= PG_ACCESSED_MASK;
-            if (is_dirty)
-                pde |= PG_DIRTY_MASK;
-            stl_raw(pde_ptr, pde);
+
+        pde_addr = ((pdpe & ~0xfff) + (((addr >> 21) & 0x1ff) << 3)) &
+            env->a20_mask;
+        pde = ldl_phys_aligned(pde_addr);
+        if (!(pde & PG_PRESENT_MASK)) {
+            error_code = 0;
+            goto do_fault;
         }
-        
-        pte = pde & ~0x003ff000; /* align to 4MB */
-        ptep = pte;
-        page_size = 4096 * 1024;
-        virt_addr = addr & ~0x003fffff;
-    } else {
-        if (!(pde & PG_ACCESSED_MASK)) {
-            pde |= PG_ACCESSED_MASK;
-            stl_raw(pde_ptr, pde);
+        if (pde & PG_PSE_MASK) {
+            /* 2 MB page */
+            page_size = 2048 * 1024;
+            goto handle_big_page;
+        } else {
+            /* 4 KB page */
+            if (!(pde & PG_ACCESSED_MASK)) {
+                pde |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pde_addr, pde);
+            }
+            pte_addr = ((pde & ~0xfff) + (((addr >> 12) & 0x1ff) << 3)) &
+                env->a20_mask;
+            goto handle_4k_page;
         }
-
+    } else {
         /* page directory entry */
-        pte_ptr = phys_ram_base + 
-            (((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask);
-        pte = ldl_raw(pte_ptr);
-        if (!(pte & PG_PRESENT_MASK)) {
+        pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & ~3)) & 
+            env->a20_mask;
+        pde = ldl_phys_aligned(pde_addr);
+        if (!(pde & PG_PRESENT_MASK)) {
             error_code = 0;
             goto do_fault;
         }
-        /* combine pde and pte user and rw protections */
-        ptep = pte & pde;
-        if (is_user) {
-            if (!(ptep & PG_USER_MASK))
-                goto do_fault_protect;
-            if (is_write && !(ptep & PG_RW_MASK))
-                goto do_fault_protect;
+        /* if PSE bit is set, then we use a 4MB page */
+        if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+            page_size = 4096 * 1024;
+        handle_big_page:
+            if (is_user) {
+                if (!(pde & PG_USER_MASK))
+                    goto do_fault_protect;
+                if (is_write && !(pde & PG_RW_MASK))
+                    goto do_fault_protect;
+            } else {
+                if ((env->cr[0] & CR0_WP_MASK) && 
+                    is_write && !(pde & PG_RW_MASK)) 
+                    goto do_fault_protect;
+            }
+            is_dirty = is_write && !(pde & PG_DIRTY_MASK);
+            if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
+                pde |= PG_ACCESSED_MASK;
+                if (is_dirty)
+                    pde |= PG_DIRTY_MASK;
+                stl_phys_aligned(pde_addr, pde);
+            }
+        
+            pte = pde & ~( (page_size - 1) & ~0xfff); /* align to page_size */
+            ptep = pte;
+            virt_addr = addr & ~(page_size - 1);
         } else {
-            if ((env->cr[0] & CR0_WP_MASK) &&
-                is_write && !(ptep & PG_RW_MASK)) 
-                goto do_fault_protect;
-        }
-        is_dirty = is_write && !(pte & PG_DIRTY_MASK);
-        if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
-            pte |= PG_ACCESSED_MASK;
-            if (is_dirty)
-                pte |= PG_DIRTY_MASK;
-            stl_raw(pte_ptr, pte);
+            if (!(pde & PG_ACCESSED_MASK)) {
+                pde |= PG_ACCESSED_MASK;
+                stl_phys_aligned(pde_addr, pde);
+            }
+
+            /* page directory entry */
+            pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & 
+                env->a20_mask;
+        handle_4k_page:
+            pte = ldl_phys_aligned(pte_addr);
+            if (!(pte & PG_PRESENT_MASK)) {
+                error_code = 0;
+                goto do_fault;
+            }
+            /* combine pde and pte user and rw protections */
+            ptep = pte & pde;
+            if (is_user) {
+                if (!(ptep & PG_USER_MASK))
+                    goto do_fault_protect;
+                if (is_write && !(ptep & PG_RW_MASK))
+                    goto do_fault_protect;
+            } else {
+                if ((env->cr[0] & CR0_WP_MASK) &&
+                    is_write && !(ptep & PG_RW_MASK)) 
+                    goto do_fault_protect;
+            }
+            is_dirty = is_write && !(pte & PG_DIRTY_MASK);
+            if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
+                pte |= PG_ACCESSED_MASK;
+                if (is_dirty)
+                    pte |= PG_DIRTY_MASK;
+                stl_phys_aligned(pte_addr, pte);
+            }
+            page_size = 4096;
+            virt_addr = addr & ~0xfff;
         }
-        page_size = 4096;
-        virt_addr = addr & ~0xfff;
-    }
  
-    /* the page can be put in the TLB */
-    prot = PAGE_READ;
-    if (pte & PG_DIRTY_MASK) {
-        /* only set write access if already dirty... otherwise wait
-           for dirty access */
-        if (is_user) {
-            if (ptep & PG_RW_MASK)
-                prot |= PAGE_WRITE;
-        } else {
-            if (!(env->cr[0] & CR0_WP_MASK) ||
-                (ptep & PG_RW_MASK))
-                prot |= PAGE_WRITE;
+        /* the page can be put in the TLB */
+        prot = PAGE_READ;
+        if (pte & PG_DIRTY_MASK) {
+            /* only set write access if already dirty... otherwise wait
+               for dirty access */
+            if (is_user) {
+                if (ptep & PG_RW_MASK)
+                    prot |= PAGE_WRITE;
+            } else {
+                if (!(env->cr[0] & CR0_WP_MASK) ||
+                    (ptep & PG_RW_MASK))
+                    prot |= PAGE_WRITE;
+            }
         }
     }
-
  do_mapping:
     pte = pte & env->a20_mask;
  
@@ -22,7 +22,7 @@
 #include "exec.h"
  
 /* n must be a constant to be efficient */
-static inline int lshift(int x, int n)
+static inline target_long lshift(target_long x, int n)
 {
     if (n >= 0)
         return x << n;
@@ -80,6 +80,58 @@ static inline int lshift(int x, int n)
 #undef REG
 #undef REGNAME
  
+#ifdef TARGET_X86_64
+
+#define REG (env->regs[8])
+#define REGNAME _R8
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[9])
+#define REGNAME _R9
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[10])
+#define REGNAME _R10
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[11])
+#define REGNAME _R11
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[12])
+#define REGNAME _R12
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[13])
+#define REGNAME _R13
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[14])
+#define REGNAME _R14
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#define REG (env->regs[15])
+#define REGNAME _R15
+#include "opreg_template.h"
+#undef REG
+#undef REGNAME
+
+#endif
+
 /* operations with flags */
  
 /* update flags with T0 and T1 (add/sub case) */
@@ -170,6 +222,13 @@ void OPPROTO op_bswapl_T0(void)
     T0 = bswap32(T0);
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_bswapq_T0(void)
+{
+    T0 = bswap64(T0);
+}
+#endif
+
 /* multiply/divide */
  
 /* XXX: add eflags optimizations */
@@ -179,7 +238,7 @@ void OPPROTO op_mulb_AL_T0(void)
 {
     unsigned int res;
     res = (uint8_t)EAX * (uint8_t)T0;
-    EAX = (EAX & 0xffff0000) | res;
+    EAX = (EAX & ~0xffff) | res;
     CC_DST = res;
     CC_SRC = (res & 0xff00);
 }
@@ -188,7 +247,7 @@ void OPPROTO op_imulb_AL_T0(void)
 {
     int res;
     res = (int8_t)EAX * (int8_t)T0;
-    EAX = (EAX & 0xffff0000) | (res & 0xffff);
+    EAX = (EAX & ~0xffff) | (res & 0xffff);
     CC_DST = res;
     CC_SRC = (res != (int8_t)res);
 }
@@ -197,8 +256,8 @@ void OPPROTO op_mulw_AX_T0(void)
 {
     unsigned int res;
     res = (uint16_t)EAX * (uint16_t)T0;
-    EAX = (EAX & 0xffff0000) | (res & 0xffff);
-    EDX = (EDX & 0xffff0000) | ((res >> 16) & 0xffff);
+    EAX = (EAX & ~0xffff) | (res & 0xffff);
+    EDX = (EDX & ~0xffff) | ((res >> 16) & 0xffff);
     CC_DST = res;
     CC_SRC = res >> 16;
 }
@@ -207,8 +266,8 @@ void OPPROTO op_imulw_AX_T0(void)
 {
     int res;
     res = (int16_t)EAX * (int16_t)T0;
-    EAX = (EAX & 0xffff0000) | (res & 0xffff);
-    EDX = (EDX & 0xffff0000) | ((res >> 16) & 0xffff);
+    EAX = (EAX & ~0xffff) | (res & 0xffff);
+    EDX = (EDX & ~0xffff) | ((res >> 16) & 0xffff);
     CC_DST = res;
     CC_SRC = (res != (int16_t)res);
 }
@@ -217,10 +276,10 @@ void OPPROTO op_mull_EAX_T0(void)
 {
     uint64_t res;
     res = (uint64_t)((uint32_t)EAX) * (uint64_t)((uint32_t)T0);
-    EAX = res;
-    EDX = res >> 32;
-    CC_DST = res;
-    CC_SRC = res >> 32;
+    EAX = (uint32_t)res;
+    EDX = (uint32_t)(res >> 32);
+    CC_DST = (uint32_t)res;
+    CC_SRC = (uint32_t)(res >> 32);
 }
  
 void OPPROTO op_imull_EAX_T0(void)
@@ -251,6 +310,23 @@ void OPPROTO op_imull_T0_T1(void)
     CC_SRC = (res != (int32_t)res);
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_mulq_EAX_T0(void)
+{
+    helper_mulq_EAX_T0();
+}
+
+void OPPROTO op_imulq_EAX_T0(void)
+{
+    helper_imulq_EAX_T0();
+}
+
+void OPPROTO op_imulq_T0_T1(void)
+{
+    helper_imulq_T0_T1();
+}
+#endif
+
 /* division, flags are undefined */
 /* XXX: add exceptions for overflow */
  
@@ -261,12 +337,11 @@ void OPPROTO op_divb_AL_T0(void)
     num = (EAX & 0xffff);
     den = (T0 & 0xff);
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xff;
     r = (num % den) & 0xff;
-    EAX = (EAX & 0xffff0000) | (r << 8) | q;
+    EAX = (EAX & ~0xffff) | (r << 8) | q;
 }
  
 void OPPROTO op_idivb_AL_T0(void)
@@ -276,12 +351,11 @@ void OPPROTO op_idivb_AL_T0(void)
     num = (int16_t)EAX;
     den = (int8_t)T0;
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xff;
     r = (num % den) & 0xff;
-    EAX = (EAX & 0xffff0000) | (r << 8) | q;
+    EAX = (EAX & ~0xffff) | (r << 8) | q;
 }
  
 void OPPROTO op_divw_AX_T0(void)
@@ -291,13 +365,12 @@ void OPPROTO op_divw_AX_T0(void)
     num = (EAX & 0xffff) | ((EDX & 0xffff) << 16);
     den = (T0 & 0xffff);
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xffff;
     r = (num % den) & 0xffff;
-    EAX = (EAX & 0xffff0000) | q;
-    EDX = (EDX & 0xffff0000) | r;
+    EAX = (EAX & ~0xffff) | q;
+    EDX = (EDX & ~0xffff) | r;
 }
  
 void OPPROTO op_idivw_AX_T0(void)
@@ -307,30 +380,47 @@ void OPPROTO op_idivw_AX_T0(void)
     num = (EAX & 0xffff) | ((EDX & 0xffff) << 16);
     den = (int16_t)T0;
     if (den == 0) {
-        EIP = PARAM1;
         raise_exception(EXCP00_DIVZ);
     }
     q = (num / den) & 0xffff;
     r = (num % den) & 0xffff;
-    EAX = (EAX & 0xffff0000) | q;
-    EDX = (EDX & 0xffff0000) | r;
+    EAX = (EAX & ~0xffff) | q;
+    EDX = (EDX & ~0xffff) | r;
 }
  
 void OPPROTO op_divl_EAX_T0(void)
 {
-    helper_divl_EAX_T0(PARAM1);
+    helper_divl_EAX_T0();
 }
  
 void OPPROTO op_idivl_EAX_T0(void)
 {
-    helper_idivl_EAX_T0(PARAM1);
+    helper_idivl_EAX_T0();
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_divq_EAX_T0(void)
+{
+    helper_divq_EAX_T0();
+}
+
+void OPPROTO op_idivq_EAX_T0(void)
+{
+    helper_idivq_EAX_T0();
+}
+#endif
+
 /* constant load & misc op */
  
+/* XXX: consistent names */
+void OPPROTO op_movl_T0_imu(void)
+{
+    T0 = (uint32_t)PARAM1;
+}
+
 void OPPROTO op_movl_T0_im(void)
 {
-    T0 = PARAM1;
+    T0 = (int32_t)PARAM1;
 }
  
 void OPPROTO op_addl_T0_im(void)
@@ -353,9 +443,14 @@ void OPPROTO op_movl_T0_T1(void)
     T0 = T1;
 }
  
+void OPPROTO op_movl_T1_imu(void)
+{
+    T1 = (uint32_t)PARAM1;
+}
+
 void OPPROTO op_movl_T1_im(void)
 {
-    T1 = PARAM1;
+    T1 = (int32_t)PARAM1;
 }
  
 void OPPROTO op_addl_T1_im(void)
@@ -370,19 +465,95 @@ void OPPROTO op_movl_T1_A0(void)
  
 void OPPROTO op_movl_A0_im(void)
 {
-    A0 = PARAM1;
+    A0 = (uint32_t)PARAM1;
 }
  
 void OPPROTO op_addl_A0_im(void)
 {
-    A0 += PARAM1;
+    A0 = (uint32_t)(A0 + PARAM1);
+}
+
+void OPPROTO op_movl_A0_seg(void)
+{
+    A0 = (uint32_t)*(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_addl_A0_seg(void)
+{
+    A0 = (uint32_t)(A0 + *(target_ulong *)((char *)env + PARAM1));
 }
  
 void OPPROTO op_addl_A0_AL(void)
 {
-    A0 += (EAX & 0xff);
+    A0 = (uint32_t)(A0 + (EAX & 0xff));
+}
+
+#ifdef WORDS_BIGENDIAN
+typedef union UREG64 {
+    struct { uint16_t v3, v2, v1, v0; } w;
+    struct { uint32_t v1, v0; } l;
+    uint64_t q;
+} UREG64;
+#else
+typedef union UREG64 {
+    struct { uint16_t v0, v1, v2, v3; } w;
+    struct { uint32_t v0, v1; } l;
+    uint64_t q;
+} UREG64;
+#endif
+
+#ifdef TARGET_X86_64
+
+#define PARAMQ1 \
+({\
+    UREG64 __p;\
+    __p.l.v1 = PARAM1;\
+    __p.l.v0 = PARAM2;\
+    __p.q;\
+}) 
+
+void OPPROTO op_movq_T0_im64(void)
+{
+    T0 = PARAMQ1;
 }
  
+void OPPROTO op_movq_A0_im(void)
+{
+    A0 = (int32_t)PARAM1;
+}
+
+void OPPROTO op_movq_A0_im64(void)
+{
+    A0 = PARAMQ1;
+}
+
+void OPPROTO op_addq_A0_im(void)
+{
+    A0 = (A0 + (int32_t)PARAM1);
+}
+
+void OPPROTO op_addq_A0_im64(void)
+{
+    A0 = (A0 + PARAMQ1);
+}
+
+void OPPROTO op_movq_A0_seg(void)
+{
+    A0 = *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_addq_A0_seg(void)
+{
+    A0 += *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_addq_A0_AL(void)
+{
+    A0 = (A0 + (EAX & 0xff));
+}
+
+#endif
+
 void OPPROTO op_andl_A0_ffff(void)
 {
     A0 = A0 & 0xffff;
@@ -401,29 +572,29 @@ void OPPROTO op_andl_A0_ffff(void)
 #include "ops_mem.h"
 #endif
  
-/* used for bit operations */
+/* indirect jump */
  
-void OPPROTO op_add_bitw_A0_T1(void)
+void OPPROTO op_jmp_T0(void)
 {
-    A0 += ((int16_t)T1 >> 4) << 1;
+    EIP = T0;
 }
  
-void OPPROTO op_add_bitl_A0_T1(void)
+void OPPROTO op_movl_eip_im(void)
 {
-    A0 += ((int32_t)T1 >> 5) << 2;
+    EIP = (uint32_t)PARAM1;
 }
  
-/* indirect jump */
-
-void OPPROTO op_jmp_T0(void)
+#ifdef TARGET_X86_64
+void OPPROTO op_movq_eip_im(void)
 {
-    EIP = T0;
+    EIP = (int32_t)PARAM1;
 }
  
-void OPPROTO op_jmp_im(void)
+void OPPROTO op_movq_eip_im64(void)
 {
-    EIP = PARAM1;
+    EIP = PARAMQ1;
 }
+#endif
  
 void OPPROTO op_hlt(void)
 {
@@ -505,11 +676,10 @@ void OPPROTO op_sti_vm(void)
 void OPPROTO op_boundw(void)
 {
     int low, high, v;
-    low = ldsw((uint8_t *)A0);
-    high = ldsw((uint8_t *)A0 + 2);
+    low = ldsw(A0);
+    high = ldsw(A0 + 2);
     v = (int16_t)T0;
     if (v < low || v > high) {
-        EIP = PARAM1;
         raise_exception(EXCP05_BOUND);
     }
     FORCE_RET();
@@ -518,11 +688,10 @@ void OPPROTO op_boundw(void)
 void OPPROTO op_boundl(void)
 {
     int low, high, v;
-    low = ldl((uint8_t *)A0);
-    high = ldl((uint8_t *)A0 + 4);
+    low = ldl(A0);
+    high = ldl(A0 + 4);
     v = T0;
     if (v < low || v > high) {
-        EIP = PARAM1;
         raise_exception(EXCP05_BOUND);
     }
     FORCE_RET();
@@ -533,11 +702,6 @@ void OPPROTO op_cmpxchg8b(void)
     helper_cmpxchg8b();
 }
  
-void OPPROTO op_jmp(void)
-{
-    JUMP_TB(op_jmp, PARAM1, 0, PARAM2);
-}
-
 void OPPROTO op_movl_T0_0(void)
 {
     T0 = 0;
@@ -564,6 +728,14 @@ void OPPROTO op_exit_tb(void)
 #include "ops_template.h"
 #undef SHIFT
  
+#ifdef TARGET_X86_64
+
+#define SHIFT 3
+#include "ops_template.h"
+#undef SHIFT
+
+#endif
+
 /* sign extend */
  
 void OPPROTO op_movsbl_T0_T0(void)
@@ -581,6 +753,11 @@ void OPPROTO op_movswl_T0_T0(void)
     T0 = (int16_t)T0;
 }
  
+void OPPROTO op_movslq_T0_T0(void)
+{
+    T0 = (int32_t)T0;
+}
+
 void OPPROTO op_movzwl_T0_T0(void)
 {
     T0 = (uint16_t)T0;
@@ -591,9 +768,16 @@ void OPPROTO op_movswl_EAX_AX(void)
     EAX = (int16_t)EAX;
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_movslq_RAX_EAX(void)
+{
+    EAX = (int32_t)EAX;
+}
+#endif
+
 void OPPROTO op_movsbw_AX_AL(void)
 {
-    EAX = (EAX & 0xffff0000) | ((int8_t)EAX & 0xffff);
+    EAX = (EAX & ~0xffff) | ((int8_t)EAX & 0xffff);
 }
  
 void OPPROTO op_movslq_EDX_EAX(void)
@@ -603,14 +787,21 @@ void OPPROTO op_movslq_EDX_EAX(void)
  
 void OPPROTO op_movswl_DX_AX(void)
 {
-    EDX = (EDX & 0xffff0000) | (((int16_t)EAX >> 15) & 0xffff);
+    EDX = (EDX & ~0xffff) | (((int16_t)EAX >> 15) & 0xffff);
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO op_movsqo_RDX_RAX(void)
+{
+    EDX = (int64_t)EAX >> 63;
 }
+#endif
  
 /* string ops helpers */
  
 void OPPROTO op_addl_ESI_T0(void)
 {
-    ESI += T0;
+    ESI = (uint32_t)(ESI + T0);
 }
  
 void OPPROTO op_addw_ESI_T0(void)
@@ -620,7 +811,7 @@ void OPPROTO op_addw_ESI_T0(void)
  
 void OPPROTO op_addl_EDI_T0(void)
 {
-    EDI += T0;
+    EDI = (uint32_t)(EDI + T0);
 }
  
 void OPPROTO op_addw_EDI_T0(void)
@@ -630,7 +821,7 @@ void OPPROTO op_addw_EDI_T0(void)
  
 void OPPROTO op_decl_ECX(void)
 {
-    ECX--;
+    ECX = (uint32_t)(ECX - 1);
 }
  
 void OPPROTO op_decw_ECX(void)
@@ -638,6 +829,23 @@ void OPPROTO op_decw_ECX(void)
     ECX = (ECX & ~0xffff) | ((ECX - 1) & 0xffff);
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_addq_ESI_T0(void)
+{
+    ESI = (ESI + T0);
+}
+
+void OPPROTO op_addq_EDI_T0(void)
+{
+    EDI = (EDI + T0);
+}
+
+void OPPROTO op_decq_ECX(void)
+{
+    ECX--;
+}
+#endif
+
 /* push/pop utils */
  
 void op_addl_A0_SS(void)
@@ -647,22 +855,22 @@ void op_addl_A0_SS(void)
  
 void op_subl_A0_2(void)
 {
-    A0 -= 2;
+    A0 = (uint32_t)(A0 - 2);
 }
  
 void op_subl_A0_4(void)
 {
-    A0 -= 4;
+    A0 = (uint32_t)(A0 - 4);
 }
  
 void op_addl_ESP_4(void)
 {
-    ESP += 4;
+    ESP = (uint32_t)(ESP + 4);
 }
  
 void op_addl_ESP_2(void)
 {
-    ESP += 2;
+    ESP = (uint32_t)(ESP + 2);
 }
  
 void op_addw_ESP_4(void)
@@ -677,7 +885,7 @@ void op_addw_ESP_2(void)
  
 void op_addl_ESP_im(void)
 {
-    ESP += PARAM1;
+    ESP = (uint32_t)(ESP + PARAM1);
 }
  
 void op_addw_ESP_im(void)
@@ -685,6 +893,23 @@ void op_addw_ESP_im(void)
     ESP = (ESP & ~0xffff) | ((ESP + PARAM1) & 0xffff);
 }
  
+#ifdef TARGET_X86_64
+void op_subq_A0_8(void)
+{
+    A0 -= 8;
+}
+
+void op_addq_ESP_8(void)
+{
+    ESP += 8;
+}
+
+void op_addq_ESP_im(void)
+{
+    ESP += PARAM1;
+}
+#endif
+
 void OPPROTO op_rdtsc(void)
 {
     helper_rdtsc();
@@ -710,6 +935,18 @@ void OPPROTO op_sysexit(void)
     helper_sysexit();
 }
  
+#ifdef TARGET_X86_64
+void OPPROTO op_syscall(void)
+{
+    helper_syscall();
+}
+
+void OPPROTO op_sysret(void)
+{
+    helper_sysret(PARAM1);
+}
+#endif
+
 void OPPROTO op_rdmsr(void)
 {
     helper_rdmsr();
@@ -868,7 +1105,7 @@ void OPPROTO op_movl_seg_T0_vm(void)
     /* env->segs[] access */
     sc = (SegmentCache *)((char *)env + PARAM1);
     sc->selector = selector;
-    sc->base = (void *)(selector << 4);
+    sc->base = (selector << 4);
 }
  
 void OPPROTO op_movl_T0_seg(void)
@@ -876,16 +1113,6 @@ void OPPROTO op_movl_T0_seg(void)
     T0 = env->segs[PARAM1].selector;
 }
  
-void OPPROTO op_movl_A0_seg(void)
-{
-    A0 = *(unsigned long *)((char *)env + PARAM1);
-}
-
-void OPPROTO op_addl_A0_seg(void)
-{
-    A0 += *(unsigned long *)((char *)env + PARAM1);
-}
-
 void OPPROTO op_lsl(void)
 {
     helper_lsl();
@@ -1006,6 +1233,26 @@ void OPPROTO op_movl_env_T1(void)
     *(uint32_t *)((char *)env + PARAM1) = T1;
 }
  
+void OPPROTO op_movtl_T0_env(void)
+{
+    T0 = *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_movtl_env_T0(void)
+{
+    *(target_ulong *)((char *)env + PARAM1) = T0;
+}
+
+void OPPROTO op_movtl_T1_env(void)
+{
+    T1 = *(target_ulong *)((char *)env + PARAM1);
+}
+
+void OPPROTO op_movtl_env_T1(void)
+{
+    *(target_ulong *)((char *)env + PARAM1) = T1;
+}
+
 void OPPROTO op_clts(void)
 {
     env->cr[0] &= ~CR0_TS_MASK;
@@ -1014,25 +1261,31 @@ void OPPROTO op_clts(void)
  
 /* flags handling */
  
-/* slow jumps cases : in order to avoid calling a function with a
-   pointer (which can generate a stack frame on PowerPC), we use
-   op_setcc to set T0 and then call op_jcc. */
-void OPPROTO op_jcc(void)
+void OPPROTO op_goto_tb0(void)
 {
-    if (T0)
-        JUMP_TB(op_jcc, PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(op_jcc, PARAM1, 1, PARAM3);
-    FORCE_RET();
+    GOTO_TB(op_goto_tb0, 0);
+}
+
+void OPPROTO op_goto_tb1(void)
+{
+    GOTO_TB(op_goto_tb1, 1);
+}
+
+void OPPROTO op_jmp_label(void)
+{
+    GOTO_LABEL_PARAM(1);
 }
  
-void OPPROTO op_jcc_im(void)
+void OPPROTO op_jnz_T0_label(void)
 {
     if (T0)
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
-    FORCE_RET();
+        GOTO_LABEL_PARAM(1);
+}
+
+void OPPROTO op_jz_T0_label(void)
+{
+    if (!T0)
+        GOTO_LABEL_PARAM(1);
 }
  
 /* slow set cases (compute x86 flags) */
@@ -1299,6 +1552,28 @@ CCTable cc_table[CC_OP_NB] = {
     [CC_OP_SARB] = { compute_all_sarb, compute_c_sarl },
     [CC_OP_SARW] = { compute_all_sarw, compute_c_sarl },
     [CC_OP_SARL] = { compute_all_sarl, compute_c_sarl },
+
+#ifdef TARGET_X86_64
+    [CC_OP_MULQ] = { compute_all_mulq, compute_c_mull },
+
+    [CC_OP_ADDQ] = { compute_all_addq, compute_c_addq  },
+
+    [CC_OP_ADCQ] = { compute_all_adcq, compute_c_adcq  },
+
+    [CC_OP_SUBQ] = { compute_all_subq, compute_c_subq  },
+    
+    [CC_OP_SBBQ] = { compute_all_sbbq, compute_c_sbbq  },
+    
+    [CC_OP_LOGICQ] = { compute_all_logicq, compute_c_logicq },
+    
+    [CC_OP_INCQ] = { compute_all_incq, compute_c_incl },
+
+    [CC_OP_DECQ] = { compute_all_decq, compute_c_incl },
+
+    [CC_OP_SHLQ] = { compute_all_shlq, compute_c_shlq },
+
+    [CC_OP_SARQ] = { compute_all_sarq, compute_c_sarl },
+#endif
 };
  
 /* floating point support. Some of the code for complicated x87
@@ -1330,20 +1605,20 @@ double qemu_rint(double x)
 void OPPROTO op_flds_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldl((void *)A0);
+    FP_CONVERT.i32 = ldl(A0);
     FT0 = FP_CONVERT.f;
 #else
-    FT0 = ldfl((void *)A0);
+    FT0 = ldfl(A0);
 #endif
 }
  
 void OPPROTO op_fldl_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = ldq((void *)A0);
+    FP_CONVERT.i64 = ldq(A0);
     FT0 = FP_CONVERT.d;
 #else
-    FT0 = ldfq((void *)A0);
+    FT0 = ldfq(A0);
 #endif
 }
  
@@ -1352,17 +1627,17 @@ void OPPROTO op_fldl_FT0_A0(void)
  
 void helper_fild_FT0_A0(void)
 {
-    FT0 = (CPU86_LDouble)ldsw((void *)A0);
+    FT0 = (CPU86_LDouble)ldsw(A0);
 }
  
 void helper_fildl_FT0_A0(void)
 {
-    FT0 = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    FT0 = (CPU86_LDouble)((int32_t)ldl(A0));
 }
  
 void helper_fildll_FT0_A0(void)
 {
-    FT0 = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    FT0 = (CPU86_LDouble)((int64_t)ldq(A0));
 }
  
 void OPPROTO op_fild_FT0_A0(void)
@@ -1385,30 +1660,30 @@ void OPPROTO op_fildll_FT0_A0(void)
 void OPPROTO op_fild_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldsw((void *)A0);
+    FP_CONVERT.i32 = ldsw(A0);
     FT0 = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    FT0 = (CPU86_LDouble)ldsw((void *)A0);
+    FT0 = (CPU86_LDouble)ldsw(A0);
 #endif
 }
  
 void OPPROTO op_fildl_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = (int32_t) ldl((void *)A0);
+    FP_CONVERT.i32 = (int32_t) ldl(A0);
     FT0 = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    FT0 = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    FT0 = (CPU86_LDouble)((int32_t)ldl(A0));
 #endif
 }
  
 void OPPROTO op_fildll_FT0_A0(void)
 {
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = (int64_t) ldq((void *)A0);
+    FP_CONVERT.i64 = (int64_t) ldq(A0);
     FT0 = (CPU86_LDouble)FP_CONVERT.i64;
 #else
-    FT0 = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    FT0 = (CPU86_LDouble)((int64_t)ldq(A0));
 #endif
 }
 #endif
@@ -1420,10 +1695,10 @@ void OPPROTO op_flds_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldl((void *)A0);
+    FP_CONVERT.i32 = ldl(A0);
     env->fpregs[new_fpstt] = FP_CONVERT.f;
 #else
-    env->fpregs[new_fpstt] = ldfl((void *)A0);
+    env->fpregs[new_fpstt] = ldfl(A0);
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1434,10 +1709,10 @@ void OPPROTO op_fldl_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = ldq((void *)A0);
+    FP_CONVERT.i64 = ldq(A0);
     env->fpregs[new_fpstt] = FP_CONVERT.d;
 #else
-    env->fpregs[new_fpstt] = ldfq((void *)A0);
+    env->fpregs[new_fpstt] = ldfq(A0);
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1455,7 +1730,7 @@ void helper_fild_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw((void *)A0);
+    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw(A0);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
@@ -1464,7 +1739,7 @@ void helper_fildl_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl(A0));
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
@@ -1473,7 +1748,7 @@ void helper_fildll_ST0_A0(void)
 {
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq(A0));
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
@@ -1500,10 +1775,10 @@ void OPPROTO op_fild_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = ldsw((void *)A0);
+    FP_CONVERT.i32 = ldsw(A0);
     env->fpregs[new_fpstt] = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw((void *)A0);
+    env->fpregs[new_fpstt] = (CPU86_LDouble)ldsw(A0);
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1514,10 +1789,10 @@ void OPPROTO op_fildl_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i32 = (int32_t) ldl((void *)A0);
+    FP_CONVERT.i32 = (int32_t) ldl(A0);
     env->fpregs[new_fpstt] = (CPU86_LDouble)FP_CONVERT.i32;
 #else
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int32_t)ldl(A0));
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1528,10 +1803,10 @@ void OPPROTO op_fildll_ST0_A0(void)
     int new_fpstt;
     new_fpstt = (env->fpstt - 1) & 7;
 #ifdef USE_FP_CONVERT
-    FP_CONVERT.i64 = (int64_t) ldq((void *)A0);
+    FP_CONVERT.i64 = (int64_t) ldq(A0);
     env->fpregs[new_fpstt] = (CPU86_LDouble)FP_CONVERT.i64;
 #else
-    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq((void *)A0));
+    env->fpregs[new_fpstt] = (CPU86_LDouble)((int64_t)ldq(A0));
 #endif
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
@@ -1545,15 +1820,15 @@ void OPPROTO op_fsts_ST0_A0(void)
 {
 #ifdef USE_FP_CONVERT
     FP_CONVERT.f = (float)ST0;
-    stfl((void *)A0, FP_CONVERT.f);
+    stfl(A0, FP_CONVERT.f);
 #else
-    stfl((void *)A0, (float)ST0);
+    stfl(A0, (float)ST0);
 #endif
 }
  
 void OPPROTO op_fstl_ST0_A0(void)
 {
-    stfq((void *)A0, (double)ST0);
+    stfq(A0, (double)ST0);
 }
  
 void OPPROTO op_fstt_ST0_A0(void)
@@ -1574,7 +1849,7 @@ void OPPROTO op_fist_ST0_A0(void)
     val = lrint(d);
     if (val != (int16_t)val)
         val = -32768;
-    stw((void *)A0, val);
+    stw(A0, val);
 }
  
 void OPPROTO op_fistl_ST0_A0(void)
@@ -1588,7 +1863,7 @@ void OPPROTO op_fistl_ST0_A0(void)
  
     d = ST0;
     val = lrint(d);
-    stl((void *)A0, val);
+    stl(A0, val);
 }
  
 void OPPROTO op_fistll_ST0_A0(void)
@@ -1602,7 +1877,7 @@ void OPPROTO op_fistll_ST0_A0(void)
  
     d = ST0;
     val = llrint(d);
-    stq((void *)A0, val);
+    stq(A0, val);
 }
  
 void OPPROTO op_fbld_ST0_A0(void)
@@ -1934,25 +2209,25 @@ void OPPROTO op_fnstsw_A0(void)
 {
     int fpus;
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
-    stw((void *)A0, fpus);
+    stw(A0, fpus);
 }
  
 void OPPROTO op_fnstsw_EAX(void)
 {
     int fpus;
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
-    EAX = (EAX & 0xffff0000) | fpus;
+    EAX = (EAX & ~0xffff) | fpus;
 }
  
 void OPPROTO op_fnstcw_A0(void)
 {
-    stw((void *)A0, env->fpuc);
+    stw(A0, env->fpuc);
 }
  
 void OPPROTO op_fldcw_A0(void)
 {
     int rnd_type;
-    env->fpuc = lduw((void *)A0);
+    env->fpuc = lduw(A0);
     /* set rounding mode */
     switch(env->fpuc & RC_MASK) {
     default:
@@ -2001,22 +2276,22 @@ void OPPROTO op_fninit(void)
  
 void OPPROTO op_fnstenv_A0(void)
 {
-    helper_fstenv((uint8_t *)A0, PARAM1);
+    helper_fstenv(A0, PARAM1);
 }
  
 void OPPROTO op_fldenv_A0(void)
 {
-    helper_fldenv((uint8_t *)A0, PARAM1);
+    helper_fldenv(A0, PARAM1);
 }
  
 void OPPROTO op_fnsave_A0(void)
 {
-    helper_fsave((uint8_t *)A0, PARAM1);
+    helper_fsave(A0, PARAM1);
 }
  
 void OPPROTO op_frstor_A0(void)
 {
-    helper_frstor((uint8_t *)A0, PARAM1);
+    helper_frstor(A0, PARAM1);
 }
  
 /* threading support */
@@ -2030,3 +2305,30 @@ void OPPROTO op_unlock(void)
     cpu_unlock();
 }
  
+/* SSE support */
+static inline void memcpy16(void *d, void *s)
+{
+    ((uint32_t *)d)[0] = ((uint32_t *)s)[0];
+    ((uint32_t *)d)[1] = ((uint32_t *)s)[1];
+    ((uint32_t *)d)[2] = ((uint32_t *)s)[2];
+    ((uint32_t *)d)[3] = ((uint32_t *)s)[3];
+}
+
+void OPPROTO op_movo(void)
+{
+    /* XXX: badly generated code */
+    XMMReg *d, *s;
+    d = (XMMReg *)((char *)env + PARAM1);
+    s = (XMMReg *)((char *)env + PARAM2);
+    memcpy16(d, s);
+}
+
+void OPPROTO op_fxsave_A0(void)
+{
+    helper_fxsave(A0, PARAM1);
+}
+
+void OPPROTO op_fxrstor_A0(void)
+{
+    helper_fxrstor(A0, PARAM1);
+}
@@ -20,29 +20,56 @@
  */
 void OPPROTO glue(op_movl_A0,REGNAME)(void)
 {
-    A0 = REG;
+    A0 = (uint32_t)REG;
 }
  
 void OPPROTO glue(op_addl_A0,REGNAME)(void)
 {
-    A0 += REG;
+    A0 = (uint32_t)(A0 + REG);
 }
  
 void OPPROTO glue(glue(op_addl_A0,REGNAME),_s1)(void)
 {
-    A0 += REG << 1;
+    A0 = (uint32_t)(A0 + (REG << 1));
 }
  
 void OPPROTO glue(glue(op_addl_A0,REGNAME),_s2)(void)
 {
-    A0 += REG << 2;
+    A0 = (uint32_t)(A0 + (REG << 2));
 }
  
 void OPPROTO glue(glue(op_addl_A0,REGNAME),_s3)(void)
 {
-    A0 += REG << 3;
+    A0 = (uint32_t)(A0 + (REG << 3));
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(op_movq_A0,REGNAME)(void)
+{
+    A0 = REG;
+}
+
+void OPPROTO glue(op_addq_A0,REGNAME)(void)
+{
+    A0 = (A0 + REG);
+}
+
+void OPPROTO glue(glue(op_addq_A0,REGNAME),_s1)(void)
+{
+    A0 = (A0 + (REG << 1));
+}
+
+void OPPROTO glue(glue(op_addq_A0,REGNAME),_s2)(void)
+{
+    A0 = (A0 + (REG << 2));
 }
  
+void OPPROTO glue(glue(op_addq_A0,REGNAME),_s3)(void)
+{
+    A0 = (A0 + (REG << 3));
+}
+#endif
+
 void OPPROTO glue(op_movl_T0,REGNAME)(void)
 {
     T0 = REG;
@@ -65,72 +92,99 @@ void OPPROTO glue(op_movh_T1,REGNAME)(void)
  
 void OPPROTO glue(glue(op_movl,REGNAME),_T0)(void)
 {
-    REG = T0;
+    REG = (uint32_t)T0;
 }
  
 void OPPROTO glue(glue(op_movl,REGNAME),_T1)(void)
 {
-    REG = T1;
+    REG = (uint32_t)T1;
 }
  
 void OPPROTO glue(glue(op_movl,REGNAME),_A0)(void)
 {
+    REG = (uint32_t)A0;
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(glue(op_movq,REGNAME),_T0)(void)
+{
+    REG = T0;
+}
+
+void OPPROTO glue(glue(op_movq,REGNAME),_T1)(void)
+{
+    REG = T1;
+}
+
+void OPPROTO glue(glue(op_movq,REGNAME),_A0)(void)
+{
     REG = A0;
 }
+#endif
  
 /* mov T1 to REG if T0 is true */
 void OPPROTO glue(glue(op_cmovw,REGNAME),_T1_T0)(void)
 {
     if (T0)
-        REG = (REG & 0xffff0000) | (T1 & 0xffff);
+        REG = (REG & ~0xffff) | (T1 & 0xffff);
     FORCE_RET();
 }
  
 void OPPROTO glue(glue(op_cmovl,REGNAME),_T1_T0)(void)
 {
     if (T0)
+        REG = (uint32_t)T1;
+    FORCE_RET();
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(glue(op_cmovq,REGNAME),_T1_T0)(void)
+{
+    if (T0)
         REG = T1;
     FORCE_RET();
 }
+#endif
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movw,REGNAME),_T0)(void)
 {
-    REG = (REG & 0xffff0000) | (T0 & 0xffff);
+    REG = (REG & ~0xffff) | (T0 & 0xffff);
 }
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movw,REGNAME),_T1)(void)
 {
-    REG = (REG & 0xffff0000) | (T1 & 0xffff);
+    REG = (REG & ~0xffff) | (T1 & 0xffff);
 }
  
 /* NOTE: A0 high order bits are ignored */
 void OPPROTO glue(glue(op_movw,REGNAME),_A0)(void)
 {
-    REG = (REG & 0xffff0000) | (A0 & 0xffff);
+    REG = (REG & ~0xffff) | (A0 & 0xffff);
 }
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movb,REGNAME),_T0)(void)
 {
-    REG = (REG & 0xffffff00) | (T0 & 0xff);
+    REG = (REG & ~0xff) | (T0 & 0xff);
 }
  
 /* NOTE: T0 high order bits are ignored */
 void OPPROTO glue(glue(op_movh,REGNAME),_T0)(void)
 {
-    REG = (REG & 0xffff00ff) | ((T0 & 0xff) << 8);
+    REG = (REG & ~0xff00) | ((T0 & 0xff) << 8);
 }
  
 /* NOTE: T1 high order bits are ignored */
 void OPPROTO glue(glue(op_movb,REGNAME),_T1)(void)
 {
-    REG = (REG & 0xffffff00) | (T1 & 0xff);
+    REG = (REG & ~0xff) | (T1 & 0xff);
 }
  
 /* NOTE: T1 high order bits are ignored */
 void OPPROTO glue(glue(op_movh,REGNAME),_T1)(void)
 {
-    REG = (REG & 0xffff00ff) | ((T1 & 0xff) << 8);
+    REG = (REG & ~0xff00) | ((T1 & 0xff) << 8);
 }
+
 void OPPROTO glue(glue(op_ldub, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldub, MEMSUFFIX)((uint8_t *)A0);
+    T0 = glue(ldub, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsb, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldsb, MEMSUFFIX)((int8_t *)A0);
+    T0 = glue(ldsb, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_lduw, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(lduw, MEMSUFFIX)((uint8_t *)A0);
+    T0 = glue(lduw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsw, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldsw, MEMSUFFIX)((int8_t *)A0);
+    T0 = glue(ldsw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldl, MEMSUFFIX), _T0_A0)(void)
 {
-    T0 = glue(ldl, MEMSUFFIX)((uint8_t *)A0);
+    T0 = (uint32_t)glue(ldl, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldub, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldub, MEMSUFFIX)((uint8_t *)A0);
+    T1 = glue(ldub, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsb, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldsb, MEMSUFFIX)((int8_t *)A0);
+    T1 = glue(ldsb, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_lduw, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(lduw, MEMSUFFIX)((uint8_t *)A0);
+    T1 = glue(lduw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldsw, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldsw, MEMSUFFIX)((int8_t *)A0);
+    T1 = glue(ldsw, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_ldl, MEMSUFFIX), _T1_A0)(void)
 {
-    T1 = glue(ldl, MEMSUFFIX)((uint8_t *)A0);
+    T1 = glue(ldl, MEMSUFFIX)(A0);
 }
  
 void OPPROTO glue(glue(op_stb, MEMSUFFIX), _T0_A0)(void)
 {
-    glue(stb, MEMSUFFIX)((uint8_t *)A0, T0);
+    glue(stb, MEMSUFFIX)(A0, T0);
 }
  
 void OPPROTO glue(glue(op_stw, MEMSUFFIX), _T0_A0)(void)
 {
-    glue(stw, MEMSUFFIX)((uint8_t *)A0, T0);
+    glue(stw, MEMSUFFIX)(A0, T0);
 }
  
 void OPPROTO glue(glue(op_stl, MEMSUFFIX), _T0_A0)(void)
 {
-    glue(stl, MEMSUFFIX)((uint8_t *)A0, T0);
+    glue(stl, MEMSUFFIX)(A0, T0);
 }
  
 #if 0
 void OPPROTO glue(glue(op_stb, MEMSUFFIX), _T1_A0)(void)
 {
-    glue(stb, MEMSUFFIX)((uint8_t *)A0, T1);
+    glue(stb, MEMSUFFIX)(A0, T1);
 }
 #endif
  
 void OPPROTO glue(glue(op_stw, MEMSUFFIX), _T1_A0)(void)
 {
-    glue(stw, MEMSUFFIX)((uint8_t *)A0, T1);
+    glue(stw, MEMSUFFIX)(A0, T1);
 }
  
 void OPPROTO glue(glue(op_stl, MEMSUFFIX), _T1_A0)(void)
 {
-    glue(stl, MEMSUFFIX)((uint8_t *)A0, T1);
+    glue(stl, MEMSUFFIX)(A0, T1);
 }
  
+/* SSE support */
+void OPPROTO glue(glue(op_ldo, MEMSUFFIX), _env_A0)(void)
+{
+    XMMReg *p;
+    p = (XMMReg *)((char *)env + PARAM1);
+    /* XXX: host endianness ? */
+    p->u.q[0] = glue(ldq, MEMSUFFIX)(A0);
+    p->u.q[1] = glue(ldq, MEMSUFFIX)(A0 + 8);
+}
+
+void OPPROTO glue(glue(op_sto, MEMSUFFIX), _env_A0)(void)
+{
+    XMMReg *p;
+    p = (XMMReg *)((char *)env + PARAM1);
+    /* XXX: host endianness ? */
+    glue(stq, MEMSUFFIX)(A0, p->u.q[0]);
+    glue(stq, MEMSUFFIX)(A0 + 8, p->u.q[1]);
+}
+
+#ifdef TARGET_X86_64
+void OPPROTO glue(glue(op_ldsl, MEMSUFFIX), _T0_A0)(void)
+{
+    T0 = (int32_t)glue(ldl, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_ldsl, MEMSUFFIX), _T1_A0)(void)
+{
+    T1 = (int32_t)glue(ldl, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_ldq, MEMSUFFIX), _T0_A0)(void)
+{
+    T0 = glue(ldq, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_ldq, MEMSUFFIX), _T1_A0)(void)
+{
+    T1 = glue(ldq, MEMSUFFIX)(A0);
+}
+
+void OPPROTO glue(glue(op_stq, MEMSUFFIX), _T0_A0)(void)
+{
+    glue(stq, MEMSUFFIX)(A0, T0);
+}
+
+void OPPROTO glue(glue(op_stq, MEMSUFFIX), _T1_A0)(void)
+{
+    glue(stq, MEMSUFFIX)(A0, T1);
+}
+#endif
+
 #undef MEMSUFFIX
@@ -20,7 +20,12 @@
  */
 #define DATA_BITS (1 << (3 + SHIFT))
 #define SHIFT_MASK (DATA_BITS - 1)
-#define SIGN_MASK (1 << (DATA_BITS - 1))
+#define SIGN_MASK (((target_ulong)1) << (DATA_BITS - 1))
+#if DATA_BITS <= 32
+#define SHIFT1_MASK 0x1f
+#else
+#define SHIFT1_MASK 0x3f
+#endif
  
 #if DATA_BITS == 8
 #define SUFFIX b
@@ -37,6 +42,11 @@
 #define DATA_TYPE uint32_t
 #define DATA_STYPE int32_t
 #define DATA_MASK 0xffffffff
+#elif DATA_BITS == 64
+#define SUFFIX q
+#define DATA_TYPE uint64_t
+#define DATA_STYPE int64_t
+#define DATA_MASK 0xffffffffffffffff
 #else
 #error unhandled operand size
 #endif
@@ -46,7 +56,7 @@
 static int glue(compute_all_add, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_SRC;
     src2 = CC_DST - CC_SRC;
     cf = (DATA_TYPE)CC_DST < (DATA_TYPE)src1;
@@ -60,7 +70,8 @@ static int glue(compute_all_add, SUFFIX)(void)
  
 static int glue(compute_c_add, SUFFIX)(void)
 {
-    int src1, cf;
+    int cf;
+    target_long src1;
     src1 = CC_SRC;
     cf = (DATA_TYPE)CC_DST < (DATA_TYPE)src1;
     return cf;
@@ -69,7 +80,7 @@ static int glue(compute_c_add, SUFFIX)(void)
 static int glue(compute_all_adc, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_SRC;
     src2 = CC_DST - CC_SRC - 1;
     cf = (DATA_TYPE)CC_DST <= (DATA_TYPE)src1;
@@ -83,7 +94,8 @@ static int glue(compute_all_adc, SUFFIX)(void)
  
 static int glue(compute_c_adc, SUFFIX)(void)
 {
-    int src1, cf;
+    int cf;
+    target_long src1;
     src1 = CC_SRC;
     cf = (DATA_TYPE)CC_DST <= (DATA_TYPE)src1;
     return cf;
@@ -92,7 +104,7 @@ static int glue(compute_c_adc, SUFFIX)(void)
 static int glue(compute_all_sub, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 < (DATA_TYPE)src2;
@@ -106,7 +118,8 @@ static int glue(compute_all_sub, SUFFIX)(void)
  
 static int glue(compute_c_sub, SUFFIX)(void)
 {
-    int src1, src2, cf;
+    int cf;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 < (DATA_TYPE)src2;
@@ -116,7 +129,7 @@ static int glue(compute_c_sub, SUFFIX)(void)
 static int glue(compute_all_sbb, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC + 1;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 <= (DATA_TYPE)src2;
@@ -130,7 +143,8 @@ static int glue(compute_all_sbb, SUFFIX)(void)
  
 static int glue(compute_c_sbb, SUFFIX)(void)
 {
-    int src1, src2, cf;
+    int cf;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC + 1;
     src2 = CC_SRC;
     cf = (DATA_TYPE)src1 <= (DATA_TYPE)src2;
@@ -157,7 +171,7 @@ static int glue(compute_c_logic, SUFFIX)(void)
 static int glue(compute_all_inc, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST - 1;
     src2 = 1;
     cf = CC_SRC;
@@ -179,7 +193,7 @@ static int glue(compute_c_inc, SUFFIX)(void)
 static int glue(compute_all_dec, SUFFIX)(void)
 {
     int cf, pf, af, zf, sf, of;
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + 1;
     src2 = 1;
     cf = CC_SRC;
@@ -187,7 +201,7 @@ static int glue(compute_all_dec, SUFFIX)(void)
     af = (CC_DST ^ src1 ^ src2) & 0x10;
     zf = ((DATA_TYPE)CC_DST == 0) << 6;
     sf = lshift(CC_DST, 8 - DATA_BITS) & 0x80;
-    of = ((CC_DST & DATA_MASK) == ((uint32_t)SIGN_MASK - 1)) << 11;
+    of = ((CC_DST & DATA_MASK) == ((target_ulong)SIGN_MASK - 1)) << 11;
     return cf | pf | af | zf | sf | of;
 }
  
@@ -256,71 +270,66 @@ static int glue(compute_all_mul, SUFFIX)(void)
  
 void OPPROTO glue(op_jb_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_TYPE)src1 < (DATA_TYPE)src2)
-        JUMP_TB(glue(op_jb_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jb_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jz_sub, SUFFIX)(void)
 {
     if ((DATA_TYPE)CC_DST == 0)
-        JUMP_TB(glue(op_jz_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jz_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
+    FORCE_RET();
+}
+
+void OPPROTO glue(op_jnz_sub, SUFFIX)(void)
+{
+    if ((DATA_TYPE)CC_DST != 0)
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jbe_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_TYPE)src1 <= (DATA_TYPE)src2)
-        JUMP_TB(glue(op_jbe_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jbe_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_js_sub, SUFFIX)(void)
 {
     if (CC_DST & SIGN_MASK)
-        JUMP_TB(glue(op_js_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_js_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jl_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_STYPE)src1 < (DATA_STYPE)src2)
-        JUMP_TB(glue(op_jl_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jl_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_jle_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
     if ((DATA_STYPE)src1 <= (DATA_STYPE)src2)
-        JUMP_TB(glue(op_jle_sub, SUFFIX), PARAM1, 0, PARAM2);
-    else
-        JUMP_TB(glue(op_jle_sub, SUFFIX), PARAM1, 1, PARAM3);
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
@@ -330,50 +339,33 @@ void OPPROTO glue(op_jle_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_loopnz, SUFFIX)(void)
 {
-    unsigned int tmp;
     int eflags;
     eflags = cc_table[CC_OP].compute_all();
-    tmp = (ECX - 1) & DATA_MASK;
-    ECX = (ECX & ~DATA_MASK) | tmp;
-    if (tmp != 0 && !(eflags & CC_Z))
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX != 0 && !(eflags & CC_Z))
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
 void OPPROTO glue(op_loopz, SUFFIX)(void)
 {
-    unsigned int tmp;
     int eflags;
     eflags = cc_table[CC_OP].compute_all();
-    tmp = (ECX - 1) & DATA_MASK;
-    ECX = (ECX & ~DATA_MASK) | tmp;
-    if (tmp != 0 && (eflags & CC_Z))
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX != 0 && (eflags & CC_Z))
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
-void OPPROTO glue(op_loop, SUFFIX)(void)
+void OPPROTO glue(op_jz_ecx, SUFFIX)(void)
 {
-    unsigned int tmp;
-    tmp = (ECX - 1) & DATA_MASK;
-    ECX = (ECX & ~DATA_MASK) | tmp;
-    if (tmp != 0)
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX == 0)
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
-void OPPROTO glue(op_jecxz, SUFFIX)(void)
+void OPPROTO glue(op_jnz_ecx, SUFFIX)(void)
 {
-    if ((DATA_TYPE)ECX == 0)
-        EIP = PARAM1;
-    else
-        EIP = PARAM2;
+    if ((DATA_TYPE)ECX != 0)
+        GOTO_LABEL_PARAM(1);
     FORCE_RET();
 }
  
@@ -383,7 +375,7 @@ void OPPROTO glue(op_jecxz, SUFFIX)(void)
  
 void OPPROTO glue(op_setb_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -397,7 +389,7 @@ void OPPROTO glue(op_setz_T0_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_setbe_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -411,7 +403,7 @@ void OPPROTO glue(op_sets_T0_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_setl_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -420,7 +412,7 @@ void OPPROTO glue(op_setl_T0_sub, SUFFIX)(void)
  
 void OPPROTO glue(op_setle_T0_sub, SUFFIX)(void)
 {
-    int src1, src2;
+    target_long src1, src2;
     src1 = CC_DST + CC_SRC;
     src2 = CC_SRC;
  
@@ -432,7 +424,7 @@ void OPPROTO glue(op_setle_T0_sub, SUFFIX)(void)
 void OPPROTO glue(glue(op_shl, SUFFIX), _T0_T1)(void)
 {
     int count;
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
     T0 = T0 << count;
     FORCE_RET();
 }
@@ -440,7 +432,7 @@ void OPPROTO glue(glue(op_shl, SUFFIX), _T0_T1)(void)
 void OPPROTO glue(glue(op_shr, SUFFIX), _T0_T1)(void)
 {
     int count;
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
     T0 &= DATA_MASK;
     T0 = T0 >> count;
     FORCE_RET();
@@ -448,8 +440,10 @@ void OPPROTO glue(glue(op_shr, SUFFIX), _T0_T1)(void)
  
 void OPPROTO glue(glue(op_sar, SUFFIX), _T0_T1)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     src = (DATA_STYPE)T0;
     T0 = src >> count;
     FORCE_RET();
@@ -484,7 +478,7 @@ void OPPROTO glue(glue(op_bts, SUFFIX), _T0_T1_cc)(void)
     int count;
     count = T1 & SHIFT_MASK;
     T1 = T0 >> count;
-    T0 |= (1 << count);
+    T0 |= (((target_long)1) << count);
 }
  
 void OPPROTO glue(glue(op_btr, SUFFIX), _T0_T1_cc)(void)
@@ -492,7 +486,7 @@ void OPPROTO glue(glue(op_btr, SUFFIX), _T0_T1_cc)(void)
     int count;
     count = T1 & SHIFT_MASK;
     T1 = T0 >> count;
-    T0 &= ~(1 << count);
+    T0 &= ~(((target_long)1) << count);
 }
  
 void OPPROTO glue(glue(op_btc, SUFFIX), _T0_T1_cc)(void)
@@ -500,12 +494,19 @@ void OPPROTO glue(glue(op_btc, SUFFIX), _T0_T1_cc)(void)
     int count;
     count = T1 & SHIFT_MASK;
     T1 = T0 >> count;
-    T0 ^= (1 << count);
+    T0 ^= (((target_long)1) << count);
+}
+
+void OPPROTO glue(glue(op_add_bit, SUFFIX), _A0_T1)(void)
+{
+    A0 += ((DATA_STYPE)T1 >> (3 + SHIFT)) << SHIFT;
 }
  
 void OPPROTO glue(glue(op_bsf, SUFFIX), _T0_cc)(void)
 {
-    int res, count;
+    int count;
+    target_long res;
+    
     res = T0 & DATA_MASK;
     if (res != 0) {
         count = 0;
@@ -523,7 +524,9 @@ void OPPROTO glue(glue(op_bsf, SUFFIX), _T0_cc)(void)
  
 void OPPROTO glue(glue(op_bsr, SUFFIX), _T0_cc)(void)
 {
-    int res, count;
+    int count;
+    target_long res;
+
     res = T0 & DATA_MASK;
     if (res != 0) {
         count = DATA_BITS - 1;
@@ -555,70 +558,8 @@ void OPPROTO glue(op_movl_T0_Dshift, SUFFIX)(void)
     T0 = DF << SHIFT;
 }
  
-void OPPROTO glue(op_string_jz_sub, SUFFIX)(void)
-{
-    if ((DATA_TYPE)CC_DST == 0)
-        JUMP_TB2(glue(op_string_jz_sub, SUFFIX), PARAM1, 3);
-    FORCE_RET();
-}
-
-void OPPROTO glue(op_string_jnz_sub, SUFFIX)(void)
-{
-    if ((DATA_TYPE)CC_DST != 0)
-        JUMP_TB2(glue(op_string_jnz_sub, SUFFIX), PARAM1, 3);
-    FORCE_RET();
-}
-
-void OPPROTO glue(glue(op_string_jz_sub, SUFFIX), _im)(void)
-{
-    if ((DATA_TYPE)CC_DST == 0) {
-        EIP = PARAM1;
-        if (env->eflags & TF_MASK) {
-            raise_exception(EXCP01_SSTP);
-        }
-        T0 = 0;
-        EXIT_TB();
-    }
-    FORCE_RET();
-}
-
-void OPPROTO glue(glue(op_string_jnz_sub, SUFFIX), _im)(void)
-{
-    if ((DATA_TYPE)CC_DST != 0) {
-        EIP = PARAM1;
-        if (env->eflags & TF_MASK) {
-            raise_exception(EXCP01_SSTP);
-        }
-        T0 = 0;
-        EXIT_TB();
-    }
-    FORCE_RET();
-}
-
-#if DATA_BITS >= 16
-void OPPROTO glue(op_jz_ecx, SUFFIX)(void)
-{
-    if ((DATA_TYPE)ECX == 0)
-        JUMP_TB(glue(op_jz_ecx, SUFFIX), PARAM1, 1, PARAM2);
-    FORCE_RET();
-}
-
-void OPPROTO glue(glue(op_jz_ecx, SUFFIX), _im)(void)
-{
-    if ((DATA_TYPE)ECX == 0) {
-        EIP = PARAM1;
-        if (env->eflags & TF_MASK) {
-            raise_exception(EXCP01_SSTP);
-        }
-        T0 = 0;
-        EXIT_TB();
-    }
-    FORCE_RET();
-}
-#endif
-
 /* port I/O */
-
+#if DATA_BITS <= 32
 void OPPROTO glue(glue(op_out, SUFFIX), _T0_T1)(void)
 {
     glue(cpu_out, SUFFIX)(env, T0, T1 & DATA_MASK);
@@ -648,9 +589,11 @@ void OPPROTO glue(glue(op_check_io, SUFFIX), _DX)(void)
 {
     glue(glue(check_io, SUFFIX), _DX)();
 }
+#endif
  
 #undef DATA_BITS
 #undef SHIFT_MASK
+#undef SHIFT1_MASK
 #undef SIGN_MASK
 #undef DATA_TYPE
 #undef DATA_STYPE
@@ -28,6 +28,8 @@
 #define MEM_SUFFIX w_raw
 #elif DATA_BITS == 32
 #define MEM_SUFFIX l_raw
+#elif DATA_BITS == 64
+#define MEM_SUFFIX q_raw
 #endif
  
 #elif MEM_WRITE == 1
@@ -38,6 +40,8 @@
 #define MEM_SUFFIX w_kernel
 #elif DATA_BITS == 32
 #define MEM_SUFFIX l_kernel
+#elif DATA_BITS == 64
+#define MEM_SUFFIX q_kernel
 #endif
  
 #elif MEM_WRITE == 2
@@ -48,6 +52,8 @@
 #define MEM_SUFFIX w_user
 #elif DATA_BITS == 32
 #define MEM_SUFFIX l_user
+#elif DATA_BITS == 64
+#define MEM_SUFFIX q_user
 #endif
  
 #else
@@ -64,14 +70,16 @@
  
 void OPPROTO glue(glue(op_rol, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
+    int count;
+    target_long src;
+
     count = T1 & SHIFT_MASK;
     if (count) {
         src = T0;
         T0 &= DATA_MASK;
         T0 = (T0 << count) | (T0 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #else
         /* gcc 3.2 workaround. This is really a bug in gcc. */
         asm volatile("" : : "r" (T0));
@@ -86,14 +94,16 @@ void OPPROTO glue(glue(op_rol, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_ror, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
+    int count;
+    target_long src;
+
     count = T1 & SHIFT_MASK;
     if (count) {
         src = T0;
         T0 &= DATA_MASK;
         T0 = (T0 >> count) | (T0 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #else
         /* gcc 3.2 workaround. This is really a bug in gcc. */
         asm volatile("" : : "r" (T0));
@@ -114,7 +124,7 @@ void OPPROTO glue(glue(op_rol, MEM_SUFFIX), _T0_T1)(void)
         T0 &= DATA_MASK;
         T0 = (T0 << count) | (T0 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     }
     FORCE_RET();
@@ -128,7 +138,7 @@ void OPPROTO glue(glue(op_ror, MEM_SUFFIX), _T0_T1)(void)
         T0 &= DATA_MASK;
         T0 = (T0 >> count) | (T0 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     }
     FORCE_RET();
@@ -136,10 +146,11 @@ void OPPROTO glue(glue(op_ror, MEM_SUFFIX), _T0_T1)(void)
  
 void OPPROTO glue(glue(op_rcl, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, res, eflags;
-    unsigned int src;
+    int count, eflags;
+    target_ulong src;
+    target_long res;
  
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
 #if DATA_BITS == 16
     count = rclw_table[count];
 #elif DATA_BITS == 8
@@ -154,7 +165,7 @@ void OPPROTO glue(glue(op_rcl, MEM_SUFFIX), _T0_T1_cc)(void)
             res |= T0 >> (DATA_BITS + 1 - count);
         T0 = res;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = (eflags & ~(CC_C | CC_O)) |
             (lshift(src ^ T0, 11 - (DATA_BITS - 1)) & CC_O) | 
@@ -166,10 +177,11 @@ void OPPROTO glue(glue(op_rcl, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_rcr, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, res, eflags;
-    unsigned int src;
+    int count, eflags;
+    target_ulong src;
+    target_long res;
  
-    count = T1 & 0x1f;
+    count = T1 & SHIFT1_MASK;
 #if DATA_BITS == 16
     count = rclw_table[count];
 #elif DATA_BITS == 8
@@ -184,7 +196,7 @@ void OPPROTO glue(glue(op_rcr, MEM_SUFFIX), _T0_T1_cc)(void)
             res |= T0 << (DATA_BITS + 1 - count);
         T0 = res;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = (eflags & ~(CC_C | CC_O)) |
             (lshift(src ^ T0, 11 - (DATA_BITS - 1)) & CC_O) | 
@@ -196,13 +208,15 @@ void OPPROTO glue(glue(op_rcr, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_shl, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     if (count) {
         src = (DATA_TYPE)T0 << (count - 1);
         T0 = T0 << count;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = src;
         CC_DST = T0;
@@ -213,14 +227,16 @@ void OPPROTO glue(glue(op_shl, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_shr, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     if (count) {
         T0 &= DATA_MASK;
         src = T0 >> (count - 1);
         T0 = T0 >> count;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = src;
         CC_DST = T0;
@@ -231,14 +247,16 @@ void OPPROTO glue(glue(op_shr, MEM_SUFFIX), _T0_T1_cc)(void)
  
 void OPPROTO glue(glue(op_sar, MEM_SUFFIX), _T0_T1_cc)(void)
 {
-    int count, src;
-    count = T1 & 0x1f;
+    int count;
+    target_long src;
+
+    count = T1 & SHIFT1_MASK;
     if (count) {
         src = (DATA_STYPE)T0;
         T0 = src >> count;
         src = src >> (count - 1);
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = src;
         CC_DST = T0;
@@ -262,7 +280,7 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_im_cc)(void)
         res |= T1 << (count - 16);
     T0 = res >> 16;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -282,7 +300,7 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
           res |= T1 << (count - 16);
         T0 = res >> 16;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -304,7 +322,7 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_im_cc)(void)
         res |= T1 << (32 - count);
     T0 = res;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -325,7 +343,7 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
             res |= T1 << (32 - count);
         T0 = res;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -335,17 +353,19 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
 }
 #endif
  
-#if DATA_BITS == 32
+#if DATA_BITS >= 32
 void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_im_cc)(void)
 {
-    int count, tmp;
+    int count;
+    target_long tmp;
+
     count = PARAM1;
     T0 &= DATA_MASK;
     T1 &= DATA_MASK;
     tmp = T0 << (count - 1);
     T0 = (T0 << count) | (T1 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -353,15 +373,17 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_im_cc)(void)
  
 void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
 {
-    int count, tmp;
-    count = ECX & 0x1f;
+    int count;
+    target_long tmp;
+
+    count = ECX & SHIFT1_MASK;
     if (count) {
         T0 &= DATA_MASK;
         T1 &= DATA_MASK;
         tmp = T0 << (count - 1);
         T0 = (T0 << count) | (T1 >> (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -372,14 +394,16 @@ void OPPROTO glue(glue(op_shld, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
  
 void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_im_cc)(void)
 {
-    int count, tmp;
+    int count;
+    target_long tmp;
+
     count = PARAM1;
     T0 &= DATA_MASK;
     T1 &= DATA_MASK;
     tmp = T0 >> (count - 1);
     T0 = (T0 >> count) | (T1 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = tmp;
     CC_DST = T0;
@@ -388,15 +412,17 @@ void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_im_cc)(void)
  
 void OPPROTO glue(glue(op_shrd, MEM_SUFFIX), _T0_T1_ECX_cc)(void)
 {
-    int count, tmp;
-    count = ECX & 0x1f;
+    int count;
+    target_long tmp;
+
+    count = ECX & SHIFT1_MASK;
     if (count) {
         T0 &= DATA_MASK;
         T1 &= DATA_MASK;
         tmp = T0 >> (count - 1);
         T0 = (T0 >> count) | (T1 << (DATA_BITS - count));
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
         CC_SRC = tmp;
         CC_DST = T0;
@@ -414,11 +440,11 @@ void OPPROTO glue(glue(op_adc, MEM_SUFFIX), _T0_T1_cc)(void)
     cf = cc_table[CC_OP].compute_c();
     T0 = T0 + T1 + cf;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = T1;
     CC_DST = T0;
-    CC_OP = CC_OP_ADDB + SHIFT + cf * 3;
+    CC_OP = CC_OP_ADDB + SHIFT + cf * 4;
 }
  
 void OPPROTO glue(glue(op_sbb, MEM_SUFFIX), _T0_T1_cc)(void)
@@ -427,23 +453,23 @@ void OPPROTO glue(glue(op_sbb, MEM_SUFFIX), _T0_T1_cc)(void)
     cf = cc_table[CC_OP].compute_c();
     T0 = T0 - T1 - cf;
 #ifdef MEM_WRITE
-    glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+    glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     CC_SRC = T1;
     CC_DST = T0;
-    CC_OP = CC_OP_SUBB + SHIFT + cf * 3;
+    CC_OP = CC_OP_SUBB + SHIFT + cf * 4;
 }
  
 void OPPROTO glue(glue(op_cmpxchg, MEM_SUFFIX), _T0_T1_EAX_cc)(void)
 {
-    unsigned int src, dst;
+    target_ulong src, dst;
  
     src = T0;
     dst = EAX - T0;
     if ((DATA_TYPE)dst == 0) {
         T0 = T1;
 #ifdef MEM_WRITE
-        glue(st, MEM_SUFFIX)((uint8_t *)A0, T0);
+        glue(st, MEM_SUFFIX)(A0, T0);
 #endif
     } else {
         EAX = (EAX & ~DATA_MASK) | (T0 & DATA_MASK);