make the TB cache independent of MMU mappings (faster MMU context switches and n…

…eeded for SMP support) git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@1632 c046a42c-6fe2-441c-8c8c-71466251a162

make the TB cache independent of MMU mappings (faster MMU context switches and n…
…eeded for SMP support) git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@1632 c046a42c-6fe2-441c-8c8c-71466251a162
bellard
1 parent 313adae9
Showing 3 changed files with 206 additions and 426 deletions
cpu-exec.c
exec-all.h
exec.c
@@ -73,6 +73,137 @@ void cpu_resume_from_signal(CPUState *env1, void *puc)
     longjmp(env->jmp_env, 1);
 }
+
+static TranslationBlock *tb_find_slow(target_ulong pc,
+                                      target_ulong cs_base,
+                                      unsigned int flags)
+{
+    TranslationBlock *tb, **ptb1;
+    int code_gen_size;
+    unsigned int h;
+    target_ulong phys_pc, phys_page1, phys_page2, virt_page2;
+    uint8_t *tc_ptr;
+    
+    spin_lock(&tb_lock);
+
+    tb_invalidated_flag = 0;
+    
+    regs_to_env(); /* XXX: do it just before cpu_gen_code() */
+    
+    /* find translated block using physical mappings */
+    phys_pc = get_phys_addr_code(env, pc);
+    phys_page1 = phys_pc & TARGET_PAGE_MASK;
+    phys_page2 = -1;
+    h = tb_phys_hash_func(phys_pc);
+    ptb1 = &tb_phys_hash[h];
+    for(;;) {
+        tb = *ptb1;
+        if (!tb)
+            goto not_found;
+        if (tb->pc == pc && 
+            tb->page_addr[0] == phys_page1 &&
+            tb->cs_base == cs_base && 
+            tb->flags == flags) {
+            /* check next page if needed */
+            if (tb->page_addr[1] != -1) {
+                virt_page2 = (pc & TARGET_PAGE_MASK) + 
+                    TARGET_PAGE_SIZE;
+                phys_page2 = get_phys_addr_code(env, virt_page2);
+                if (tb->page_addr[1] == phys_page2)
+                    goto found;
+            } else {
+                goto found;
+            }
+        }
+        ptb1 = &tb->phys_hash_next;
+    }
+ not_found:
+    /* if no translated code available, then translate it now */
+    tb = tb_alloc(pc);
+    if (!tb) {
+        /* flush must be done */
+        tb_flush(env);
+        /* cannot fail at this point */
+        tb = tb_alloc(pc);
+        /* don't forget to invalidate previous TB info */
+        T0 = 0;
+    }
+    tc_ptr = code_gen_ptr;
+    tb->tc_ptr = tc_ptr;
+    tb->cs_base = cs_base;
+    tb->flags = flags;
+    cpu_gen_code(env, tb, CODE_GEN_MAX_SIZE, &code_gen_size);
+    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+    
+    /* check next page if needed */
+    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
+    phys_page2 = -1;
+    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
+        phys_page2 = get_phys_addr_code(env, virt_page2);
+    }
+    tb_link_phys(tb, phys_pc, phys_page2);
+    
+ found:
+    if (tb_invalidated_flag) {
+        /* as some TB could have been invalidated because
+           of memory exceptions while generating the code, we
+           must recompute the hash index here */
+        T0 = 0;
+    }
+    /* we add the TB in the virtual pc hash table */
+    env->tb_jmp_cache[tb_jmp_cache_hash_func(pc)] = tb;
+    spin_unlock(&tb_lock);
+    return tb;
+}
+
+static inline TranslationBlock *tb_find_fast(void)
+{
+    TranslationBlock *tb;
+    target_ulong cs_base, pc;
+    unsigned int flags;
+
+    /* we record a subset of the CPU state. It will
+       always be the same before a given translated block
+       is executed. */
+#if defined(TARGET_I386)
+    flags = env->hflags;
+    flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
+    cs_base = env->segs[R_CS].base;
+    pc = cs_base + env->eip;
+#elif defined(TARGET_ARM)
+    flags = env->thumb | (env->vfp.vec_len << 1)
+        | (env->vfp.vec_stride << 4);
+    cs_base = 0;
+    pc = env->regs[15];
+#elif defined(TARGET_SPARC)
+#ifdef TARGET_SPARC64
+    flags = (env->pstate << 2) | ((env->lsu & (DMMU_E | IMMU_E)) >> 2);
+#else
+    flags = env->psrs | ((env->mmuregs[0] & (MMU_E | MMU_NF)) << 1);
+#endif
+    cs_base = env->npc;
+    pc = env->pc;
+#elif defined(TARGET_PPC)
+    flags = (msr_pr << MSR_PR) | (msr_fp << MSR_FP) |
+        (msr_se << MSR_SE) | (msr_le << MSR_LE);
+    cs_base = 0;
+    pc = env->nip;
+#elif defined(TARGET_MIPS)
+    flags = env->hflags & MIPS_HFLAGS_TMASK;
+    cs_base = NULL;
+    pc = env->PC;
+#else
+#error unsupported CPU
+#endif
+    tb = env->tb_jmp_cache[tb_jmp_cache_hash_func(pc)];
+    if (__builtin_expect(!tb || tb->pc != pc || tb->cs_base != cs_base ||
+                         tb->flags != flags, 0)) {
+        tb = tb_find_slow(pc, cs_base, flags);
+    }
+    return tb;
+}
+
+
 /* main execution loop */
 int cpu_exec(CPUState *env1)
@@ -115,12 +246,10 @@ int cpu_exec(CPUState *env1)
 #ifdef __sparc__
     int saved_i7, tmp_T0;
 #endif
-    int code_gen_size, ret, interrupt_request;
+    int ret, interrupt_request;
     void (*gen_func)(void);
-    TranslationBlock *tb, **ptb;
-    target_ulong cs_base, pc;
+    TranslationBlock *tb;
     uint8_t *tc_ptr;
-    unsigned int flags;
     /* first we save global registers */
     saved_env = env;
@@ -290,19 +419,29 @@ int cpu_exec(CPUState *env1)
                     }
 #endif
                     if (msr_ee != 0) {
-                    if ((interrupt_request & CPU_INTERRUPT_HARD)) {
+                        if ((interrupt_request & CPU_INTERRUPT_HARD)) {
 			    /* Raise it */
 			    env->exception_index = EXCP_EXTERNAL;
 			    env->error_code = 0;
                             do_interrupt(env);
-                        env->interrupt_request &= ~CPU_INTERRUPT_HARD;
-			} else if ((interrupt_request & CPU_INTERRUPT_TIMER)) {
-			    /* Raise it */
-			    env->exception_index = EXCP_DECR;
-			    env->error_code = 0;
-			    do_interrupt(env);
+                            env->interrupt_request &= ~CPU_INTERRUPT_HARD;
+#ifdef __sparc__
+                            tmp_T0 = 0;
+#else
+                            T0 = 0;
+#endif
+                        } else if ((interrupt_request & CPU_INTERRUPT_TIMER)) {
+                            /* Raise it */
+                            env->exception_index = EXCP_DECR;
+                            env->error_code = 0;
+                            do_interrupt(env);
                             env->interrupt_request &= ~CPU_INTERRUPT_TIMER;
-			}
+#ifdef __sparc__
+                            tmp_T0 = 0;
+#else
+                            T0 = 0;
+#endif
+                        }
                     }
 #elif defined(TARGET_MIPS)
                     if ((interrupt_request & CPU_INTERRUPT_HARD) &&
@@ -316,6 +455,11 @@ int cpu_exec(CPUState *env1)
                         env->error_code = 0;
                         do_interrupt(env);
                         env->interrupt_request &= ~CPU_INTERRUPT_HARD;
+#ifdef __sparc__
+                        tmp_T0 = 0;
+#else
+                        T0 = 0;
+#endif
                     }
 #elif defined(TARGET_SPARC)
                     if ((interrupt_request & CPU_INTERRUPT_HARD) &&
@@ -329,6 +473,11 @@ int cpu_exec(CPUState *env1)
 			    env->interrupt_request &= ~CPU_INTERRUPT_HARD;
 			    do_interrupt(env->interrupt_index);
 			    env->interrupt_index = 0;
+#ifdef __sparc__
+                            tmp_T0 = 0;
+#else
+                            T0 = 0;
+#endif
 			}
 		    } else if (interrupt_request & CPU_INTERRUPT_TIMER) {
 			//do_interrupt(0, 0, 0, 0, 0);
@@ -399,123 +548,7 @@ int cpu_exec(CPUState *env1)
 #endif
                 }
 #endif
-                /* we record a subset of the CPU state. It will
-                   always be the same before a given translated block
-                   is executed. */
-#if defined(TARGET_I386)
-                flags = env->hflags;
-                flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
-                cs_base = env->segs[R_CS].base;
-                pc = cs_base + env->eip;
-#elif defined(TARGET_ARM)
-                flags = env->thumb | (env->vfp.vec_len << 1)
-                        | (env->vfp.vec_stride << 4);
-                cs_base = 0;
-                pc = env->regs[15];
-#elif defined(TARGET_SPARC)
-#ifdef TARGET_SPARC64
-                flags = (env->pstate << 2) | ((env->lsu & (DMMU_E | IMMU_E)) >> 2);
-#else
-                flags = env->psrs | ((env->mmuregs[0] & (MMU_E | MMU_NF)) << 1);
-#endif
-                cs_base = env->npc;
-                pc = env->pc;
-#elif defined(TARGET_PPC)
-                flags = (msr_pr << MSR_PR) | (msr_fp << MSR_FP) |
-                    (msr_se << MSR_SE) | (msr_le << MSR_LE);
-                cs_base = 0;
-                pc = env->nip;
-#elif defined(TARGET_MIPS)
-                flags = env->hflags & MIPS_HFLAGS_TMASK;
-                cs_base = NULL;
-                pc = env->PC;
-#else
-#error unsupported CPU
-#endif
-                tb = tb_find(&ptb, pc, cs_base, 
-                             flags);
-                if (!tb) {
-                    TranslationBlock **ptb1;
-                    unsigned int h;
-                    target_ulong phys_pc, phys_page1, phys_page2, virt_page2;
-                    
-                    
-                    spin_lock(&tb_lock);
-
-                    tb_invalidated_flag = 0;
-                    
-                    regs_to_env(); /* XXX: do it just before cpu_gen_code() */
-
-                    /* find translated block using physical mappings */
-                    phys_pc = get_phys_addr_code(env, pc);
-                    phys_page1 = phys_pc & TARGET_PAGE_MASK;
-                    phys_page2 = -1;
-                    h = tb_phys_hash_func(phys_pc);
-                    ptb1 = &tb_phys_hash[h];
-                    for(;;) {
-                        tb = *ptb1;
-                        if (!tb)
-                            goto not_found;
-                        if (tb->pc == pc && 
-                            tb->page_addr[0] == phys_page1 &&
-                            tb->cs_base == cs_base && 
-                            tb->flags == flags) {
-                            /* check next page if needed */
-                            if (tb->page_addr[1] != -1) {
-                                virt_page2 = (pc & TARGET_PAGE_MASK) + 
-                                    TARGET_PAGE_SIZE;
-                                phys_page2 = get_phys_addr_code(env, virt_page2);
-                                if (tb->page_addr[1] == phys_page2)
-                                    goto found;
-                            } else {
-                                goto found;
-                            }
-                        }
-                        ptb1 = &tb->phys_hash_next;
-                    }
-                not_found:
-                    /* if no translated code available, then translate it now */
-                    tb = tb_alloc(pc);
-                    if (!tb) {
-                        /* flush must be done */
-                        tb_flush(env);
-                        /* cannot fail at this point */
-                        tb = tb_alloc(pc);
-                        /* don't forget to invalidate previous TB info */
-                        ptb = &tb_hash[tb_hash_func(pc)];
-                        T0 = 0;
-                    }
-                    tc_ptr = code_gen_ptr;
-                    tb->tc_ptr = tc_ptr;
-                    tb->cs_base = cs_base;
-                    tb->flags = flags;
-                    cpu_gen_code(env, tb, CODE_GEN_MAX_SIZE, &code_gen_size);
-                    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
-                    
-                    /* check next page if needed */
-                    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
-                    phys_page2 = -1;
-                    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
-                        phys_page2 = get_phys_addr_code(env, virt_page2);
-                    }
-                    tb_link_phys(tb, phys_pc, phys_page2);
-
-                found:
-                    if (tb_invalidated_flag) {
-                        /* as some TB could have been invalidated because
-                           of memory exceptions while generating the code, we
-                           must recompute the hash index here */
-                        ptb = &tb_hash[tb_hash_func(pc)];
-                        while (*ptb != NULL)
-                            ptb = &(*ptb)->hash_next;
-                        T0 = 0;
-                    }
-                    /* we add the TB in the virtual pc hash table */
-                    *ptb = tb;
-                    tb->hash_next = NULL;
-                    tb_link(tb);
-                    spin_unlock(&tb_lock);
-                }
+                tb = tb_find_fast();
 #ifdef DEBUG_EXEC
                 if ((loglevel & CPU_LOG_EXEC)) {
                     fprintf(logfile, "Trace 0x%08lx [" TARGET_FMT_lx "] %s\n",
@@ -526,9 +559,12 @@ int cpu_exec(CPUState *env1)
 #ifdef __sparc__
                 T0 = tmp_T0;
 #endif	    
-                /* see if we can patch the calling TB. */
+                /* see if we can patch the calling TB. When the TB
+                   spans two pages, we cannot safely do a direct
+                   jump. */
                 {
-                    if (T0 != 0
+                    if (T0 != 0 &&
+                        tb->page_addr[1] == -1
 #if defined(TARGET_I386) && defined(USE_CODE_COPY)
                     && (tb->cflags & CF_CODE_COPY) == 
                     (((TranslationBlock *)(T0 & ~3))->cflags & CF_CODE_COPY)
@@ -105,9 +105,6 @@ int tlb_set_page(CPUState *env, target_ulong vaddr,
 #define CODE_GEN_MAX_SIZE        65536
 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
-#define CODE_GEN_HASH_BITS     15
-#define CODE_GEN_HASH_SIZE     (1 << CODE_GEN_HASH_BITS)
-
 #define CODE_GEN_PHYS_HASH_BITS     15
 #define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
@@ -167,7 +164,6 @@ typedef struct TranslationBlock {
 #define CF_SINGLE_INSN 0x0008 /* compile only a single instruction */
     uint8_t *tc_ptr;    /* pointer to the translated code */
-    struct TranslationBlock *hash_next; /* next matching tb for virtual address */
     /* next matching tb for physical address. */
     struct TranslationBlock *phys_hash_next; 
     /* first and second physical page containing code. The lower bit
@@ -191,9 +187,9 @@ typedef struct TranslationBlock {
     struct TranslationBlock *jmp_first;
 } TranslationBlock;
-static inline unsigned int tb_hash_func(target_ulong pc)
+static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
 {
-    return pc & (CODE_GEN_HASH_SIZE - 1);
+    return (pc ^ (pc >> TB_JMP_CACHE_BITS)) & (TB_JMP_CACHE_SIZE - 1);
 }
 static inline unsigned int tb_phys_hash_func(unsigned long pc)
@@ -203,41 +199,14 @@ static inline unsigned int tb_phys_hash_func(unsigned long pc)
 TranslationBlock *tb_alloc(target_ulong pc);
 void tb_flush(CPUState *env);
-void tb_link(TranslationBlock *tb);
 void tb_link_phys(TranslationBlock *tb, 
                   target_ulong phys_pc, target_ulong phys_page2);
-extern TranslationBlock *tb_hash[CODE_GEN_HASH_SIZE];
 extern TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
 extern uint8_t code_gen_buffer[CODE_GEN_BUFFER_SIZE];
 extern uint8_t *code_gen_ptr;
-/* find a translation block in the translation cache. If not found,
-   return NULL and the pointer to the last element of the list in pptb */
-static inline TranslationBlock *tb_find(TranslationBlock ***pptb,
-                                        target_ulong pc, 
-                                        target_ulong cs_base,
-                                        unsigned int flags)
-{
-    TranslationBlock **ptb, *tb;
-    unsigned int h;
- 
-    h = tb_hash_func(pc);
-    ptb = &tb_hash[h];
-    for(;;) {
-        tb = *ptb;
-        if (!tb)
-            break;
-        if (tb->pc == pc && tb->cs_base == cs_base && tb->flags == flags)
-            return tb;
-        ptb = &tb->hash_next;
-    }
-    *pptb = ptb;
-    return NULL;
-}
-
-
 #if defined(USE_DIRECT_JUMP)
 #if defined(__powerpc__)
@@ -61,7 +61,6 @@
 #endif
 TranslationBlock tbs[CODE_GEN_MAX_BLOCKS];
-TranslationBlock *tb_hash[CODE_GEN_HASH_SIZE];
 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
 int nb_tbs;
 /* any access to the tbs or the page table must use this lock */
@@ -92,20 +91,6 @@ typedef struct PhysPageDesc {
     uint32_t phys_offset;
 } PhysPageDesc;
-/* Note: the VirtPage handling is absolete and will be suppressed
-   ASAP */
-typedef struct VirtPageDesc {
-    /* physical address of code page. It is valid only if 'valid_tag'
-       matches 'virt_valid_tag' */ 
-    target_ulong phys_addr; 
-    unsigned int valid_tag;
-#if !defined(CONFIG_SOFTMMU)
-    /* original page access rights. It is valid only if 'valid_tag'
-       matches 'virt_valid_tag' */
-    unsigned int prot;
-#endif
-} VirtPageDesc;
-
 #define L2_BITS 10
 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
@@ -123,17 +108,6 @@ unsigned long qemu_host_page_mask;
 static PageDesc *l1_map[L1_SIZE];
 PhysPageDesc **l1_phys_map;
-#if !defined(CONFIG_USER_ONLY)
-#if TARGET_LONG_BITS > 32
-#define VIRT_L_BITS 9
-#define VIRT_L_SIZE (1 << VIRT_L_BITS)
-static void *l1_virt_map[VIRT_L_SIZE];
-#else
-static VirtPageDesc *l1_virt_map[L1_SIZE];
-#endif
-static unsigned int virt_valid_tag;
-#endif
-
 /* io memory support */
 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
@@ -190,9 +164,6 @@ static void page_init(void)
     while ((1 << qemu_host_page_bits) < qemu_host_page_size)
         qemu_host_page_bits++;
     qemu_host_page_mask = ~(qemu_host_page_size - 1);
-#if !defined(CONFIG_USER_ONLY)
-    virt_valid_tag = 1;
-#endif
     l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
     memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
 }
@@ -266,120 +237,6 @@ static void tlb_protect_code(CPUState *env, ram_addr_t ram_addr,
                              target_ulong vaddr);
 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr, 
                                     target_ulong vaddr);
-
-static VirtPageDesc *virt_page_find_alloc(target_ulong index, int alloc)
-{
-#if TARGET_LONG_BITS > 32
-    void **p, **lp;
-
-    p = l1_virt_map;
-    lp = p + ((index >> (5 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));
-    p = *lp;
-    if (!p) {
-        if (!alloc)
-            return NULL;
-        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);
-        *lp = p;
-    }
-    lp = p + ((index >> (4 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));
-    p = *lp;
-    if (!p) {
-        if (!alloc)
-            return NULL;
-        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);
-        *lp = p;
-    }
-    lp = p + ((index >> (3 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));
-    p = *lp;
-    if (!p) {
-        if (!alloc)
-            return NULL;
-        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);
-        *lp = p;
-    }
-    lp = p + ((index >> (2 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));
-    p = *lp;
-    if (!p) {
-        if (!alloc)
-            return NULL;
-        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);
-        *lp = p;
-    }
-    lp = p + ((index >> (1 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));
-    p = *lp;
-    if (!p) {
-        if (!alloc)
-            return NULL;
-        p = qemu_mallocz(sizeof(VirtPageDesc) * VIRT_L_SIZE);
-        *lp = p;
-    }
-    return ((VirtPageDesc *)p) + (index & (VIRT_L_SIZE - 1));
-#else
-    VirtPageDesc *p, **lp;
-
-    lp = &l1_virt_map[index >> L2_BITS];
-    p = *lp;
-    if (!p) {
-        /* allocate if not found */
-        if (!alloc)
-            return NULL;
-        p = qemu_mallocz(sizeof(VirtPageDesc) * L2_SIZE);
-        *lp = p;
-    }
-    return p + (index & (L2_SIZE - 1));
-#endif
-}
-
-static inline VirtPageDesc *virt_page_find(target_ulong index)
-{
-    return virt_page_find_alloc(index, 0);
-}
-
-#if TARGET_LONG_BITS > 32
-static void virt_page_flush_internal(void **p, int level)
-{
-    int i; 
-    if (level == 0) {
-        VirtPageDesc *q = (VirtPageDesc *)p;
-        for(i = 0; i < VIRT_L_SIZE; i++)
-            q[i].valid_tag = 0;
-    } else {
-        level--;
-        for(i = 0; i < VIRT_L_SIZE; i++) {
-            if (p[i])
-                virt_page_flush_internal(p[i], level);
-        }
-    }
-}
-#endif
-
-static void virt_page_flush(void)
-{
-    virt_valid_tag++;
-
-    if (virt_valid_tag == 0) {
-        virt_valid_tag = 1;
-#if TARGET_LONG_BITS > 32
-        virt_page_flush_internal(l1_virt_map, 5);
-#else
-        {
-            int i, j;
-            VirtPageDesc *p;
-            for(i = 0; i < L1_SIZE; i++) {
-                p = l1_virt_map[i];
-                if (p) {
-                    for(j = 0; j < L2_SIZE; j++)
-                        p[j].valid_tag = 0;
-                }
-            }
-        }
-#endif
-    }
-}
-#else
-static void virt_page_flush(void)
-{
-}
 #endif
 void cpu_exec_init(void)
@@ -429,8 +286,7 @@ void tb_flush(CPUState *env)
            nb_tbs > 0 ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0);
 #endif
     nb_tbs = 0;
-    memset (tb_hash, 0, CODE_GEN_HASH_SIZE * sizeof (void *));
-    virt_page_flush();
+    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
     memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
     page_flush_tb();
@@ -566,28 +422,36 @@ static inline void tb_reset_jump(TranslationBlock *tb, int n)
     tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
 }
-static inline void tb_invalidate(TranslationBlock *tb)
+static inline void tb_phys_invalidate(TranslationBlock *tb, unsigned int page_addr)
 {
+    PageDesc *p;
     unsigned int h, n1;
-    TranslationBlock *tb1, *tb2, **ptb;
+    target_ulong phys_pc;
+    TranslationBlock *tb1, *tb2;
+    /* remove the TB from the hash list */
+    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
+    h = tb_phys_hash_func(phys_pc);
+    tb_remove(&tb_phys_hash[h], tb, 
+              offsetof(TranslationBlock, phys_hash_next));
+
+    /* remove the TB from the page list */
+    if (tb->page_addr[0] != page_addr) {
+        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
+        tb_page_remove(&p->first_tb, tb);
+        invalidate_page_bitmap(p);
+    }
+    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
+        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
+        tb_page_remove(&p->first_tb, tb);
+        invalidate_page_bitmap(p);
+    }
+
     tb_invalidated_flag = 1;
     /* remove the TB from the hash list */
-    h = tb_hash_func(tb->pc);
-    ptb = &tb_hash[h];
-    for(;;) {
-        tb1 = *ptb;
-        /* NOTE: the TB is not necessarily linked in the hash. It
-           indicates that it is not currently used */
-        if (tb1 == NULL)
-            return;
-        if (tb1 == tb) {
-            *ptb = tb1->hash_next;
-            break;
-        }
-        ptb = &tb1->hash_next;
-    }
+    h = tb_jmp_cache_hash_func(tb->pc);
+    cpu_single_env->tb_jmp_cache[h] = NULL;
     /* suppress this TB from the two jump lists */
     tb_jmp_remove(tb, 0);
@@ -606,33 +470,7 @@ static inline void tb_invalidate(TranslationBlock *tb)
         tb1 = tb2;
     }
     tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
-}
-
-static inline void tb_phys_invalidate(TranslationBlock *tb, unsigned int page_addr)
-{
-    PageDesc *p;
-    unsigned int h;
-    target_ulong phys_pc;
-    
-    /* remove the TB from the hash list */
-    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
-    h = tb_phys_hash_func(phys_pc);
-    tb_remove(&tb_phys_hash[h], tb, 
-              offsetof(TranslationBlock, phys_hash_next));
-
-    /* remove the TB from the page list */
-    if (tb->page_addr[0] != page_addr) {
-        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
-        tb_page_remove(&p->first_tb, tb);
-        invalidate_page_bitmap(p);
-    }
-    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
-        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
-        tb_page_remove(&p->first_tb, tb);
-        invalidate_page_bitmap(p);
-    }
-    tb_invalidate(tb);
     tb_phys_invalidate_count++;
 }
@@ -1025,57 +863,6 @@ void tb_link_phys(TranslationBlock *tb,
         tb_alloc_page(tb, 1, phys_page2);
     else
         tb->page_addr[1] = -1;
-#ifdef DEBUG_TB_CHECK
-    tb_page_check();
-#endif
-}
-
-/* link the tb with the other TBs */
-void tb_link(TranslationBlock *tb)
-{
-#if !defined(CONFIG_USER_ONLY)
-    {
-        VirtPageDesc *vp;
-        target_ulong addr;
-        
-        /* save the code memory mappings (needed to invalidate the code) */
-        addr = tb->pc & TARGET_PAGE_MASK;
-        vp = virt_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
-#ifdef DEBUG_TLB_CHECK 
-        if (vp->valid_tag == virt_valid_tag &&
-            vp->phys_addr != tb->page_addr[0]) {
-            printf("Error tb addr=0x%x phys=0x%x vp->phys_addr=0x%x\n",
-                   addr, tb->page_addr[0], vp->phys_addr);
-        }
-#endif
-        vp->phys_addr = tb->page_addr[0];
-        if (vp->valid_tag != virt_valid_tag) {
-            vp->valid_tag = virt_valid_tag;
-#if !defined(CONFIG_SOFTMMU)
-            vp->prot = 0;
-#endif
-        }
-        
-        if (tb->page_addr[1] != -1) {
-            addr += TARGET_PAGE_SIZE;
-            vp = virt_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
-#ifdef DEBUG_TLB_CHECK 
-            if (vp->valid_tag == virt_valid_tag &&
-                vp->phys_addr != tb->page_addr[1]) { 
-                printf("Error tb addr=0x%x phys=0x%x vp->phys_addr=0x%x\n",
-                       addr, tb->page_addr[1], vp->phys_addr);
-            }
-#endif
-            vp->phys_addr = tb->page_addr[1];
-            if (vp->valid_tag != virt_valid_tag) {
-                vp->valid_tag = virt_valid_tag;
-#if !defined(CONFIG_SOFTMMU)
-                vp->prot = 0;
-#endif
-            }
-        }
-    }
-#endif
     tb->jmp_first = (TranslationBlock *)((long)tb | 2);
     tb->jmp_next[0] = NULL;
@@ -1091,6 +878,10 @@ void tb_link(TranslationBlock *tb)
         tb_reset_jump(tb, 0);
     if (tb->tb_next_offset[1] != 0xffff)
         tb_reset_jump(tb, 1);
+
+#ifdef DEBUG_TB_CHECK
+    tb_page_check();
+#endif
 }
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
@@ -1396,8 +1187,7 @@ void tlb_flush(CPUState *env, int flush_global)
         env->tlb_write[1][i].address = -1;
     }
-    virt_page_flush();
-    memset (tb_hash, 0, CODE_GEN_HASH_SIZE * sizeof (void *));
+    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
 #if !defined(CONFIG_SOFTMMU)
     munmap((void *)MMAP_AREA_START, MMAP_AREA_END - MMAP_AREA_START);
@@ -1419,9 +1209,7 @@ static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 void tlb_flush_page(CPUState *env, target_ulong addr)
 {
-    int i, n;
-    VirtPageDesc *vp;
-    PageDesc *p;
+    int i;
     TranslationBlock *tb;
 #if defined(DEBUG_TLB)
@@ -1438,26 +1226,13 @@ void tlb_flush_page(CPUState *env, target_ulong addr)
     tlb_flush_entry(&env->tlb_read[1][i], addr);
     tlb_flush_entry(&env->tlb_write[1][i], addr);
-    /* remove from the virtual pc hash table all the TB at this
-       virtual address */
-    
-    vp = virt_page_find(addr >> TARGET_PAGE_BITS);
-    if (vp && vp->valid_tag == virt_valid_tag) {
-        p = page_find(vp->phys_addr >> TARGET_PAGE_BITS);
-        if (p) {
-            /* we remove all the links to the TBs in this virtual page */
-            tb = p->first_tb;
-            while (tb != NULL) {
-                n = (long)tb & 3;
-                tb = (TranslationBlock *)((long)tb & ~3);
-                if ((tb->pc & TARGET_PAGE_MASK) == addr ||
-                    ((tb->pc + tb->size - 1) & TARGET_PAGE_MASK) == addr) {
-                    tb_invalidate(tb);
-                }
-                tb = tb->page_next[n];
-            }
+    for(i = 0; i < TB_JMP_CACHE_SIZE; i++) {
+        tb = env->tb_jmp_cache[i];
+        if (tb && 
+            ((tb->pc & TARGET_PAGE_MASK) == addr ||
+             ((tb->pc + tb->size - 1) & TARGET_PAGE_MASK) == addr)) {
+            env->tb_jmp_cache[i] = NULL;
         }
-        vp->valid_tag = 0;
     }
 #if !defined(CONFIG_SOFTMMU)