Ticket #6310: x86_64_kernel_sources.finished.patch

src/system/kernel/arch/x86_64/arch_elf.cpp

@@ +1 @@
+/*
+ * Copyright 2004-2010, Haiku Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT license.
+ *
+ * Copyright 2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+#include <KernelExport.h>
+
+#include <elf_priv.h>
+#include <arch/elf.h>
+
+
+//#define TRACE_ARCH_ELF
+#ifdef TRACE_ARCH_ELF
+#   define TRACE(x) dprintf x
+#else
+#   define TRACE(x) ;
+#endif
+
+
+#ifdef TRACE_ARCH_ELF
+static const char *kRelocations[] = {
+    "R_X86_64_NONE",
+    "R_X86_64_32",          /* add symbol value */
+    "R_X86_64_PC32",        /* add PC relative symbol value */
+    "R_X86_64_GOT32",       /* add PC relative GOT offset */
+    "R_X86_64_PLT32",       /* add PC relative PLT offset */
+    "R_X86_64_COPY",        /* copy data from shared object */
+    "R_X86_64_GLOB_DAT",        /* set GOT entry to data address */
+    "R_X86_64_JMP_SLOT",        /* set GOT entry to code address */
+    "R_X86_64_RELATIVE",        /* add load address of shared object */
+    "R_X86_64_GOTOFF",      /* add GOT relative symbol address */
+};
+#endif
+
+
+static bool
+is_in_image(struct elf_image_info *image, addr_t address)
+{
+    return (address >= image->text_region.start
+            && address < image->text_region.start + image->text_region.size)
+        || (address >= image->data_region.start
+            && address < image->data_region.start + image->data_region.size);
+}
+
+
+int arch_elf_relocate_rel(struct elf_image_info *image,
+    struct elf_image_info *resolveImage, struct Elf64_Rel *rel, int relLength)
+{
+    addr_t S;
+    addr_t A;
+    addr_t P;
+    addr_t finalAddress;
+    addr_t *resolveAddress;
+    int i;
+
+    S = A = P = 0;
+
+    for (i = 0; i * (int)sizeof(struct Elf64_Rel) < relLength; i++) {
+        TRACE(("looking at rel type %s, offset 0x%lx\n",
+            kRelocations[ELF64_R_TYPE(rel[i].r_info)], rel[i].r_offset));
+
+        // calc S
+        switch (ELF64_R_TYPE(rel[i].r_info)) {
+            case R_X86_64_64:
+            case R_X86_64_PC32:
+            case R_X86_64_GLOB_DAT:
+            case R_X86_64_JMP_SLOT:
+            case R_X86_64_GOTOFF:
+            {
+                struct Elf64_Sym *symbol;
+                status_t status;
+
+                symbol = SYMBOL64(image, ELF64_R_SYM(rel[i].r_info));
+/* TODO: Add resolove symbol for 64bit elf */
+//              status = elf_resolve_symbol(image, symbol, resolveImage, &S);
+
+                if (status < B_OK)
+                    return status;
+                TRACE(("S %p (%s)\n", (void *)S, SYMNAME(image, symbol)));
+            }
+        }
+        // calc A
+        switch (ELF64_R_TYPE(rel[i].r_info)) {
+            case R_X86_64_64:
+            case R_X86_64_PC32:
+            case R_X86_64_GOT32:
+            case R_X86_64_PLT32:
+            case R_X86_64_RELATIVE:
+            case R_X86_64_GOTOFF:
+                A = *(addr_t *)(image->text_region.delta + rel[i].r_offset);
+                TRACE(("A %p\n", (void *)A));
+                break;
+        }
+        // calc P
+        switch (ELF64_R_TYPE(rel[i].r_info)) {
+            case R_X86_64_PC32:
+            case R_X86_64_GOT32:
+            case R_X86_64_PLT32:
+                P = image->text_region.delta + rel[i].r_offset;
+                TRACE(("P %p\n", (void *)P));
+                break;
+        }
+
+        switch (ELF64_R_TYPE(rel[i].r_info)) {
+            case R_X86_64_NONE:
+                continue;
+            case R_X86_64_64:
+                finalAddress = S + A;
+                break;
+            case R_X86_64_PC32:
+                finalAddress = S + A - P;
+                break;
+            case R_X86_64_RELATIVE:
+                // B + A;
+                finalAddress = image->text_region.delta + A;
+                break;
+            case R_X86_64_JMP_SLOT:
+            case R_X86_64_GLOB_DAT:
+                finalAddress = S;
+                break;
+
+            default:
+                dprintf("arch_elf_relocate_rel: unhandled relocation type %d\n",
+                    ELF64_R_TYPE(rel[i].r_info));
+                return B_BAD_DATA;
+        }
+
+        resolveAddress = (addr_t *)(image->text_region.delta + rel[i].r_offset);
+
+        if (!is_in_image(image, (addr_t)resolveAddress)) {
+            dprintf("arch_elf_relocate_rel: invalid offset %#lx\n",
+                rel[i].r_offset);
+            return B_BAD_ADDRESS;
+        }
+
+        *resolveAddress = finalAddress;
+        TRACE(("-> offset %#lx = %#lx\n",
+            (image->text_region.delta + rel[i].r_offset), finalAddress));
+    }
+
+    return B_NO_ERROR;
+}
+
+
+int
+arch_elf_relocate_rela(struct elf_image_info *image,
+    struct elf_image_info *resolveImage, struct Elf64_Rela *rela, int relLength)
+{
+    addr_t S;
+    addr_t A;
+    addr_t P;
+    addr_t finalAddress;
+    addr_t *resolveAddress;
+    int i;
+
+    S = A = P = 0;
+
+    for (i = 0; i * (int)sizeof(struct Elf64_Rela) < relLength; i++) {
+        TRACE(("looking at rel type %s, offset 0x%lx\n",
+            kRelocations[ELF64_R_TYPE(rela[i].r_info)], rela[i].r_offset));
+
+        // calc S
+        switch (ELF64_R_TYPE(rela[i].r_info)) {
+            case R_X86_64_32:
+            case R_X86_64_PC32:
+            case R_X86_64_GLOB_DAT:
+            case R_X86_64_JMP_SLOT:
+            {
+                struct Elf64_Sym *symbol;
+                status_t status;
+
+                symbol = SYMBOL64(image, ELF64_R_SYM(rela[i].r_info));
+/* TODO: ADD RESOLVE SYMBOL FOR ELF64 */
+//              status = boot_elf64_resolve_symbol(image, symbol, &S);
+
+                if (status < B_OK)
+                    return status;
+                TRACE(("S %p (%s)\n", (void *)S, SYMNAME(image, symbol)));
+            }
+        }
+        // calc A
+        switch (ELF64_R_TYPE(rela[i].r_info)) {
+            case R_X86_64_32:
+            case R_X86_64_PC32:
+            case R_X86_64_GOT32:
+            case R_X86_64_PLT32:
+            case R_X86_64_RELATIVE:
+            case R_X86_64_GOTOFF:
+                A = *(addr_t *)(image->text_region.delta + rela[i].r_offset);
+                TRACE(("A %p\n", (void *)A));
+                break;
+        }
+        // calc P
+        switch (ELF64_R_TYPE(rela[i].r_info)) {
+            case R_X86_64_PC32:
+            case R_X86_64_GOT32:
+            case R_X86_64_PLT32:
+                P = image->text_region.delta + rela[i].r_offset;
+                TRACE(("P %p\n", (void *)P));
+                break;
+        }
+
+        switch (ELF64_R_TYPE(rela[i].r_info)) {
+            case R_X86_64_NONE:
+                continue;
+            case R_X86_64_32:
+                finalAddress = S + A;
+                break;
+            case R_X86_64_PC32:
+                finalAddress = S + A - P;
+                break;
+            case R_X86_64_RELATIVE:
+                // B + A;
+                finalAddress = image->text_region.delta + A;
+                break;
+            case R_X86_64_JMP_SLOT:
+            case R_X86_64_GLOB_DAT:
+                finalAddress = S;
+                break;
+
+            default:
+                dprintf("arch_elf_relocate_rel: unhandled relocation type %d\n",
+                    ELF64_R_TYPE(rela[i].r_info));
+                return B_BAD_DATA;
+        }
+
+        resolveAddress = (addr_t *)(image->text_region.delta + rela[i].r_offset);
+
+        *resolveAddress = finalAddress;
+        TRACE(("-> offset %#lx = %#lx\n",
+            (image->text_region.delta + rela[i].r_offset), finalAddress));
+    }
+
+    return B_NO_ERROR;
+}
+
+
+int
+arch_elf_relocate_rel(struct elf_image_info *image,
+    struct elf_image_info *resolveImage, struct Elf32_Rel *rel, int relLength)
+{
+    addr_t S;
+    addr_t A;
+    addr_t P;
+    addr_t finalAddress;
+    addr_t *resolveAddress;
+    int i;
+
+    S = A = P = 0;
+
+    for (i = 0; i * (int)sizeof(struct Elf32_Rel) < relLength; i++) {
+        TRACE(("looking at rel type %s, offset 0x%lx\n",
+            kRelocations[ELF32_R_TYPE(rel[i].r_info)], rel[i].r_offset));
+
+        // calc S
+        switch (ELF32_R_TYPE(rel[i].r_info)) {
+            case R_386_32:
+            case R_386_PC32:
+            case R_386_GLOB_DAT:
+            case R_386_JMP_SLOT:
+            case R_386_GOTOFF:
+            {
+                struct Elf32_Sym *symbol;
+                status_t status;
+
+                symbol = SYMBOL(image, ELF32_R_SYM(rel[i].r_info));
+
+                status = elf_resolve_symbol(image, symbol, resolveImage, &S);
+
+                if (status < B_OK)
+                    return status;
+                TRACE(("S %p (%s)\n", (void *)S, SYMNAME(image, symbol)));
+            }
+        }
+        // calc A
+        switch (ELF32_R_TYPE(rel[i].r_info)) {
+            case R_386_32:
+            case R_386_PC32:
+            case R_386_GOT32:
+            case R_386_PLT32:
+            case R_386_RELATIVE:
+            case R_386_GOTOFF:
+            case R_386_GOTPC:
+                A = *(addr_t *)(image->text_region.delta + rel[i].r_offset);
+                TRACE(("A %p\n", (void *)A));
+                break;
+        }
+        // calc P
+        switch (ELF32_R_TYPE(rel[i].r_info)) {
+            case R_386_PC32:
+            case R_386_GOT32:
+            case R_386_PLT32:
+            case R_386_GOTPC:
+                P = image->text_region.delta + rel[i].r_offset;
+                TRACE(("P %p\n", (void *)P));
+                break;
+        }
+
+        switch (ELF32_R_TYPE(rel[i].r_info)) {
+            case R_386_NONE:
+                continue;
+            case R_386_32:
+                finalAddress = S + A;
+                break;
+            case R_386_PC32:
+                finalAddress = S + A - P;
+                break;
+            case R_386_RELATIVE:
+                // B + A;
+                finalAddress = image->text_region.delta + A;
+                break;
+            case R_386_JMP_SLOT:
+            case R_386_GLOB_DAT:
+                finalAddress = S;
+                break;
+
+            default:
+                dprintf("arch_elf_relocate_rel: unhandled relocation type %d\n",
+                    ELF32_R_TYPE(rel[i].r_info));
+                return B_BAD_DATA;
+        }
+
+        resolveAddress = (addr_t *)(image->text_region.delta + rel[i].r_offset);
+
+        if (!is_in_image(image, (addr_t)resolveAddress)) {
+            dprintf("arch_elf_relocate_rel: invalid offset %#lx\n",
+                rel[i].r_offset);
+            return B_BAD_ADDRESS;
+        }
+
+        *resolveAddress = finalAddress;
+        TRACE(("-> offset %#lx = %#lx\n",
+            (image->text_region.delta + rel[i].r_offset), finalAddress));
+    }
+
+    return B_NO_ERROR;
+}
+
+
+
+
+int
+arch_elf_relocate_rela(struct elf_image_info *image,
+    struct elf_image_info *resolveImage, struct Elf32_Rela *rel, int relLength)
+{
+    dprintf("arch_elf_relocate_rela: not supported on x86\n");
+    return B_ERROR;
+}

src/system/kernel/arch/x86_64/arch_thread.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+
+#include <arch/thread.h>
+
+#include <string.h>
+
+#include <arch/user_debugger.h>
+#include <arch_cpu.h>
+#include <cpu.h>
+#include <debug.h>
+#include <kernel.h>
+#include <ksignal.h>
+#include <int.h>
+#include <team.h>
+#include <thread.h>
+#include <tls.h>
+#include <tracing.h>
+#include <vm/vm_types.h>
+#include <vm/VMAddressSpace.h>
+
+#include "paging/X86_64PagingStructures.h"
+#include "paging/X86_64VMTranslationMap.h"
+
+
+//#define TRACE_ARCH_THREAD
+#ifdef TRACE_ARCH_THREAD
+#   define TRACE(x) dprintf x
+#else
+#   define TRACE(x) ;
+#endif
+
+
+// from arch_interrupts.S
+extern "C" void x86_64_stack_init(struct farcall *interrupt_stack_offset);
+extern "C" void x86_64_restore_frame_from_syscall(struct iframe frame);
+
+// from arch_cpu.cpp
+extern void (*gX86SwapFPUFunc)(void *oldState, const void *newState);
+
+//from arch_commpage.cpp
+extern void x86_64_set_syscall_stack(addr_t stackTop);
+
+static struct arch_thread sInitialState _ALIGNED(16);
+    // the fpu_state must be aligned on a 16 byte boundary, so that fxsave can use it
+
+
+static inline void
+set_fs_register(uint32 segment)
+{
+    asm("mov %0,%%fs" :: "r" (segment));
+}
+
+
+static void
+set_tls_context(struct thread *thread)
+{
+    int entry = smp_get_current_cpu() + TLS_BASE_SEGMENT;
+
+    set_segment_descriptor_base(&gGDT[entry], thread->user_local_storage);
+    set_fs_register((entry << 3) | DPL_USER);
+}
+
+
+static struct iframe *
+find_previous_iframe(struct thread *thread, addr_t frame)
+{
+    // iterate backwards through the stack frames, until we hit an iframe
+    while (frame >= thread->kernel_stack_base
+        && frame < thread->kernel_stack_top) {
+        addr_t previousFrame = *(addr_t*)frame;
+        if ((previousFrame & ~IFRAME_TYPE_MASK) == 0) {
+            if (previousFrame == 0)
+                return NULL;
+            return (struct iframe*)frame;
+        }
+
+        frame = previousFrame;
+    }
+
+    return NULL;
+}
+
+static struct iframe*
+get_previous_iframe(struct iframe* frame)
+{
+    if (frame == NULL)
+        return NULL;
+
+    return find_previous_iframe(thread_get_current_thread(), frame->rbp);
+}
+
+static struct iframe*
+get_current_iframe(void)
+{
+    return find_previous_iframe(thread_get_current_thread(), x86_64_read_rbp());
+}
+
+struct iframe *
+x86_64_get_current_iframe(void)
+{
+    return get_current_iframe();
+}
+
+
+/*!
+    \brief Returns the current thread's topmost (i.e. most recent)
+    userland->kernel transition iframe (usually the first one, save for
+    interrupts in signal handlers).
+    \return The iframe, or \c NULL, if there is no such iframe (e.g. when
+            the thread is a kernel thread).
+*/
+struct iframe *
+x86_64_get_user_iframe(void)
+{
+    struct iframe* frame = get_current_iframe();
+
+    while (frame != NULL) {
+        if (IFRAME_IS_USER(frame))
+            return frame;
+        frame = get_previous_iframe(frame);
+    }
+
+    return NULL;
+}
+
+
+uint64
+x86_64_next_page_directory(struct thread *from, struct thread *to)
+{
+    VMAddressSpace* toAddressSpace = to->team->address_space;
+    if (from->team->address_space == toAddressSpace) {
+        // don't change the pgdir, same address space
+        return 0;
+    }
+
+    if (toAddressSpace == NULL)
+        toAddressSpace = VMAddressSpace::Kernel();
+
+    return static_cast<X86_64VMTranslationMap*>(toAddressSpace->TranslationMap())
+        ->PagingStructures()->pgdir_phys;
+}
+
+static uint64 *
+get_signal_stack(struct thread *thread, struct iframe *frame, int signal)
+{
+    // use the alternate signal stack if we should and can
+    if (thread->signal_stack_enabled
+        && (thread->sig_action[signal - 1].sa_flags & SA_ONSTACK) != 0
+        && (frame->user_rsp < thread->signal_stack_base
+            || frame->user_rsp >= thread->signal_stack_base
+                + thread->signal_stack_size)) {
+        return (uint64 *)(thread->signal_stack_base
+            + thread->signal_stack_size);
+    }
+
+    return (uint64 *)frame->user_rsp;
+}
+
+status_t
+arch_thread_init(struct kernel_args *args)
+{
+    // save one global valid FPU state; it will be copied in the arch dependent
+    // part of each new thread
+
+    asm volatile ("clts; fninit; fnclex;");
+
+    x86_64_fxsave(sInitialState.fpu_state);
+
+    return B_OK;
+}
+
+
+status_t
+arch_team_init_team_struct(struct team *team, bool kernel)
+{
+    return B_OK;
+}
+
+
+status_t
+arch_thread_init_thread_struct(struct thread *thread)
+{
+    // set up an initial state (stack & fpu)
+    memcpy(&thread->arch_info, &sInitialState, sizeof(struct arch_thread));
+    return B_OK;
+}
+
+
+status_t
+arch_thread_init_kthread_stack(struct thread *t, int (*start_func)(void),
+    void (*entry_func)(void), void (*exit_func)(void))
+{
+    addr_t *kstack = (addr_t *)t->kernel_stack_base;
+    addr_t *kstack_top = (addr_t *)t->kernel_stack_top;
+    int i;
+
+    TRACE(("arch_thread_initialize_kthread_stack: kstack 0x%p, start_func 0x%p, entry_func 0x%p\n",
+        kstack, start_func, entry_func));
+
+    // clear the kernel stack
+#ifdef DEBUG_KERNEL_STACKS
+#   ifdef STACK_GROWS_DOWNWARDS
+    memset((void *)((addr_t)kstack + KERNEL_STACK_GUARD_PAGES * B_PAGE_SIZE), 0,
+        KERNEL_STACK_SIZE);
+#   else
+    memset(kstack, 0, KERNEL_STACK_SIZE);
+#   endif
+#else
+    memset(kstack, 0, KERNEL_STACK_SIZE);
+#endif
+
+    // set the final return address to be thread_kthread_exit
+    kstack_top--;
+    *kstack_top = (addr_t)exit_func;
+
+    // set the return address to be the start of the first function
+    kstack_top--;
+    *kstack_top = (addr_t)start_func;
+
+    // set the return address to be the start of the entry (thread setup)
+    // function
+    kstack_top--;
+    *kstack_top = (addr_t)entry_func;
+
+    // simulate pushfl
+//  kstack_top--;
+//  *kstack_top = 0x00; // interrupts still disabled after the switch
+
+    // simulate initial popad
+    for (i = 0; i < 8; i++) {
+        kstack_top--;
+        *kstack_top = 0;
+    }
+
+    // save the stack position
+    t->arch_info.current_stack.rsp = kstack_top;
+    t->arch_info.current_stack.ss = (addr_t *)KERNEL_DATA_SEG;
+
+    return B_OK;
+}
+
+
+status_t
+arch_thread_init_tls(struct thread *thread)
+{
+    uint32 tls[TLS_USER_THREAD_SLOT + 1];
+
+    thread->user_local_storage = thread->user_stack_base
+        + thread->user_stack_size;
+
+    // initialize default TLS fields
+    memset(tls, 0, sizeof(tls));
+    tls[TLS_BASE_ADDRESS_SLOT] = thread->user_local_storage;
+    tls[TLS_THREAD_ID_SLOT] = thread->id;
+    tls[TLS_USER_THREAD_SLOT] = (addr_t)thread->user_thread;
+
+    return user_memcpy((void *)thread->user_local_storage, tls, sizeof(tls));
+}
+
+
+void
+arch_thread_context_switch(struct thread *from, struct thread *to)
+{
+    x86_64_set_tss_and_kstack(to->kernel_stack_top);
+    x86_64_set_syscall_stack(to->kernel_stack_top);
+
+    // set TLS GDT entry to the current thread - since this action is
+    // dependent on the current CPU, we have to do it here
+    if (to->user_local_storage != 0)
+        set_tls_context(to);
+
+    struct cpu_ent* cpuData = to->cpu;
+    X86_64PagingStructures* activePagingStructures
+        = cpuData->arch.active_paging_structures;
+    VMAddressSpace* toAddressSpace = to->team->address_space;
+
+    X86_64PagingStructures* toPagingStructures;
+    if (toAddressSpace != NULL
+        && (toPagingStructures = static_cast<X86_64VMTranslationMap*>(
+                toAddressSpace->TranslationMap())->PagingStructures())
+                    != activePagingStructures) {
+        // update on which CPUs the address space is used
+        int cpu = cpuData->cpu_num;
+        atomic_and(&activePagingStructures->active_on_cpus,
+            ~((uint32)1 << cpu));
+        atomic_or(&toPagingStructures->active_on_cpus, (uint32)1 << cpu);
+
+        // assign the new paging structures to the CPU
+        toPagingStructures->AddReference();
+        cpuData->arch.active_paging_structures = toPagingStructures;
+
+        // set the page directory, if it changes
+        uint32 newPageDirectory = toPagingStructures->pgdir_phys;
+        if (newPageDirectory != activePagingStructures->pgdir_phys)
+            x86_64_swap_pgdir(newPageDirectory);
+
+        // This CPU no longer uses the previous paging structures.
+        activePagingStructures->RemoveReference();
+    }
+
+    gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state);
+    x86_64_context_switch(&from->arch_info, &to->arch_info);
+}
+
+
+void
+arch_thread_dump_info(void *info)
+{
+    struct arch_thread *at = (struct arch_thread *)info;
+
+    kprintf("\trsp: %p\n", at->current_stack.rsp);
+    kprintf("\tss: %p\n", at->current_stack.ss);
+    kprintf("\tfpu_state at %p\n", at->fpu_state);
+}
+
+
+status_t
+arch_thread_enter_userspace(struct thread *thread, addr_t entry, void *arg1, void *arg2)
+{
+    addr_t stackTop = thread->user_stack_base + thread->user_stack_size;
+    uint64 codeSize = (addr_t)x86_64_end_userspace_thread_exit
+        - (addr_t)x86_64_userspace_thread_exit;
+    uint64 args[3];
+
+    TRACE(("arch_thread_enter_uspace: entry 0x%lx, args %p %p, ustack_top 0x%lx\n",
+        entry, args1, args2, stackTop));
+
+    // copy the little stub that calls exit_thread() when the thread entry
+    // function returns, as well as the arguments of the entry function
+    stackTop -= codeSize;
+
+    if (user_memcpy((void *)stackTop, (const void *)&x86_64_userspace_thread_exit, codeSize) < B_OK)
+        return B_BAD_ADDRESS;
+
+    args[0] = stackTop;
+    args[1] = (uint64)arg1;
+    args[2] = (uint64)arg2;
+    stackTop -= sizeof(args);
+
+    if (user_memcpy((void *)stackTop, args, sizeof(args)) < B_OK)
+        return B_BAD_ADDRESS;
+
+    thread_at_kernel_exit();
+        // also disables interrupts
+
+    // install user breakpoints, if any
+    if ((thread->flags & THREAD_FLAGS_BREAKPOINTS_DEFINED) != 0)
+        x86_64_init_user_debug_at_kernel_exit(NULL);
+
+    x86_64_set_tss_and_kstack(thread->kernel_stack_top);
+
+    // set the CPU dependent GDT entry for TLS
+    set_tls_context(thread);
+
+    x86_64_set_syscall_stack(thread->kernel_stack_top);
+    x86_64_enter_userspace(entry, stackTop);
+
+    return B_OK;
+}
+
+
+bool
+arch_on_signal_stack(struct thread *thread)
+{
+    struct iframe *frame = get_current_iframe();
+
+    return frame->user_rsp >= thread->signal_stack_base
+        && frame->user_rsp < thread->signal_stack_base
+            + thread->signal_stack_size;
+}
+
+
+status_t
+arch_setup_signal_frame(struct thread *thread, struct sigaction *action, int sig, int sigMask)
+{
+    struct iframe *frame = get_current_iframe();
+    if (!IFRAME_IS_USER(frame)) {
+        panic("arch_setup_signal_frame(): No user iframe!");
+        return B_BAD_VALUE;
+    }
+
+    uint64 *signalCode;
+    addr_t *userRegs;
+    struct vregs regs;
+    uint64 buffer[6];
+    status_t status;
+
+    // start stuffing stuff on the user stack
+    addr_t* userStack = get_signal_stack(thread, frame, sig);
+
+    // copy syscall restart info onto the user stack
+    userStack -= (sizeof(thread->syscall_restart.parameters) + 12 + 3) / 4;
+    uint32 threadFlags = atomic_and(&thread->flags,
+        ~(THREAD_FLAGS_RESTART_SYSCALL | THREAD_FLAGS_64_BIT_SYSCALL_RETURN));
+    if (user_memcpy(userStack, &threadFlags, 4) < B_OK
+        || user_memcpy(userStack + 1, &frame->orig_rax, 4) < B_OK
+        || user_memcpy(userStack + 2, &frame->orig_rdx, 4) < B_OK)
+        return B_BAD_ADDRESS;
+    status = user_memcpy(userStack + 3, thread->syscall_restart.parameters,
+        sizeof(thread->syscall_restart.parameters));
+    if (status < B_OK)
+        return status;
+
+    // store the saved regs onto the user stack
+    regs.rip = frame->rip;
+    regs.eflags = frame->flags;
+    regs.rax = frame->rax;
+    regs.rcx = frame->rcx;
+    regs.rdx = frame->rdx;
+    regs.rbp = frame->rbp;
+    regs.rsp = frame->rsp;
+    regs.r8 = frame->r8;
+    regs.r9 = frame->r9;
+    regs.r10 = frame->r10;
+    regs.r11 = frame->r11;
+    regs.r12 = frame->r12;
+    regs.r13 = frame->r13;
+    regs.r14 = frame->r14;
+    regs.r15 = frame->r15;
+    regs._reserved_1 = frame->user_rsp;
+    regs._reserved_2[0] = frame->rdi;
+    regs._reserved_2[1] = frame->rsi;
+    regs._reserved_2[2] = frame->rbx;
+    x86_64_fnsave((void *)(&regs.xregs));
+
+    userStack -= (sizeof(struct vregs) + 3) / 4;
+    userRegs = userStack;
+    status = user_memcpy(userRegs, &regs, sizeof(regs));
+    if (status < B_OK)
+        return status;
+
+    // now store a code snippet on the stack
+    userStack -= ((uint64)x86_64_end_return_from_signal + 3
+        - (uint64)x86_64_return_from_signal) / 4;
+    signalCode = userStack;
+    status = user_memcpy(signalCode, (const void *)&x86_64_return_from_signal,
+        ((uint64)x86_64_end_return_from_signal
+            - (uint64)x86_64_return_from_signal));
+    if (status < B_OK)
+        return status;
+
+    // now set up the final part
+    buffer[0] = (uint64)signalCode; // return address when sa_handler done
+    buffer[1] = (uint64)sig;                // arguments to sa_handler
+    buffer[2] = (uint64)action->sa_userdata;
+    buffer[3] = (uint64)userRegs;
+
+    buffer[4] = sigMask;            // Old signal mask to restore
+    buffer[5] = (uint64)userRegs;   // Int frame + extra regs to restore
+
+    userStack -= sizeof(buffer) / 4;
+
+    status = user_memcpy(userStack, buffer, sizeof(buffer));
+    if (status < B_OK)
+        return status;
+
+    frame->user_rsp = (uint64)userStack;
+    frame->rip = (uint64)action->sa_handler;
+
+    return B_OK;
+}
+
+
+int64
+arch_restore_signal_frame(void)
+{
+    struct thread *thread = thread_get_current_thread();
+    struct iframe *frame = get_current_iframe();
+    int32 signalMask;
+    uint64 *userStack;
+    struct vregs* regsPointer;
+    struct vregs regs;
+
+    TRACE(("### arch_restore_signal_frame: entry\n"));
+
+    userStack = (uint64 *)frame->user_rsp;
+    if (user_memcpy(&signalMask, &userStack[0], 4) < B_OK
+        || user_memcpy(&regsPointer, &userStack[1], 4) < B_OK
+        || user_memcpy(&regs, regsPointer, sizeof(vregs)) < B_OK) {
+        return B_BAD_ADDRESS;
+    }
+
+    uint64* syscallRestartInfo
+        = (uint64*)regsPointer + (sizeof(struct vregs) + 3) / 4;
+    uint32 threadFlags;
+    if (user_memcpy(&threadFlags, syscallRestartInfo, 4) < B_OK
+        || user_memcpy(&frame->orig_rax, syscallRestartInfo + 1, 4) < B_OK
+        || user_memcpy(&frame->orig_rdx, syscallRestartInfo + 2, 4) < B_OK
+        || user_memcpy(thread->syscall_restart.parameters,
+            syscallRestartInfo + 3,
+            sizeof(thread->syscall_restart.parameters)) < B_OK) {
+        return B_BAD_ADDRESS;
+    }
+
+    // set restart/64bit return value flags from previous syscall
+    atomic_and(&thread->flags,
+        ~(THREAD_FLAGS_RESTART_SYSCALL | THREAD_FLAGS_64_BIT_SYSCALL_RETURN));
+    atomic_or(&thread->flags, threadFlags
+        & (THREAD_FLAGS_RESTART_SYSCALL | THREAD_FLAGS_64_BIT_SYSCALL_RETURN));
+
+    // TODO: Verify that just restoring the old signal mask is right! Bash for
+    // instance changes the procmask in a signal handler. Those changes are
+    // lost the way we do it.
+    atomic_set(&thread->sig_block_mask, signalMask);
+    update_current_thread_signals_flag();
+
+    frame->rip = regs.rip;
+    frame->flags = regs.eflags;
+    frame->rax = regs.rax;
+    frame->rcx = regs.rcx;
+    frame->rdx = regs.rdx;
+    frame->rbp = regs.rbp;
+    frame->rsp = regs.rsp;
+    frame->r8  = regs.r8;
+    frame->r9  = regs.r9;
+    frame->r10 = regs.r10;
+    frame->r11 = regs.r11;
+    frame->r12 = regs.r12;
+    frame->r13 = regs.r13;
+    frame->r14 = regs.r14;
+    frame->r15 = regs.r15;
+    frame->user_rsp = regs._reserved_1;
+    frame->rdi = regs._reserved_2[0];
+    frame->rsi = regs._reserved_2[1];
+    frame->rbx = regs._reserved_2[2];
+
+    x86_64_frstor((void *)(&regs.xregs));
+
+    TRACE(("### arch_restore_signal_frame: exit\n"));
+
+    return (int64)frame->rax | ((int64)frame->rdx << 32);
+}
+
+
+void
+arch_store_fork_frame(struct arch_fork_arg *arg)
+{
+    struct iframe *frame = get_current_iframe();
+
+    // we need to copy the threads current iframe
+    arg->iframe = *frame;
+
+    // we also want fork() to return 0 for the child
+    arg->iframe.rax = 0;
+}
+
+void
+arch_restore_fork_frame(struct arch_fork_arg *arg)
+{
+    struct thread *thread = thread_get_current_thread();
+
+    disable_interrupts();
+
+    x86_64_set_tss_and_kstack(thread->kernel_stack_top);
+
+    // set the CPU dependent GDT entry for TLS (set the current %fs register)
+    set_tls_context(thread);
+
+    x86_64_restore_frame_from_syscall(arg->iframe);
+}
+
+void
+arch_syscall_64_bit_return_value(void)
+{
+    struct thread* thread = thread_get_current_thread();
+    atomic_or(&thread->flags, THREAD_FLAGS_64_BIT_SYSCALL_RETURN);
+}

src/system/kernel/arch/x86_64/arch_debug_console.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com
+ * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de
+ * Copyright 2001, Rob Judd <judd@ob-wan.com>
+ * Copyright 2002, Marcus Overhagen <marcus@overhagen.de>
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+#include "debugger_keymaps.h"
+
+#include <KernelExport.h>
+#include <driver_settings.h>
+#include <int.h>
+
+#include <arch/cpu.h>
+#include <arch/debug_console.h>
+#include <boot/stage2.h>
+#include <debug.h>
+
+#include <string.h>
+#include <stdlib.h>
+
+
+#define INT_PS2_KEYBOARD                0x01
+
+#define PS2_PORT_DATA                   0x60
+#define PS2_PORT_CTRL                   0x64
+
+#define PS2_STATUS_OUTPUT_BUFFER_FULL           0x01
+#define PS2_STATUS_AUX_DATA             0x20
+
+
+enum serial_register_offsets {
+    SERIAL_TRANSMIT_BUFFER          = 0,
+    SERIAL_RECEIVE_BUFFER           = 0,
+    SERIAL_DIVISOR_LATCH_LOW        = 0,
+    SERIAL_DIVISOR_LATCH_HIGH       = 1,
+    SERIAL_FIFO_CONTROL         = 2,
+    SERIAL_LINE_CONTROL         = 3,
+    SERIAL_MODEM_CONTROL            = 4,
+    SERIAL_LINE_STATUS          = 5,
+    SERIAL_MODEM_STATUS         = 6,
+};
+
+enum keycodes {
+    LEFT_SHIFT      = 42,
+    RIGHT_SHIFT     = 54,
+
+    LEFT_CONTROL        = 29,
+
+    LEFT_ALT        = 56,
+    RIGHT_ALT       = 58,
+
+    CURSOR_LEFT     = 75,
+    CURSOR_RIGHT        = 77,
+    CURSOR_UP       = 72,
+    CURSOR_DOWN     = 80,
+    CURSOR_HOME     = 71,
+    CURSOR_END      = 79,
+    PAGE_UP         = 73,
+    PAGE_DOWN       = 81,
+
+    DELETE          = 83,
+    SYS_REQ         = 84,
+    F12         = 88,
+};
+
+
+static const uint32 kSerialBaudRate = 115200;
+static uint16 sSerialBasePort = 0x3f8;
+    // COM1 is the default debug output port
+
+static bool sKeyboardHandlerInstalled = false;
+
+static spinlock sSerialOutputSpinlock = B_SPINLOCK_INITIALIZER;
+
+
+static void
+put_char(const char c)
+{
+    // wait until the transmitter empty bit is set
+    while ((in8(sSerialBasePort + SERIAL_LINE_STATUS) & 0x20) == 0)
+        asm volatile ("pause;");
+
+    out8(c, sSerialBasePort + SERIAL_TRANSMIT_BUFFER);
+}
+
+
+static void
+_arch_debug_serial_putchar(const char c)
+{
+    if (c == '\n') {
+        put_char('\r');
+        put_char('\n');
+    } else if (c != '\r')
+        put_char(c);
+}
+
+
+/** Minimal keyboard handler to be able to get into the debugger and
+ *  reboot the machine before the input_server is up and running.
+ *  It is added as soon as interrupts become available, and removed
+ *  again if anything else requests the interrupt 1.
+ */
+
+static int32
+debug_keyboard_interrupt(void *data)
+{
+    static bool controlPressed = false;
+    static bool altPressed = false;
+    static bool sysReqPressed = false;
+    uint8 key;
+
+    key = in8(PS2_PORT_DATA);
+    //dprintf("debug_keyboard_interrupt: key = 0x%x\n", key);
+
+    if (key & 0x80) {
+        if (key == LEFT_CONTROL)
+            controlPressed = false;
+        else if (key == LEFT_ALT)
+            altPressed = false;
+        else if (key == SYS_REQ)
+            sysReqPressed = false;
+
+        return B_HANDLED_INTERRUPT;
+    }
+
+    switch (key) {
+        case LEFT_CONTROL:
+            controlPressed = true;
+            break;
+
+        case LEFT_ALT:
+        case RIGHT_ALT:
+            altPressed = true;
+            break;
+
+        case SYS_REQ:
+            sysReqPressed = true;
+            break;
+
+        case DELETE:
+            if (controlPressed && altPressed)
+                arch_cpu_shutdown(true);
+            break;
+
+        default:
+            if (altPressed && sysReqPressed) {
+                if (debug_emergency_key_pressed(kUnshiftedKeymap[key])) {
+                    // we probably have lost some keys, so reset our key states
+                    controlPressed = false;
+                    sysReqPressed = false;
+                    altPressed = false;
+                }
+            }
+            break;
+    }
+
+    return B_HANDLED_INTERRUPT;
+}
+
+
+void
+arch_debug_remove_interrupt_handler(uint32 line)
+{
+    if (line != INT_PS2_KEYBOARD || !sKeyboardHandlerInstalled)
+        return;
+
+    remove_io_interrupt_handler(INT_PS2_KEYBOARD, &debug_keyboard_interrupt,
+        NULL);
+    sKeyboardHandlerInstalled = false;
+}
+
+
+void
+arch_debug_install_interrupt_handlers(void)
+{
+    install_io_interrupt_handler(INT_PS2_KEYBOARD, &debug_keyboard_interrupt,
+        NULL, 0);
+    sKeyboardHandlerInstalled = true;
+}
+
+
+char
+arch_debug_blue_screen_getchar(void)
+{
+
+    /* polling the keyboard, similar to code in keyboard
+     * driver, but without using an interrupt
+     */
+    static bool shiftPressed = false;
+    static bool controlPressed = false;
+    static bool altPressed = false;
+    static uint8 special = 0;
+    static uint8 special2 = 0;
+    uint8 key = 0;
+
+    if (special & 0x80) {
+        special &= ~0x80;
+        return '[';
+    }
+    if (special != 0) {
+        key = special;
+        special = 0;
+        return key;
+    }
+    if (special2 != 0) {
+        key = special2;
+        special2 = 0;
+        return key;
+    }
+
+    while (true) {
+        uint8 status = in8(PS2_PORT_CTRL);
+
+        if ((status & PS2_STATUS_OUTPUT_BUFFER_FULL) == 0) {
+            // no data in keyboard buffer
+            spin(200);
+            continue;
+        }
+
+        spin(200);
+        key = in8(PS2_PORT_DATA);
+
+        if (status & PS2_STATUS_AUX_DATA) {
+            // we read mouse data, ignore it
+            continue;
+        }
+
+        if (key & 0x80) {
+            // key up
+            switch (key & ~0x80) {
+                case LEFT_SHIFT:
+                case RIGHT_SHIFT:
+                    shiftPressed = false;
+                    break;
+                case LEFT_CONTROL:
+                    controlPressed = false;
+                    break;
+                case LEFT_ALT:
+                    altPressed = false;
+                    break;
+            }
+        } else {
+            // key down
+            switch (key) {
+                case LEFT_SHIFT:
+                case RIGHT_SHIFT:
+                    shiftPressed = true;
+                    break;
+
+                case LEFT_CONTROL:
+                    controlPressed = true;
+                    break;
+
+                case LEFT_ALT:
+                    altPressed = true;
+                    break;
+
+                // start escape sequence for cursor movement
+                case CURSOR_UP:
+                    special = 0x80 | 'A';
+                    return '\x1b';
+                case CURSOR_DOWN:
+                    special = 0x80 | 'B';
+                    return '\x1b';
+                case CURSOR_RIGHT:
+                    special = 0x80 | 'C';
+                    return '\x1b';
+                case CURSOR_LEFT:
+                    special = 0x80 | 'D';
+                    return '\x1b';
+                case CURSOR_HOME:
+                    special = 0x80 | 'H';
+                    return '\x1b';
+                case CURSOR_END:
+                    special = 0x80 | 'F';
+                    return '\x1b';
+                case PAGE_UP:
+                    special = 0x80 | '5';
+                    special2 = '~';
+                    return '\x1b';
+                case PAGE_DOWN:
+                    special = 0x80 | '6';
+                    special2 = '~';
+                    return '\x1b';
+
+
+                case DELETE:
+                    if (controlPressed && altPressed)
+                        arch_cpu_shutdown(true);
+
+                    special = 0x80 | '3';
+                    special2 = '~';
+                    return '\x1b';
+
+                default:
+                    if (controlPressed) {
+                        char c = kShiftedKeymap[key];
+                        if (c >= 'A' && c <= 'Z')
+                            return 0x1f & c;
+                    }
+
+                    if (altPressed)
+                        return kAltedKeymap[key];
+
+                    return shiftPressed
+                        ? kShiftedKeymap[key] : kUnshiftedKeymap[key];
+            }
+        }
+    }
+}
+
+
+char
+arch_debug_serial_getchar(void)
+{
+    while ((in8(sSerialBasePort + SERIAL_LINE_STATUS) & 0x1) == 0)
+        asm volatile ("pause;");
+
+    return in8(sSerialBasePort + SERIAL_RECEIVE_BUFFER);
+}
+
+
+void
+arch_debug_serial_putchar(const char c)
+{
+    cpu_status state = 0;
+    if (!debug_debugger_running()) {
+        state = disable_interrupts();
+        acquire_spinlock(&sSerialOutputSpinlock);
+    }
+
+    _arch_debug_serial_putchar(c);
+
+    if (!debug_debugger_running()) {
+        release_spinlock(&sSerialOutputSpinlock);
+        restore_interrupts(state);
+    }
+}
+
+
+void
+arch_debug_serial_puts(const char *s)
+{
+    cpu_status state = 0;
+    if (!debug_debugger_running()) {
+        state = disable_interrupts();
+        acquire_spinlock(&sSerialOutputSpinlock);
+    }
+
+    while (*s != '\0') {
+        _arch_debug_serial_putchar(*s);
+        s++;
+    }
+
+    if (!debug_debugger_running()) {
+        release_spinlock(&sSerialOutputSpinlock);
+        restore_interrupts(state);
+    }
+}
+
+
+void
+arch_debug_serial_early_boot_message(const char *string)
+{
+    // this function will only be called in fatal situations
+    // ToDo: also enable output via text console?!
+    arch_debug_console_init(NULL);
+    arch_debug_serial_puts(string);
+}
+
+
+status_t
+arch_debug_console_init(kernel_args *args)
+{
+    uint16 divisor = (uint16)(115200 / kSerialBaudRate);
+
+    // only use the port if we could find one, else use the standard port
+    if (args != NULL && args->platform_args.serial_base_ports[0] != 0)
+        sSerialBasePort = args->platform_args.serial_base_ports[0];
+
+    out8(0x80, sSerialBasePort + SERIAL_LINE_CONTROL);  /* set divisor latch access bit */
+    out8(divisor & 0xf, sSerialBasePort + SERIAL_DIVISOR_LATCH_LOW);
+    out8(divisor >> 8, sSerialBasePort + SERIAL_DIVISOR_LATCH_HIGH);
+    out8(3, sSerialBasePort + SERIAL_LINE_CONTROL);     /* 8N1 */
+
+    return B_OK;
+}
+
+
+status_t
+arch_debug_console_init_settings(kernel_args *args)
+{
+    uint32 baudRate = kSerialBaudRate;
+    uint16 basePort = sSerialBasePort;
+    uint16 divisor;
+    void *handle;
+
+    // get debug settings
+    handle = load_driver_settings("kernel");
+    if (handle != NULL) {
+        const char *value = get_driver_parameter(handle, "serial_debug_port",
+            NULL, NULL);
+        if (value != NULL) {
+            int32 number = strtol(value, NULL, 0);
+            if (number >= MAX_SERIAL_PORTS) {
+                // use as port number directly
+                basePort = number;
+            } else if (number >= 0) {
+                // use as index into port array
+                if (args->platform_args.serial_base_ports[number] != 0)
+                    basePort = args->platform_args.serial_base_ports[number];
+            } else {
+                // ignore value and use default
+            }
+        }
+
+        value = get_driver_parameter(handle, "serial_debug_speed", NULL, NULL);
+        if (value != NULL) {
+            int32 number = strtol(value, NULL, 0);
+            switch (number) {
+                case 9600:
+                case 19200:
+                case 38400:
+                case 57600:
+                case 115200:
+                //case 230400:
+                    baudRate = number;
+            }
+        }
+
+        unload_driver_settings(handle);
+    }
+
+    if (sSerialBasePort == basePort && baudRate == kSerialBaudRate)
+        return B_OK;
+
+    sSerialBasePort = basePort;
+    divisor = (uint16)(115200 / baudRate);
+
+    out8(0x80, sSerialBasePort + SERIAL_LINE_CONTROL);  /* set divisor latch access bit */
+    out8(divisor & 0xf, sSerialBasePort + SERIAL_DIVISOR_LATCH_LOW);
+    out8(divisor >> 8, sSerialBasePort + SERIAL_DIVISOR_LATCH_HIGH);
+    out8(3, sSerialBasePort + SERIAL_LINE_CONTROL);     /* 8N1 */
+
+    return B_OK;
+}
+

src/system/kernel/arch/x86_64/asm_offsets.cpp

@@ +1 @@
+/*
+ * Copyright 2007-2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+// This file is used to get C structure offsets into assembly code.
+// The build system assembles the file and processes the output to create
+// a header file with macro definitions, that can be included from assembly
+// code.
+
+
+#include <computed_asm_macros.h>
+
+#include <cpu.h>
+#include <ksyscalls.h>
+#include <thread_types.h>
+
+
+#define DEFINE_MACRO(macro, value) DEFINE_COMPUTED_ASM_MACRO(macro, value)
+
+#define DEFINE_OFFSET_MACRO(prefix, structure, member) \
+    DEFINE_MACRO(prefix##_##member, offsetof(struct structure, member));
+
+#define DEFINE_SIZEOF_MACRO(prefix, structure) \
+    DEFINE_MACRO(prefix##_sizeof, sizeof(struct structure));
+
+
+void
+dummy()
+{
+    // struct thread
+    DEFINE_OFFSET_MACRO(THREAD, thread, kernel_time);
+    DEFINE_OFFSET_MACRO(THREAD, thread, user_time);
+    DEFINE_OFFSET_MACRO(THREAD, thread, last_time);
+    DEFINE_OFFSET_MACRO(THREAD, thread, in_kernel);
+    DEFINE_OFFSET_MACRO(THREAD, thread, flags);
+    DEFINE_OFFSET_MACRO(THREAD, thread, kernel_stack_top);
+    DEFINE_OFFSET_MACRO(THREAD, thread, fault_handler);
+
+    // struct iframe
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, cs);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, rax);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, rdx);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, orig_rax);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, vector);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, rip);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, flags);
+    DEFINE_OFFSET_MACRO(IFRAME, iframe, user_rsp);
+
+    // struct syscall_info
+    DEFINE_SIZEOF_MACRO(SYSCALL_INFO, syscall_info);
+    DEFINE_OFFSET_MACRO(SYSCALL_INFO, syscall_info, function);
+    DEFINE_OFFSET_MACRO(SYSCALL_INFO, syscall_info, parameter_size);
+
+    // struct x86_optimized_functions
+    DEFINE_OFFSET_MACRO(X86_64_OPTIMIZED_FUNCTIONS, x86_64_optimized_functions,
+        memcpy);
+    DEFINE_OFFSET_MACRO(X86_64_OPTIMIZED_FUNCTIONS, x86_64_optimized_functions,
+        memset);
+}

src/system/kernel/arch/x86_64/apic.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2010, Michael Lotz, mmlr@mlotz.ch. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+#include <arch/x86_64/apic.h>
+
+#include <debug.h>
+#include <vm/vm.h>
+
+#include "timers/apic_timer.h"
+
+
+static void *sLocalAPIC = NULL;
+
+
+bool
+apic_available()
+{
+    return sLocalAPIC != NULL;
+}
+
+
+uint32
+apic_read(uint32 offset)
+{
+    return *(volatile uint32 *)((char *)sLocalAPIC + offset);
+}
+
+
+void
+apic_write(uint32 offset, uint32 data)
+{
+    *(volatile uint32 *)((char *)sLocalAPIC + offset) = data;
+}
+
+
+uint32
+apic_local_id()
+{
+    return (apic_read(APIC_ID) & 0xffffffff) >> 24;
+}
+
+
+void
+apic_end_of_interrupt()
+{
+    apic_write(APIC_EOI, 0);
+}
+
+
+status_t
+apic_init(kernel_args *args)
+{
+    if (args->arch_args.apic == NULL)
+        return B_NO_INIT;
+
+    sLocalAPIC = args->arch_args.apic;
+    dprintf("mapping local apic at %p\n", sLocalAPIC);
+    if (vm_map_physical_memory(B_SYSTEM_TEAM, "local apic", &sLocalAPIC,
+            B_EXACT_ADDRESS, B_PAGE_SIZE,
+            B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
+            args->arch_args.apic_phys, true) < 0) {
+        panic("mapping the local apic failed");
+        return B_ERROR;
+    }
+
+    return B_OK;
+}
+
+
+status_t
+apic_per_cpu_init(kernel_args *args, int32 cpu)
+{
+    dprintf("setting up apic for CPU %ld: apic id %lu, version %lu\n", cpu,
+        apic_local_id(), apic_read(APIC_VERSION));
+
+    /* set spurious interrupt vector to 0xff */
+    uint32 config = apic_read(APIC_SPURIOUS_INTR_VECTOR) & 0xffffff00;
+    config |= APIC_ENABLE | 0xff;
+    apic_write(APIC_SPURIOUS_INTR_VECTOR, config);
+
+
+    apic_timer_per_cpu_init(args, cpu);
+
+    /* setup error vector to 0xfe */
+    config = (apic_read(APIC_LVT_ERROR) & 0xffffff00) | 0xfe;
+    apic_write(APIC_LVT_ERROR, config);
+
+    /* accept all interrupts */
+    config = apic_read(APIC_TASK_PRIORITY) & 0xffffff00;
+    apic_write(APIC_TASK_PRIORITY, config);
+
+    config = apic_read(APIC_SPURIOUS_INTR_VECTOR);
+    apic_end_of_interrupt();
+
+    return B_OK;
+}

src/system/kernel/arch/x86_64/arch_timer.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2005, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+
+#include <boot/stage2.h>
+#include <kernel.h>
+
+#include <arch/int.h>
+#include <arch/cpu.h>
+
+#include <console.h>
+#include <debug.h>
+#include <timer.h>
+#include <int.h>
+#include <safemode.h>
+
+#include <arch/timer.h>
+
+
+//#define TRACE_TIMER
+#ifdef TRACE_TIMER
+#   define TRACE(x) dprintf x
+#else
+#   define TRACE(x) ;
+#endif
+
+extern timer_info gPITTimer;
+extern timer_info gAPICTimer;
+extern timer_info gHPETTimer;
+
+static timer_info *sTimers[] = {
+    &gHPETTimer,
+    &gAPICTimer,
+    &gPITTimer,
+    NULL
+};
+
+static timer_info *sTimer = NULL;
+
+
+static void
+sort_timers(timer_info *timers[], int numTimers)
+{
+    timer_info *tempPtr;
+    int max = 0;    
+    int i = 0;
+    int j = 0;
+    
+    for (i = 0; i < numTimers - 1; i++) {
+        max = i;
+        for (j = i + 1; j < numTimers; j++) {
+            if (timers[j]->get_priority() > timers[max]->get_priority())
+                max = j;
+        }
+        if (max != i) {
+            tempPtr = timers[max];
+            timers[max] = timers[i];
+            timers[i] = tempPtr;        
+        }
+    }
+}
+
+
+void 
+arch_timer_set_hardware_timer(bigtime_t timeout)
+{
+    TRACE(("arch_timer_set_hardware_timer: timeout %lld\n", timeout));
+    sTimer->set_hardware_timer(timeout);
+}
+
+
+void 
+arch_timer_clear_hardware_timer()
+{
+    TRACE(("arch_timer_clear_hardware_timer\n"));
+    sTimer->clear_hardware_timer();
+}
+
+
+int 
+arch_init_timer(kernel_args *args)
+{
+    // Sort timers by priority
+    sort_timers(sTimers, (sizeof(sTimers) / sizeof(sTimers[0])) - 1);
+
+    timer_info *timer = NULL;
+    cpu_status state = disable_interrupts();
+    
+    for (int i = 0; (timer = sTimers[i]) != NULL; i++) {
+        if (timer->init(args) == B_OK)
+            break;
+    }
+
+    sTimer = timer;
+
+    if (sTimer != NULL) {
+        dprintf("arch_init_timer: using %s timer.\n", sTimer->name);
+    } else {
+        panic("No system timers were found usable.\n");
+    }
+
+    restore_interrupts(state);
+
+    return 0;
+}
+

src/system/kernel/arch/x86_64/irq_routing_table.h

@@ +1 @@
+/*
+ * Copyright 2010, Clemens Zeidler, haiku@clemens-zeidler.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef IRQ_ROUTING_TABLE_H
+#define IRQ_ROUTING_TABLE_H
+
+
+#include <ACPI.h>
+#include <PCI.h>
+
+
+#include "util/Vector.h"
+
+
+struct irq_routing_entry {
+    int             device_address;
+    int8            pin;
+
+    acpi_handle     source;
+    int             source_index;
+
+    // pci busmanager connection
+    uchar           pci_bus;
+    uchar           pci_device;
+};
+
+
+typedef Vector<irq_routing_entry> IRQRoutingTable;
+
+
+struct irq_descriptor {
+    irq_descriptor();
+    // bit 0 is interrupt 0, bit 2 is interrupt 2, and so on
+    int16           irq;
+    bool            shareable;
+    // B_LOW_ACTIVE_POLARITY or B_HIGH_ACTIVE_POLARITY
+    int8            polarity;
+    // B_LEVEL_TRIGGERED or B_EDGE_TRIGGERED
+    int8            interrupt_mode;
+};
+
+
+// Similar to bus_managers/acpi/include/acrestyp.h definition
+typedef struct acpi_prt {
+    uint32          length;
+    uint32          pin;
+    int             address;        // here for 64-bit alignment
+    uint32          sourceIndex;
+    char            source[4];      // pad to 64 bits so sizeof() works in
+                                    // all cases
+} acpi_pci_routing_table;
+
+
+void print_irq_descriptor(irq_descriptor* descriptor);
+void print_irq_routing_table(IRQRoutingTable* table);
+
+
+status_t read_irq_routing_table(pci_module_info *pci, acpi_module_info* acpi,
+            IRQRoutingTable* table);
+status_t read_irq_descriptor(acpi_module_info* acpi, acpi_handle device,
+            const char* method, irq_descriptor* descriptor);
+
+status_t read_current_irq(acpi_module_info* acpi, acpi_handle device,
+            irq_descriptor* descriptor);
+status_t read_possible_irq(acpi_module_info* acpi, acpi_handle device,
+            irq_descriptor* descriptor);
+
+status_t set_acpi_irq(acpi_module_info* acpi, acpi_handle device,
+            irq_descriptor* descriptor);
+
+#endif  // IRQ_ROUTING_TABLE_H

src/system/kernel/arch/x86_64/arch_vm.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2009-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2008, Jérôme Duval.
+ * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <new>
+
+#include <KernelExport.h>
+
+#include <boot/kernel_args.h>
+#include <smp.h>
+#include <util/AutoLock.h>
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_priv.h>
+#include <vm/VMAddressSpace.h>
+#include <vm/VMArea.h>
+
+#include <arch/vm.h>
+#include <arch/int.h>
+#include <arch/cpu.h>
+
+
+//#define TRACE_ARCH_VM
+#ifdef TRACE_ARCH_VM
+#   define TRACE(x) dprintf x
+#else
+#   define TRACE(x) ;
+#endif
+
+// 0: disabled, 1: some, 2: more
+#define TRACE_MTRR_ARCH_VM 0
+
+#if TRACE_MTRR_ARCH_VM >= 1
+#   define TRACE_MTRR(x...) dprintf(x)
+#else
+#   define TRACE_MTRR(x...)
+#endif
+
+#if TRACE_MTRR_ARCH_VM >= 2
+#   define TRACE_MTRR2(x...) dprintf(x)
+#else
+#   define TRACE_MTRR2(x...)
+#endif
+
+void *gDmaAddress;
+
+
+struct memory_type_range : DoublyLinkedListLinkImpl<memory_type_range> {
+    uint64                      base;
+    uint64                      size;
+    uint32                      type;
+    area_id                     area;
+};
+
+
+struct memory_type_range_point
+        : DoublyLinkedListLinkImpl<memory_type_range_point> {
+    uint64              address;
+    memory_type_range*  range;
+
+    bool IsStart() const    { return range->base == address; }
+
+    bool operator<(const memory_type_range_point& other) const
+    {
+        return address < other.address;
+    }
+};
+
+
+struct update_mtrr_info {
+    uint64  ignoreUncacheableSize;
+    uint64  shortestUncacheableSize;
+};
+
+
+typedef DoublyLinkedList<memory_type_range> MemoryTypeRangeList;
+
+static mutex sMemoryTypeLock = MUTEX_INITIALIZER("memory type ranges");
+static MemoryTypeRangeList sMemoryTypeRanges;
+static int32 sMemoryTypeRangeCount = 0;
+
+static const uint32 kMaxMemoryTypeRegisters = 32;
+static x86_64_mtrr_info sMemoryTypeRegisters[kMaxMemoryTypeRegisters];
+static uint32 sMemoryTypeRegisterCount;
+static uint32 sMemoryTypeRegistersUsed;
+
+static memory_type_range* sTemporaryRanges = NULL;
+static memory_type_range_point* sTemporaryRangePoints = NULL;
+static int32 sTemporaryRangeCount = 0;
+static int32 sTemporaryRangePointCount = 0;
+
+
+
+static void
+set_mtrrs()
+{
+    x86_64_set_mtrrs(IA32_MTR_WRITE_BACK, sMemoryTypeRegisters,
+        sMemoryTypeRegistersUsed);
+
+#if TRACE_MTRR_ARCH_VM
+    TRACE_MTRR("set MTRRs to:\n");
+    for (uint32 i = 0; i < sMemoryTypeRegistersUsed; i++) {
+        const x86_64_mtrr_info& info = sMemoryTypeRegisters[i];
+        TRACE_MTRR("  mtrr: %2" B_PRIu32 ": base: %#10" B_PRIx64  ", size: %#10"
+            B_PRIx64 ", type: %u\n", i, info.base, info.size,
+            info.type);
+    }
+#endif
+}
+
+
+static bool
+add_used_mtrr(uint64 base, uint64 size, uint32 type)
+{
+    if (sMemoryTypeRegistersUsed == sMemoryTypeRegisterCount)
+        return false;
+
+    x86_64_mtrr_info& mtrr = sMemoryTypeRegisters[sMemoryTypeRegistersUsed++];
+    mtrr.base = base;
+    mtrr.size = size;
+    mtrr.type = type;
+
+    return true;
+}
+
+
+static bool
+ensure_temporary_ranges_space(int32 count)
+{
+    if (sTemporaryRangeCount >= count && sTemporaryRangePointCount >= count)
+        return true;
+
+    // round count to power of 2
+    int32 unalignedCount = count;
+    count = 8;
+    while (count < unalignedCount)
+        count <<= 1;
+
+    // resize ranges array
+    if (sTemporaryRangeCount < count) {
+        memory_type_range* ranges = new(std::nothrow) memory_type_range[count];
+        if (ranges == NULL)
+            return false;
+
+        delete[] sTemporaryRanges;
+
+        sTemporaryRanges = ranges;
+        sTemporaryRangeCount = count;
+    }
+
+    // resize points array
+    if (sTemporaryRangePointCount < count) {
+        memory_type_range_point* points
+            = new(std::nothrow) memory_type_range_point[count];
+        if (points == NULL)
+            return false;
+
+        delete[] sTemporaryRangePoints;
+
+        sTemporaryRangePoints = points;
+        sTemporaryRangePointCount = count;
+    }
+
+    return true;
+}
+
+
+static bool
+add_mtrrs_for_range(uint64 base, uint64 size, uint32 type)
+{
+    for (uint64 interval = B_PAGE_SIZE; size > 0; interval <<= 1) {
+        if ((base & interval) != 0) {
+            if (!add_used_mtrr(base, interval, type))
+                return false;
+            base += interval;
+            size -= interval;
+        }
+
+        if ((size & interval) != 0) {
+            if (!add_used_mtrr(base + size - interval, interval, type))
+                return false;
+            size -= interval;
+        }
+    }
+
+    return true;
+}
+
+
+static memory_type_range*
+find_range(area_id areaID)
+{
+    for (MemoryTypeRangeList::Iterator it = sMemoryTypeRanges.GetIterator();
+            memory_type_range* range = it.Next();) {
+        if (range->area == areaID)
+            return range;
+    }
+
+    return NULL;
+}
+
+
+static void
+optimize_memory_ranges(MemoryTypeRangeList& ranges, uint32 type,
+    bool removeRanges)
+{
+    uint64 previousEnd = 0;
+    uint64 nextStart = 0;
+    MemoryTypeRangeList::Iterator it = ranges.GetIterator();
+    memory_type_range* range = it.Next();
+    while (range != NULL) {
+        if (range->type != type) {
+            previousEnd = range->base + range->size;
+            nextStart = 0;
+            range = it.Next();
+            continue;
+        }
+
+        // find the start of the next range we cannot join this one with
+        if (nextStart == 0) {
+            MemoryTypeRangeList::Iterator nextIt = it;
+            while (memory_type_range* nextRange = nextIt.Next()) {
+                if (nextRange->type != range->type) {
+                    nextStart = nextRange->base;
+                    break;
+                }
+            }
+
+            if (nextStart == 0) {
+                // no upper limit -- set an artificial one, so we don't need to
+                // special case below
+                nextStart = (uint64)1 << 32;
+            }
+        }
+
+        // Align the range's base and end to the greatest power of two possible.
+        // As long as we can align both without intersecting any differently
+        // range, we can extend the range without making it more complicated.
+        // Once one side hit a limit we need to be careful. We can still
+        // continue aligning the other side, if the range crosses the power of
+        // two boundary.
+        uint64 rangeBase = range->base;
+        uint64 rangeEnd = rangeBase + range->size;
+        uint64 interval = B_PAGE_SIZE * 2;
+        while (true) {
+            uint64 alignedBase = rangeBase & ~(interval - 1);
+            uint64 alignedEnd = (rangeEnd + interval - 1) & ~(interval - 1);
+
+            if (alignedBase < previousEnd)
+                alignedBase += interval;
+
+            if (alignedEnd > nextStart)
+                alignedEnd -= interval;
+
+            if (alignedBase >= alignedEnd)
+                break;
+
+            rangeBase = std::min(rangeBase, alignedBase);
+            rangeEnd = std::max(rangeEnd, alignedEnd);
+
+            interval <<= 1;
+        }
+
+        range->base = rangeBase;
+        range->size = rangeEnd - rangeBase;
+
+        if (removeRanges)
+            it.Remove();
+
+        previousEnd = rangeEnd;
+
+        // Skip the subsequent ranges we have swallowed and possible cut one
+        // we now partially intersect with.
+        while ((range = it.Next()) != NULL) {
+            if (range->base >= rangeEnd)
+                break;
+
+            if (range->base + range->size > rangeEnd) {
+                // we partially intersect -- cut the range
+                range->size = range->base + range->size - rangeEnd;
+                range->base = rangeEnd;
+                break;
+            }
+
+            // we have swallowed this range completely
+            range->size = 0;
+            it.Remove();
+        }
+    }
+}
+
+
+status_t
+update_mtrrs(update_mtrr_info& updateInfo)
+{
+    // resize the temporary points/ranges arrays, if necessary
+    if (!ensure_temporary_ranges_space(sMemoryTypeRangeCount * 2))
+        return B_NO_MEMORY;
+
+    // get the range points and sort them
+    memory_type_range_point* rangePoints = sTemporaryRangePoints;
+    int32 pointCount = 0;
+    for (MemoryTypeRangeList::Iterator it = sMemoryTypeRanges.GetIterator();
+            memory_type_range* range = it.Next();) {
+        if (range->type == IA32_MTR_UNCACHED) {
+            // Ignore uncacheable ranges below a certain size, if requested.
+            // Since we always enforce uncacheability via the PTE attributes,
+            // this is no problem (though not recommended for performance
+            // reasons).
+            if (range->size <= updateInfo.ignoreUncacheableSize)
+                continue;
+            if (range->size < updateInfo.shortestUncacheableSize)
+                updateInfo.shortestUncacheableSize = range->size;
+        }
+
+        rangePoints[pointCount].address = range->base;
+        rangePoints[pointCount++].range = range;
+        rangePoints[pointCount].address = range->base + range->size;
+        rangePoints[pointCount++].range = range;
+    }
+
+    std::sort(rangePoints, rangePoints + pointCount);
+
+#if TRACE_MTRR_ARCH_VM >= 2
+    TRACE_MTRR2("memory type range points:\n");
+    for (int32 i = 0; i < pointCount; i++) {
+        TRACE_MTRR2("%12" B_PRIx64 " (%p)\n", rangePoints[i].address,
+            rangePoints[i].range);
+    }
+#endif
+
+    // Compute the effective ranges. When ranges overlap, we go with the
+    // stricter requirement. The types are not necessarily totally ordered, so
+    // the order we use below is not always correct. To keep it simple we
+    // consider it the reponsibility of the callers not to define overlapping
+    // memory ranges with uncomparable types.
+
+    memory_type_range* ranges = sTemporaryRanges;
+    typedef DoublyLinkedList<memory_type_range_point> PointList;
+    PointList pendingPoints;
+    memory_type_range* activeRange = NULL;
+    int32 rangeCount = 0;
+
+    for (int32 i = 0; i < pointCount; i++) {
+        memory_type_range_point* point = &rangePoints[i];
+        bool terminateRange = false;
+        if (point->IsStart()) {
+            // a range start point
+            pendingPoints.Add(point);
+            if (activeRange != NULL && activeRange->type > point->range->type)
+                terminateRange = true;
+        } else {
+            // a range end point -- remove the pending start point
+            for (PointList::Iterator it = pendingPoints.GetIterator();
+                    memory_type_range_point* pendingPoint = it.Next();) {
+                if (pendingPoint->range == point->range) {
+                    it.Remove();
+                    break;
+                }
+            }
+
+            if (point->range == activeRange)
+                terminateRange = true;
+        }
+
+        if (terminateRange) {
+            ranges[rangeCount].size = point->address - ranges[rangeCount].base;
+            rangeCount++;
+            activeRange = NULL;
+        }
+
+        if (activeRange != NULL || pendingPoints.IsEmpty())
+            continue;
+
+        // we need to start a new range -- find the strictest pending range
+        for (PointList::Iterator it = pendingPoints.GetIterator();
+                memory_type_range_point* pendingPoint = it.Next();) {
+            memory_type_range* pendingRange = pendingPoint->range;
+            if (activeRange == NULL || activeRange->type > pendingRange->type)
+                activeRange = pendingRange;
+        }
+
+        memory_type_range* previousRange = rangeCount > 0
+            ? &ranges[rangeCount - 1] : NULL;
+        if (previousRange == NULL || previousRange->type != activeRange->type
+                || previousRange->base + previousRange->size
+                    < activeRange->base) {
+            // we can't join with the previous range -- add a new one
+            ranges[rangeCount].base = point->address;
+            ranges[rangeCount].type = activeRange->type;
+        } else
+            rangeCount--;
+    }
+
+#if TRACE_MTRR_ARCH_VM >= 2
+    TRACE_MTRR2("effective memory type ranges:\n");
+    for (int32 i = 0; i < rangeCount; i++) {
+        TRACE_MTRR2("%12" B_PRIx64 " - %12" B_PRIx64 ": %" B_PRIu32 "\n",
+            ranges[i].base, ranges[i].base + ranges[i].size, ranges[i].type);
+    }
+#endif
+
+    // Extend ranges to be more MTRR-friendly. A range is MTRR friendly, when it
+    // has a power of two size and a base address aligned to the size. For
+    // ranges without this property we need more than one MTRR. We improve
+    // MTRR-friendliness by aligning a range's base and end address to the
+    // greatest power of two (base rounded down, end up) such that the extended
+    // range does not intersect with any other differently typed range. We join
+    // equally typed ranges, if possible. There are two exceptions to the
+    // intersection requirement: Uncached ranges may intersect with any other
+    // range; the resulting type will still be uncached. Hence we can ignore
+    // uncached ranges when extending the other ranges. Write-through ranges may
+    // intersect with write-back ranges; the resulting type will be
+    // write-through. Hence we can ignore write-through ranges when extending
+    // write-back ranges.
+
+    MemoryTypeRangeList rangeList;
+    for (int32 i = 0; i < rangeCount; i++)
+        rangeList.Add(&ranges[i]);
+
+    static const uint32 kMemoryTypes[] = {
+        IA32_MTR_UNCACHED,
+        IA32_MTR_WRITE_COMBINING,
+        IA32_MTR_WRITE_PROTECTED,
+        IA32_MTR_WRITE_THROUGH,
+        IA32_MTR_WRITE_BACK
+    };
+    static const int32 kMemoryTypeCount = sizeof(kMemoryTypes)
+        / sizeof(*kMemoryTypes);
+
+    for (int32 i = 0; i < kMemoryTypeCount; i++) {
+        uint32 type = kMemoryTypes[i];
+
+        // Remove uncached and write-through ranges after processing them. This
+        // let's us leverage their intersection property with any other
+        // respectively write-back ranges.
+        bool removeRanges = type == IA32_MTR_UNCACHED
+            || type == IA32_MTR_WRITE_THROUGH;
+
+        optimize_memory_ranges(rangeList, type, removeRanges);
+    }
+
+#if TRACE_MTRR_ARCH_VM >= 2
+    TRACE_MTRR2("optimized memory type ranges:\n");
+    for (int32 i = 0; i < rangeCount; i++) {
+        if (ranges[i].size > 0) {
+            TRACE_MTRR2("%12" B_PRIx64 " - %12" B_PRIx64 ": %" B_PRIu32 "\n",
+                ranges[i].base, ranges[i].base + ranges[i].size,
+                ranges[i].type);
+        }
+    }
+#endif
+
+    // compute the mtrrs from the ranges
+    sMemoryTypeRegistersUsed = 0;
+    for (int32 i = 0; i < kMemoryTypeCount; i++) {
+        uint32 type = kMemoryTypes[i];
+
+        // skip write-back ranges -- that'll be the default type anyway
+        if (type == IA32_MTR_WRITE_BACK)
+            continue;
+
+        for (int32 i = 0; i < rangeCount; i++) {
+            if (ranges[i].size == 0 || ranges[i].type != type)
+                continue;
+
+            if (!add_mtrrs_for_range(ranges[i].base, ranges[i].size, type))
+                return B_BUSY;
+        }
+    }
+
+    set_mtrrs();
+
+    return B_OK;
+}
+
+
+status_t
+update_mtrrs()
+{
+    // Until we know how many MTRRs we have, pretend everything is OK.
+    if (sMemoryTypeRegisterCount == 0)
+        return B_OK;
+
+    update_mtrr_info updateInfo;
+    updateInfo.ignoreUncacheableSize = 0;
+
+    while (true) {
+        TRACE_MTRR2("update_mtrrs(): Trying with ignoreUncacheableSize %#"
+            B_PRIx64 ".\n", updateInfo.ignoreUncacheableSize);
+
+        updateInfo.shortestUncacheableSize = ~(uint64)0;
+        status_t error = update_mtrrs(updateInfo);
+        if (error != B_BUSY) {
+            if (error == B_OK && updateInfo.ignoreUncacheableSize > 0) {
+                TRACE_MTRR("update_mtrrs(): Succeeded setting MTRRs after "
+                    "ignoring uncacheable ranges up to size %#" B_PRIx64 ".\n",
+                    updateInfo.ignoreUncacheableSize);
+            }
+            return error;
+        }
+
+        // Not enough MTRRs. Retry with less uncacheable ranges.
+        if (updateInfo.shortestUncacheableSize == ~(uint64)0) {
+            // Ugh, even without any uncacheable ranges the available MTRRs do
+            // not suffice.
+            panic("update_mtrrs(): Out of MTRRs!");
+            return B_BUSY;
+        }
+
+        ASSERT(updateInfo.ignoreUncacheableSize
+            < updateInfo.shortestUncacheableSize);
+
+        updateInfo.ignoreUncacheableSize = updateInfo.shortestUncacheableSize;
+    }
+}
+
+
+static status_t
+add_memory_type_range(area_id areaID, uint64 base, uint64 size, uint32 type)
+{
+    // translate the type
+    if (type == 0)
+        return B_OK;
+
+    switch (type) {
+        case B_MTR_UC:
+            type = IA32_MTR_UNCACHED;
+            break;
+        case B_MTR_WC:
+            type = IA32_MTR_WRITE_COMBINING;
+            break;
+        case B_MTR_WT:
+            type = IA32_MTR_WRITE_THROUGH;
+            break;
+        case B_MTR_WP:
+            type = IA32_MTR_WRITE_PROTECTED;
+            break;
+        case B_MTR_WB:
+            type = IA32_MTR_WRITE_BACK;
+            break;
+        default:
+            return B_BAD_VALUE;
+    }
+
+    TRACE_MTRR("add_memory_type_range(%" B_PRId32 ", %#" B_PRIx64 ", %#"
+        B_PRIx64 ", %" B_PRIu32 ")\n", areaID, base, size, type);
+
+    MutexLocker locker(sMemoryTypeLock);
+
+    memory_type_range* range = areaID >= 0 ? find_range(areaID) : NULL;
+    int32 oldRangeType = -1;
+    if (range != NULL) {
+        if (range->base != base || range->size != size)
+            return B_BAD_VALUE;
+        if (range->type == type)
+            return B_OK;
+
+        oldRangeType = range->type;
+        range->type = type;
+    } else {
+        range = new(std::nothrow) memory_type_range;
+        if (range == NULL)
+            return B_NO_MEMORY;
+
+        range->area = areaID;
+        range->base = base;
+        range->size = size;
+        range->type = type;
+        sMemoryTypeRanges.Add(range);
+        sMemoryTypeRangeCount++;
+    }
+
+    status_t error = update_mtrrs();
+    if (error != B_OK) {
+        // revert the addition of the range/change of its type
+        if (oldRangeType < 0) {
+            sMemoryTypeRanges.Remove(range);
+            sMemoryTypeRangeCount--;
+            delete range;
+        } else
+            range->type = oldRangeType;
+
+        update_mtrrs();
+        return error;
+    }
+
+    return B_OK;
+}
+
+
+static void
+remove_memory_type_range(area_id areaID)
+{
+    MutexLocker locker(sMemoryTypeLock);
+
+    memory_type_range* range = find_range(areaID);
+    if (range != NULL) {
+        TRACE_MTRR("remove_memory_type_range(%" B_PRId32 ", %#" B_PRIx64 ", %#"
+            B_PRIx64 ", %" B_PRIu32 ")\n", range->area, range->base,
+            range->size, range->type);
+
+        sMemoryTypeRanges.Remove(range);
+        sMemoryTypeRangeCount--;
+        delete range;
+
+        update_mtrrs();
+    } else {
+        dprintf("remove_memory_type_range(): no range known for area %" B_PRId32
+            "\n", areaID);
+    }
+}
+
+
+status_t
+arch_vm_init(kernel_args *args)
+{
+    TRACE(("arch_vm_init: entry\n"));
+    return 0;
+}
+
+
+status_t
+arch_vm_init_post_area(kernel_args *args)
+{
+    area_id id;
+
+    TRACE(("arch_vm_init_post_area: entry\n"));
+
+    // account for DMA area and mark the pages unusable
+    vm_mark_page_range_inuse(0x0, 0xa0000 / B_PAGE_SIZE);
+
+    // map 0 - 0xa0000 directly
+    id = map_physical_memory("dma_region", 0x0, 0xa0000,
+        B_ANY_KERNEL_ADDRESS | B_MTR_WB,
+        B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, &gDmaAddress);
+    if (id < 0) {
+        panic("arch_vm_init_post_area: unable to map dma region\n");
+        return B_NO_MEMORY;
+    }
+
+    return B_OK;
+}
+
+
+status_t
+arch_vm_init_end(kernel_args *args)
+{
+    TRACE(("arch_vm_init_endvm: entry\n"));
+
+    // throw away anything in the kernel_args.pgtable[] that's not yet mapped
+    vm_free_unused_boot_loader_range(KERNEL_BASE,
+        args->arch_args.virtual_end - KERNEL_BASE);
+
+    return B_OK;
+}
+
+
+status_t
+arch_vm_init_post_modules(kernel_args *args)
+{
+    // the x86_64 CPU modules are now accessible
+
+    sMemoryTypeRegisterCount = x86_64_count_mtrrs();
+    if (sMemoryTypeRegisterCount == 0)
+        return B_OK;
+
+    // not very likely, but play safe here
+    if (sMemoryTypeRegisterCount > kMaxMemoryTypeRegisters)
+        sMemoryTypeRegisterCount = kMaxMemoryTypeRegisters;
+
+    // set the physical memory ranges to write-back mode
+    for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) {
+        add_memory_type_range(-1, args->physical_memory_range[i].start,
+            args->physical_memory_range[i].size, B_MTR_WB);
+    }
+
+    return B_OK;
+}
+
+
+void
+arch_vm_aspace_swap(struct VMAddressSpace *from, struct VMAddressSpace *to)
+{
+    // This functions is only invoked when a userland thread is in the process
+    // of dying. It switches to the kernel team and does whatever cleanup is
+    // necessary (in case it is the team's main thread, it will delete the
+    // team).
+    // It is however not necessary to change the page directory. Userland team's
+    // page directories include all kernel mappings as well. Furthermore our
+    // arch specific translation map data objects are ref-counted, so they won't
+    // go away as long as they are still used on any CPU.
+}
+
+
+bool
+arch_vm_supports_protection(uint32 protection)
+{
+    // x86 and x86_64 always has the same read/write properties for userland
+    // and the kernel.
+    // That's why we do not support user-read/kernel-write access. While the
+    // other way around is not supported either, we don't care in this case
+    // and give the kernel full access.
+    if ((protection & (B_READ_AREA | B_WRITE_AREA)) == B_READ_AREA
+        && (protection & B_KERNEL_WRITE_AREA) != 0) {
+        return false;
+    }
+
+    return true;
+}
+
+
+void
+arch_vm_unset_memory_type(VMArea *area)
+{
+    if (area->MemoryType() == 0)
+        return;
+
+    remove_memory_type_range(area->id);
+}
+
+
+status_t
+arch_vm_set_memory_type(VMArea *area, phys_addr_t physicalBase, uint32 type)
+{
+    return add_memory_type_range(area->id, physicalBase, area->Size(), type);
+}

src/system/kernel/arch/x86_64/arch_platform.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Haiku, Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <arch/platform.h>
+
+
+status_t
+arch_platform_init(struct kernel_args *kernelArgs)
+{
+    return B_OK;
+}
+
+
+status_t
+arch_platform_init_post_vm(struct kernel_args *kernelArgs)
+{
+    return B_OK;
+}
+
+
+/* TODO: x86 initalizes APM code here. x86_64 could do that as well. */
+status_t
+arch_platform_init_post_thread(struct kernel_args *kernelArgs)
+{
+    return B_OK;
+}

src/system/kernel/arch/x86_64/timers/x86_64_hpet.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2009-2010, Stefano Ceccherini (stefano.ceccherini@gmail.com)
+ * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+
+#include <debug.h>
+#include <int.h>
+#include <smp.h>
+#include <timer.h>
+
+#include <arch/int.h>
+#include <arch/cpu.h>
+#include <arch/x86_64/timer.h>
+#include <arch/x86_64/arch_hpet.h>
+
+#include <boot/kernel_args.h>
+#include <vm/vm.h>
+
+
+//#define TRACE_HPET
+#ifdef TRACE_HPET
+    #define TRACE(x) dprintf x
+#else
+    #define TRACE(x) ;
+#endif
+
+#define TEST_HPET
+
+static struct hpet_regs *sHPETRegs;
+static volatile struct hpet_timer *sTimer;
+static uint64 sHPETPeriod;
+
+static int hpet_get_priority();
+static status_t hpet_set_hardware_timer(bigtime_t relativeTimeout);
+static status_t hpet_clear_hardware_timer();
+static status_t hpet_init(struct kernel_args *args);
+
+
+struct timer_info gHPETTimer = {
+    "HPET",
+    &hpet_get_priority,
+    &hpet_set_hardware_timer,
+    &hpet_clear_hardware_timer,
+    &hpet_init
+};
+
+
+static int
+hpet_get_priority()
+{
+    // TODO: Fix HPET for SMP.
+    if (smp_get_num_cpus() > 1)
+        return 0;
+
+    // HPET timers, being off-chip, are more expensive to setup
+    // than the LAPIC.
+    return 0;
+}
+
+
+static int32
+hpet_timer_interrupt(void *arg)
+{
+    //dprintf_no_syslog("HPET timer_interrupt!!!!\n");
+    return timer_interrupt();
+}
+
+
+static inline bigtime_t
+hpet_convert_timeout(const bigtime_t &relativeTimeout)
+{
+    return ((relativeTimeout * 1000000000ULL)
+        / sHPETPeriod) + sHPETRegs->u0.counter64;
+}
+
+
+#define MIN_TIMEOUT 3000
+
+static status_t
+hpet_set_hardware_timer(bigtime_t relativeTimeout)
+{
+    cpu_status state = disable_interrupts();
+
+    // enable timer interrupt
+    sTimer->config |= HPET_CONF_TIMER_INT_ENABLE;
+
+
+    if (relativeTimeout < MIN_TIMEOUT)
+        relativeTimeout = MIN_TIMEOUT;
+
+    bigtime_t timerValue = hpet_convert_timeout(relativeTimeout);
+
+    sTimer->u0.comparator64 = timerValue;
+
+    restore_interrupts(state);
+
+    return B_OK;
+}
+
+
+static status_t
+hpet_clear_hardware_timer()
+{
+    // Disable timer interrupt
+    sTimer->config &= ~HPET_CONF_TIMER_INT_ENABLE;
+    return B_OK;
+}
+
+
+static status_t
+hpet_set_enabled(bool enabled)
+{
+    if (enabled)
+        sHPETRegs->config |= HPET_CONF_MASK_ENABLED;
+    else
+        sHPETRegs->config &= ~HPET_CONF_MASK_ENABLED;
+    return B_OK;
+}
+
+
+static status_t
+hpet_set_legacy(bool enabled)
+{
+    if (!HPET_IS_LEGACY_CAPABLE(sHPETRegs)) {
+        dprintf("hpet_init: HPET doesn't support legacy mode. Skipping.\n");
+        return B_NOT_SUPPORTED;
+    }
+
+    if (enabled)
+        sHPETRegs->config |= HPET_CONF_MASK_LEGACY;
+    else
+        sHPETRegs->config &= ~HPET_CONF_MASK_LEGACY;
+
+    return B_OK;
+}
+
+
+#ifdef TRACE_HPET
+static void
+hpet_dump_timer(volatile struct hpet_timer *timer)
+{
+    dprintf("HPET Timer %ld:\n", (timer - sHPETRegs->timer));
+
+    dprintf("\troutable IRQs: ");
+    uint32 interrupts = (uint32)HPET_GET_CAP_TIMER_ROUTE(timer);
+    for (int i = 0; i < 32; i++) {
+        if (interrupts & (1 << i))
+            dprintf("%d ", i);
+    }
+    dprintf("\n");
+    dprintf("\tconfiguration: 0x%llx\n", timer->config);
+    dprintf("\tFSB Enabled: %s\n",
+        timer->config & HPET_CONF_TIMER_FSB_ENABLE ? "Yes" : "No");
+    dprintf("\tInterrupt Enabled: %s\n",
+        timer->config & HPET_CONF_TIMER_INT_ENABLE ? "Yes" : "No");
+    dprintf("\tTimer type: %s\n",
+        timer->config & HPET_CONF_TIMER_TYPE ? "Periodic" : "OneShot");
+    dprintf("\tInterrupt Type: %s\n",
+        timer->config & HPET_CONF_TIMER_INT_TYPE ? "Level" : "Edge");
+
+    dprintf("\tconfigured IRQ: %lld\n",
+        HPET_GET_CONF_TIMER_INT_ROUTE(timer));
+
+    if (timer->config & HPET_CONF_TIMER_FSB_ENABLE) {
+        dprintf("\tfsb_route[0]: 0x%llx\n", timer->fsb_route[0]);
+        dprintf("\tfsb_route[1]: 0x%llx\n", timer->fsb_route[1]);
+    }
+}
+#endif
+
+
+static void
+hpet_init_timer(volatile struct hpet_timer *timer)
+{
+    sTimer = timer;
+
+    uint32 interrupt = 0;
+
+    sTimer->config |= (interrupt << HPET_CONF_TIMER_INT_ROUTE_SHIFT)
+        & HPET_CONF_TIMER_INT_ROUTE_MASK;
+
+    // Non-periodic mode, edge triggered
+    sTimer->config &= ~(HPET_CONF_TIMER_TYPE | HPET_CONF_TIMER_INT_TYPE);
+
+    sTimer->config &= ~HPET_CONF_TIMER_FSB_ENABLE;
+    sTimer->config &= ~HPET_CONF_TIMER_32MODE;
+
+    // Enable timer
+    sTimer->config |= HPET_CONF_TIMER_INT_ENABLE;
+
+#ifdef TRACE_HPET
+    hpet_dump_timer(sTimer);
+#endif
+}
+
+
+static status_t
+hpet_test()
+{
+    uint64 initialValue = sHPETRegs->u0.counter64;
+    spin(10);
+    uint64 finalValue = sHPETRegs->u0.counter64;
+
+    if (initialValue == finalValue) {
+        dprintf("hpet_test: counter does not increment\n");
+        return B_ERROR;
+    }
+
+    return B_OK;
+}
+
+
+static status_t
+hpet_init(struct kernel_args *args)
+{
+    /* hpet_acpi_probe() through a similar "scan spots" table
+       to that of smp.cpp.
+       Seems to be the most elegant solution right now. */
+    if (args->arch_args.hpet == NULL)
+        return B_ERROR;
+
+    if (sHPETRegs == NULL) {
+        sHPETRegs = (struct hpet_regs *)((void*)args->arch_args.hpet);
+        if (vm_map_physical_memory(B_SYSTEM_TEAM, "hpet",
+            (void **)&sHPETRegs, B_EXACT_ADDRESS, B_PAGE_SIZE,
+            B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
+            (phys_addr_t)args->arch_args.hpet_phys, true) < B_OK) {
+            // Would it be better to panic here?
+            dprintf("hpet_init: Failed to map memory for the HPET registers.");
+            return B_ERROR;
+        }
+    }
+
+    sHPETPeriod = HPET_GET_PERIOD(sHPETRegs);
+
+    TRACE(("hpet_init: HPET is at %p.\n\tVendor ID: %llx, rev: %llx, period: %lld\n",
+        sHPETRegs, HPET_GET_VENDOR_ID(sHPETRegs), HPET_GET_REVID(sHPETRegs),
+        sHPETPeriod));
+
+    status_t status = hpet_set_enabled(false);
+    if (status != B_OK)
+        return status;
+
+    status = hpet_set_legacy(true);
+    if (status != B_OK)
+        return status;
+
+    uint32 numTimers = HPET_GET_NUM_TIMERS(sHPETRegs) + 1;
+
+    TRACE(("hpet_init: HPET supports %lu timers, and is %s bits wide.\n",
+        numTimers, HPET_IS_64BIT(sHPETRegs) ? "64" : "32"));
+
+    TRACE(("hpet_init: configuration: 0x%llx, timer_interrupts: 0x%llx\n",
+        sHPETRegs->config, sHPETRegs->interrupt_status));
+
+    if (numTimers < 3) {
+        dprintf("hpet_init: HPET does not have at least 3 timers. Skipping.\n");
+        return B_ERROR;
+    }
+
+#ifdef TRACE_HPET
+    for (uint32 c = 0; c < numTimers; c++)
+        hpet_dump_timer(&sHPETRegs->timer[c]);
+#endif
+
+    hpet_init_timer(&sHPETRegs->timer[0]);
+
+    status = hpet_set_enabled(true);
+    if (status != B_OK)
+        return status;
+
+#ifdef TEST_HPET
+    status = hpet_test();
+    if (status != B_OK)
+        return status;
+#endif
+
+    int32 configuredIRQ = HPET_GET_CONF_TIMER_INT_ROUTE(sTimer);
+
+    install_io_interrupt_handler(configuredIRQ, &hpet_timer_interrupt,
+        NULL, B_NO_LOCK_VECTOR);
+
+    return status;
+}
+

src/system/kernel/arch/x86_64/timers/apic_timer.h

@@ +1 @@
+/*
+ * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_x86_64_TIMERS_APIC_H
+#define _KERNEL_ARCH_x86_64_TIMERS_APIC_H
+
+#include <SupportDefs.h>
+
+status_t apic_timer_per_cpu_init(struct kernel_args *args, int32 cpu);
+
+#endif /* _KERNEL_ARCH_x86_64_TIMERS_APIC_H */

src/system/kernel/arch/x86_64/timers/pit.h

@@ +1 @@
+/*
+ * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_x86_64_TIMERS_PIT_H
+#define _KERNEL_ARCH_x86_64_TIMERS_PIT_H
+
+#include <SupportDefs.h>
+
+/* ports */
+#define PIT_CTRL    0x43
+#define PIT_CNT0    0x40
+#define PIT_CNT1    0x41
+#define PIT_CNT2    0x42
+
+/* commands */
+#define PIT_SELCH0  0x00
+#define PIT_SELCH1  0x40
+#define PIT_SELCH2  0x80
+
+#define PIT_RWLOW   0x10
+#define PIT_RWHIGH  0x20
+#define PIT_RWBOTH  0x30
+
+#define PIT_MD_INTON0   0x00
+#define PIT_MD_ONESHOT  0x02
+#define PIT_MD_RTGEN    0x04
+#define PIT_MD_SQGEN    0x06
+#define PIT_MD_SW_STRB  0x08
+#define PIT_MD_HW_STRB  0x0A
+
+#define PIT_BCD     0x01
+
+#define PIT_LATCH   0x00
+
+#define PIT_READ    0xF0
+#define PIT_CNT     0x20
+#define PIT_STAT    0x10
+
+#define PIT_CLOCK_RATE  1193180
+#define PIT_MAX_TIMER_INTERVAL (0xffff * 1000000ll / PIT_CLOCK_RATE)
+
+/* Method Prototypes */
+static int pit_get_prio(void);
+static status_t pit_set_hardware_timer(bigtime_t relativeTimeout);
+static status_t pit_clear_hardware_timer(void);
+static status_t pit_init(struct kernel_args *args);
+
+#endif /* _KERNEL_ARCH_x86_64_TIMERS_PIT_H */

src/system/kernel/arch/x86_64/timers/x86_64_apic.cpp

@@ +1 @@
+/*
+ * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved.
+ * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+#include <timer.h>
+#include <arch/x86_64/timer.h>
+
+#include <int.h>
+#include <arch/x86_64/apic.h>
+
+#include <arch/cpu.h>
+#include <arch/smp.h>
+
+#include "apic_timer.h"
+
+
+/* Method Prototypes */
+static int apic_timer_get_priority();
+static status_t apic_timer_set_hardware_timer(bigtime_t relativeTimeout);
+static status_t apic_timer_clear_hardware_timer();
+static status_t apic_timer_init(struct kernel_args *args);
+
+static uint32 sApicTicsPerSec = 0;
+
+struct timer_info gAPICTimer = {
+    "APIC",
+    &apic_timer_get_priority,
+    &apic_timer_set_hardware_timer,
+    &apic_timer_clear_hardware_timer,
+    &apic_timer_init
+};
+
+
+static int
+apic_timer_get_priority()
+{
+    return 3;
+}
+
+
+static int32
+apic_timer_interrupt(void *data)
+{
+    return timer_interrupt();
+}
+
+
+#define MIN_TIMEOUT 1
+
+static status_t
+apic_timer_set_hardware_timer(bigtime_t relativeTimeout)
+{
+    if (relativeTimeout < MIN_TIMEOUT)
+        relativeTimeout = MIN_TIMEOUT;
+
+    // calculation should be ok, since it's going to be 64-bit
+    uint32 ticks = ((relativeTimeout * sApicTicsPerSec) / 1000000);
+
+    cpu_status state = disable_interrupts();
+
+    uint32 config = apic_read(APIC_LVT_TIMER) | APIC_LVT_MASKED; // mask the timer
+    apic_write(APIC_LVT_TIMER, config);
+
+    apic_write(APIC_INITIAL_TIMER_COUNT, 0); // zero out the timer
+
+    config = apic_read(APIC_LVT_TIMER) & ~APIC_LVT_MASKED; // unmask the timer
+    apic_write(APIC_LVT_TIMER, config);
+
+    //TRACE_TIMER(("arch_smp_set_apic_timer: config 0x%lx, timeout %Ld, tics/sec %lu, tics %lu\n",
+    //  config, relativeTimeout, sApicTicsPerSec, ticks));
+
+    apic_write(APIC_INITIAL_TIMER_COUNT, ticks); // start it up
+
+    restore_interrupts(state);
+
+    return B_OK;
+}
+
+
+static status_t
+apic_timer_clear_hardware_timer()
+{
+    cpu_status state = disable_interrupts();
+
+    uint32 config = apic_read(APIC_LVT_TIMER) | APIC_LVT_MASKED;
+        // mask the timer
+    apic_write(APIC_LVT_TIMER, config);
+
+    apic_write(APIC_INITIAL_TIMER_COUNT, 0); // zero out the timer
+
+    restore_interrupts(state);
+    return B_OK;
+}
+
+
+static status_t
+apic_timer_init(struct kernel_args *args)
+{
+    if (!apic_available())
+        return B_ERROR;
+
+    sApicTicsPerSec = args->arch_args.apic_time_cv_factor;
+    install_io_interrupt_handler(0xfb - ARCH_INTERRUPT_BASE,
+        &apic_timer_interrupt, NULL, B_NO_LOCK_VECTOR);
+
+    return B_OK;
+}
+
+
+status_t
+apic_timer_per_cpu_init(struct kernel_args *args, int32 cpu)
+{
+    /* setup timer */
+    uint32 config = apic_read(APIC_LVT_TIMER) & APIC_LVT_TIMER_MASK;
+    config |= 0xfb | APIC_LVT_MASKED; // vector 0xfb, timer masked
+    apic_write(APIC_LVT_TIMER, config);
+
+    apic_write(APIC_INITIAL_TIMER_COUNT, 0); // zero out the clock
+
+    config = apic_read(APIC_TIMER_DIVIDE_CONFIG) & 0xfffffff0;
+    config |= APIC_TIMER_DIVIDE_CONFIG_1; // clock division by 1
+    apic_write(APIC_TIMER_DIVIDE_CONFIG, config);
+    return B_OK;
+}

src/system/kernel/arch/x86_64/timers/x86_64_pit.cpp

@@ +1 @@
+/*
+ * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved.
+ * Copyright 2005, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+#include <timer.h>
+#include <arch/x86_64/timer.h>
+
+#include <arch/int.h>
+#include <arch/cpu.h>
+
+#include "pit.h"
+
+static bool sPITTimerInitialized = false;
+
+struct timer_info gPITTimer = {
+    "PIT",
+    &pit_get_prio,
+    &pit_set_hardware_timer,
+    &pit_clear_hardware_timer,
+    &pit_init
+};
+
+
+static int
+pit_get_prio(void)
+{
+    return 1;
+}
+
+
+static int32
+pit_timer_interrupt(void *data)
+{
+    return timer_interrupt();
+}
+
+
+static status_t
+pit_set_hardware_timer(bigtime_t relativeTimeout)
+{
+    uint16 nextEventClocks;
+
+    if (relativeTimeout <= 0)
+        nextEventClocks = 2;
+    else if (relativeTimeout < PIT_MAX_TIMER_INTERVAL)
+        nextEventClocks = relativeTimeout * PIT_CLOCK_RATE / 1000000;
+    else
+        nextEventClocks = 0xffff;
+
+    out8(PIT_SELCH0 | PIT_RWBOTH | PIT_MD_INTON0, PIT_CTRL);
+    out8(nextEventClocks & 0xff, PIT_CNT0);
+    out8((nextEventClocks >> 8) & 0xff, PIT_CNT0);
+
+    arch_int_enable_io_interrupt(0);
+    return B_OK;
+}
+
+
+static status_t
+pit_clear_hardware_timer(void)
+{
+    arch_int_disable_io_interrupt(0);
+    return B_OK;
+}
+
+
+static status_t
+pit_init(struct kernel_args *args)
+{
+    if (sPITTimerInitialized) {
+        return B_OK;
+    }
+    
+    install_io_interrupt_handler(0, &pit_timer_interrupt, NULL, 0);
+    pit_clear_hardware_timer(); 
+
+    sPITTimerInitialized = true;
+
+    return B_OK;
+}

src/system/kernel/arch/x86_64/arch_debug.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2009-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+#include <KernelExport.h>
+
+#include <arch/debug.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <TypeConstants.h>
+
+#include <cpu.h>
+#include <debug.h>
+#include <debug_heap.h>
+#include <elf.h>
+#include <kernel.h>
+#include <kimage.h>
+#include <thread.h>
+#include <vm/vm.h>
+#include <vm/vm_types.h>
+#include <vm/VMAddressSpace.h>
+#include <vm/VMArea.h>
+
+#include <arch_cpu.h>
+
+
+struct stack_frame {
+    struct stack_frame  *previous;
+    addr_t              return_address;
+};
+
+#define NUM_PREVIOUS_LOCATIONS 32
+
+
+static status_t
+lookup_symbol(struct thread* thread, addr_t address, addr_t *_baseAddress,
+    const char **_symbolName, const char **_imageName, bool *_exactMatch)
+{
+    status_t status = B_ENTRY_NOT_FOUND;
+
+    if (IS_KERNEL_ADDRESS(address)) {
+        // a kernel symbol
+        status = elf_debug_lookup_symbol_address(address, _baseAddress,
+            _symbolName, _imageName, _exactMatch);
+    } else if (thread != NULL && thread->team != NULL) {
+        // try a lookup using the userland runtime loader structures
+        status = elf_debug_lookup_user_symbol_address(thread->team, address,
+            _baseAddress, _symbolName, _imageName, _exactMatch);
+
+        if (status != B_OK) {
+            // try to locate the image in the images loaded into user space
+            status = image_debug_lookup_user_symbol_address(thread->team,
+                address, _baseAddress, _symbolName, _imageName, _exactMatch);
+        }
+    }
+
+    return status;
+}
+
+
+static bool
+is_double_fault_stack_address(int32 cpu, addr_t address)
+{
+    size_t size;
+    addr_t bottom = (addr_t)x86_64_get_double_fault_stack(cpu, &size);
+    return address >= bottom && address < bottom + size;
+}
+
+
+static bool
+is_kernel_stack_address(struct thread* thread, addr_t address)
+{
+    // We don't have a thread pointer in the early boot process, but then we are
+    // on the kernel stack for sure.
+    if (thread == NULL)
+        return IS_KERNEL_ADDRESS(address);
+
+    // Also in the early boot process we might have a thread structure, but it
+    // might not have its kernel stack attributes set yet.
+    if (thread->kernel_stack_top == 0)
+        return IS_KERNEL_ADDRESS(address);
+
+    return (address >= thread->kernel_stack_base
+            && address < thread->kernel_stack_top)
+        || (thread->cpu != NULL
+            && is_double_fault_stack_address(thread->cpu->cpu_num, address));
+}
+
+
+static bool
+is_iframe(struct thread* thread, addr_t frame)
+{
+    if (!is_kernel_stack_address(thread, frame))
+        return false;
+
+    addr_t previousFrame = *(addr_t*)frame;
+    return ((previousFrame & ~IFRAME_TYPE_MASK) == 0 && previousFrame != 0);
+}
+
+
+static struct iframe *
+find_previous_iframe(struct thread *thread, addr_t frame)
+{
+    // iterate backwards through the stack frames, until we hit an iframe
+    while (is_kernel_stack_address(thread, frame)) {
+        if (is_iframe(thread, frame))
+            return (struct iframe*)frame;
+
+        frame = *(addr_t*)frame;
+    }
+
+    return NULL;
+}
+
+static struct iframe*
+get_previous_iframe(struct thread* thread, struct iframe* frame)
+{
+    if (frame == NULL)
+        return NULL;
+
+    return find_previous_iframe(thread, frame->rbp);
+}
+
+static struct iframe*
+get_current_iframe(struct thread* thread)
+{
+    if (thread == thread_get_current_thread())
+        return x86_64_get_current_iframe();
+
+    addr_t rbp = thread->arch_info.current_stack.rsp[2];
+        // NOTE: This doesn't work, if the thread is running (on another CPU).
+    return find_previous_iframe(thread, rbp);
+}
+
+uint64*
+find_debug_variable(const char* variableName, bool& settable)
+{
+    struct iframe* frame = get_current_iframe(debug_get_debugged_thread());
+    if (frame == NULL)
+        return NULL;
+
+    settable = false;
+
+    if (strcmp(variableName, "gs") == 0) {
+        return &frame->gs;
+    } else if (strcmp(variableName, "fs") == 0) {
+        return &frame->fs;
+    } else if (strcmp(variableName, "es") == 0) {
+        return &frame->es;
+    } else if (strcmp(variableName, "ds") == 0) {
+        return &frame->ds;
+    } else if (strcmp(variableName, "cs") == 0) {
+        return &frame->cs;
+    } else if (strcmp(variableName, "rdi") == 0) {
+        settable = true;
+        return &frame->rdi;
+    } else if (strcmp(variableName, "rsi") == 0) {
+        settable = true;
+        return &frame->rsi;
+    } else if (strcmp(variableName, "rbp") == 0) {
+        settable = true;
+        return &frame->rbp;
+    } else if (strcmp(variableName, "rsp") == 0) {
+        settable = true;
+        return &frame->rsp;
+    } else if (strcmp(variableName, "rbx") == 0) {
+        settable = true;
+        return &frame->rbx;
+    } else if (strcmp(variableName, "rdx") == 0) {
+        settable = true;
+        return &frame->rdx;
+    } else if (strcmp(variableName, "rcx") == 0) {
+        settable = true;
+        return &frame->rcx;
+    } else if (strcmp(variableName, "rax") == 0) {
+        settable = true;
+        return &frame->rax;
+    } else if (strcmp(variableName, "r8") == 0) {
+        settable = true;
+        return &frame->r8;
+    } else if (strcmp(variableName, "r9") == 0) {
+        settable = true;
+        return &frame->r9;
+    } else if (strcmp(variableName, "r10") == 0) {
+        settable = true;
+        return &frame->r10;
+    } else if (strcmp(variableName, "r11") == 0) {
+        settable = true;
+        return &frame->r11;
+    } else if (strcmp(variableName, "r12") == 0) {
+        settable = true;
+        return &frame->r12;
+    } else if (strcmp(variableName, "r13") == 0) {
+        settable = true;
+        return &frame->r13;
+    } else if (strcmp(variableName, "r14") == 0) {
+        settable = true;
+        return &frame->r14;
+    } else if (strcmp(variableName, "r15") == 0) {
+        settable = true;
+        return &frame->r15;
+    } else if (strcmp(variableName, "orig_rax") == 0) {
+        settable = true;
+        return &frame->orig_rax;
+    } else if (strcmp(variableName, "orig_rdx") == 0) {
+        settable = true;
+        return &frame->orig_rdx;
+    } else if (strcmp(variableName, "rip") == 0) {
+        settable = true;
+        return &frame->rip;
+    } else if (strcmp(variableName, "rflags") == 0) {
+        settable = true;
+        return (uint64*)(&frame->flags);
+    }
+
+    if (IFRAME_IS_USER(frame)) {
+        if (strcmp(variableName, "user_rsp") == 0) {
+            settable = true;
+            return &frame->user_rsp;
+        } else if (strcmp(variableName, "user_ss") == 0) {
+            return &frame->user_ss;
+        }
+    }
+
+    return NULL;
+}
+
+
+static void
+print_iframe(struct iframe *frame)
+{
+    bool isUser = IFRAME_IS_USER(frame);
+    kprintf("%s iframe at %p (end = %p)\n", isUser ? "user" : "kernel", frame,
+        isUser ? (uint64*)(frame + 1) : &frame->user_rsp);
+
+    kprintf(" rax 0x%-9llx    rbx 0x%-9llx     rcx 0x%-9llx  rdx 0x%llx\n",
+        frame->rax, frame->rbx, frame->rcx, frame->rdx);
+    kprintf(" rsi 0x%-9llx    rdi 0x%-9llx     rbp 0x%-9llx  rsp 0x%llx\n",
+        frame->rsi, frame->rdi, frame->rbp, frame->rsp);
+    kprintf(" rip 0x%-9llx eflags 0x%-9lx", frame->rip, frame->flags);
+    if (isUser) {
+        // from user space
+        kprintf("user rsp 0x%llx", frame->user_rsp);
+    }
+    kprintf("\n");
+    kprintf(" vector: 0x%llx, error code: 0x%llx\n", frame->vector,
+        frame->error_code);
+}
+
+
+static bool
+setup_for_thread(char *arg, struct thread **_thread, uint64 *_rbp,
+    uint64 *_oldPageDirectory)
+{
+    struct thread *thread = NULL;
+
+    if (arg != NULL) {
+        thread_id id = strtoul(arg, NULL, 0);
+        thread = thread_get_thread_struct_locked(id);
+        if (thread == NULL) {
+            kprintf("could not find thread %ld\n", id);
+            return false;
+        }
+
+        if (id != thread_get_current_thread_id()) {
+            // switch to the page directory of the new thread to be
+            // able to follow the stack trace into userland
+            uint32 newPageDirectory = x86_64_next_page_directory(
+                thread_get_current_thread(), thread);
+
+            if (newPageDirectory != 0) {
+                read_cr3(*_oldPageDirectory);
+                write_cr3(newPageDirectory);
+            }
+
+            if (thread->state == B_THREAD_RUNNING) {
+                // The thread is currently running on another CPU.
+                if (thread->cpu == NULL)
+                    return false;
+                arch_debug_registers* registers = debug_get_debug_registers(
+                    thread->cpu->cpu_num);
+                if (registers == NULL)
+                    return false;
+                *_rbp = registers->rbp;
+            } else {
+                // read %ebp from the thread's stack stored by a pushad
+                *_rbp = thread->arch_info.current_stack.rsp[2];
+            }
+        } else
+            thread = NULL;
+    }
+
+    if (thread == NULL) {
+        // if we don't have a thread yet, we want the current one
+        // (ebp has been set by the caller for this case already)
+        thread = thread_get_current_thread();
+    }
+
+    *_thread = thread;
+    return true;
+}
+
+
+void
+arch_debug_save_registers(struct arch_debug_registers* registers)
+{
+    // get the caller's frame pointer
+    stack_frame* frame = (stack_frame*)x86_64_read_rbp();
+    registers->rbp = (addr_t)frame->previous;
+}
+
+
+static bool
+is_calling(struct thread *thread, addr_t rip, const char *pattern,
+    addr_t start, addr_t end)
+{
+    if (pattern == NULL)
+        return rip >= start && rip < end;
+
+    if (!IS_KERNEL_ADDRESS(rip))
+        return false;
+
+    const char *symbol;
+    if (lookup_symbol(thread, rip, NULL, &symbol, NULL, NULL) != B_OK)
+        return false;
+
+    return strstr(symbol, pattern);
+}
+
+
+static status_t
+get_next_frame_no_debugger(addr_t rbp, addr_t *_next, addr_t *_rip)
+{
+    // TODO: Do this more efficiently in assembly.
+    stack_frame frame;
+    if (user_memcpy(&frame, (void*)rbp, sizeof(frame)) != B_OK)
+        return B_BAD_ADDRESS;
+
+    *_rip = frame.return_address;
+    *_next = (addr_t)frame.previous;
+
+    return B_OK;
+}
+
+
+bool
+arch_debug_contains_call(struct thread *thread, const char *symbol,
+    addr_t start, addr_t end)
+{
+    DebuggedThreadSetter threadSetter(thread);
+
+    addr_t rbp;
+    if (thread == thread_get_current_thread())
+        rbp = x86_64_read_rbp();
+    else {
+        if (thread->state == B_THREAD_RUNNING) {
+            // The thread is currently running on another CPU.
+            if (thread->cpu == NULL)
+                return false;
+            arch_debug_registers* registers = debug_get_debug_registers(
+                thread->cpu->cpu_num);
+            if (registers == NULL)
+                return false;
+            rbp = registers->rbp;
+        } else {
+            // thread not running
+            rbp = thread->arch_info.current_stack.rsp[2];
+        }
+    }
+
+    for (;;) {
+        if (!is_kernel_stack_address(thread, rbp))
+            break;
+
+        if (is_iframe(thread, rbp)) {
+            struct iframe *frame = (struct iframe *)rbp;
+
+            if (is_calling(thread, frame->rip, symbol, start, end))
+                return true;
+
+            rbp = frame->rbp;
+        } else {
+            addr_t rip, nextRbp;
+
+            if (get_next_frame_no_debugger(rbp, &nextRbp, &rip) != B_OK
+                || rip == 0 || rbp == 0)
+                break;
+
+            if (is_calling(thread, rip, symbol, start, end))
+                return true;
+
+            rbp = nextRbp;
+        }
+
+        if (rbp == 0)
+            break;
+    }
+
+    return false;
+}
+
+
+void *
+arch_debug_get_caller(void)
+{
+    struct stack_frame *frame = (struct stack_frame *)x86_64_read_rbp();
+    return (void *)frame->previous->return_address;
+}
+
+
+void*
+arch_debug_get_interrupt_pc(bool* _isSyscall)
+{
+    struct iframe* frame = get_current_iframe(debug_get_debugged_thread());
+    if (frame == NULL)
+        return NULL;
+
+    if (_isSyscall != NULL)
+        *_isSyscall = frame->vector == 99;
+
+    return (void*)(addr_t)frame->rip;
+}
+
+
+bool
+arch_is_debug_variable_defined(const char* variableName)
+{
+    bool settable;
+    return find_debug_variable(variableName, settable);
+}
+
+
+status_t
+arch_set_debug_variable(const char* variableName, uint64 value)
+{
+    bool settable;
+    uint64* variable = find_debug_variable(variableName, settable);
+    if (variable == NULL)
+        return B_ENTRY_NOT_FOUND;
+
+    if (!settable)
+        return B_NOT_ALLOWED;
+
+    *variable = (uint64)value;
+    return B_OK;
+}
+
+
+status_t
+arch_get_debug_variable(const char* variableName, uint64* value)
+{
+    bool settable;
+    uint64* variable = find_debug_variable(variableName, settable);
+    if (variable == NULL)
+        return B_ENTRY_NOT_FOUND;
+
+    *value = *variable;
+    return B_OK;
+}
+
+/* 
+void
+arch_debug_call_with_fault_handler(cpu_ent* cpu, jmp_buf jumpBuffer,
+    void (*function)(void*), void* parameter)
+
+    moved to arch_x86_64.S
+*/
+
+
+void
+arch_debug_unset_current_thread(void)
+{
+    write_dr3(NULL);
+}
+
+
+static void
+set_debug_argument_variable(int32 index, uint64 value)
+{
+    char name[8];
+    snprintf(name, sizeof(name), "_arg%ld", index);
+    set_debug_variable(name, value);
+}
+
+
+template<typename Type>
+static Type
+read_function_argument_value(void* argument, bool& _valueKnown)
+{
+    Type value;
+    if (debug_memcpy(B_CURRENT_TEAM, &value, argument, sizeof(Type)) == B_OK) {
+        _valueKnown = true;
+        return value;
+    }
+
+    _valueKnown = false;
+    return 0;
+}
+
+
+static status_t
+print_demangled_call(const char* image, const char* symbol, addr_t args,
+    bool noObjectMethod, bool addDebugVariables)
+{
+    static const size_t kBufferSize = 256;
+    char* buffer = (char*)debug_malloc(kBufferSize);
+    if (buffer == NULL)
+        return B_NO_MEMORY;
+
+    bool isObjectMethod;
+    const char* name = debug_demangle_symbol(symbol, buffer, kBufferSize,
+        &isObjectMethod);
+    if (name == NULL) {
+        debug_free(buffer);
+        return B_ERROR;
+    }
+
+    uint64* arg = (uint64*)args;
+
+    if (noObjectMethod)
+        isObjectMethod = false;
+    if (isObjectMethod) {
+        const char* lastName = strrchr(name, ':') - 1;
+        int namespaceLength = lastName - name;
+
+        uint32 argValue = 0;
+        if (debug_memcpy(B_CURRENT_TEAM, &argValue, arg, 4) == B_OK) {
+            kprintf("<%s> %.*s<\33[32m%#" B_PRIx32 "\33[0m>%s", image,
+                namespaceLength, name, argValue, lastName);
+        } else
+            kprintf("<%s> %.*s<???>%s", image, namespaceLength, name, lastName);
+
+        if (addDebugVariables)
+            set_debug_variable("_this", argValue);
+        arg++;
+    } else
+        kprintf("<%s> %s", image, name);
+
+    kprintf("(");
+
+    size_t length;
+    int32 type, i = 0;
+    uint32 cookie = 0;
+    while (debug_get_next_demangled_argument(&cookie, symbol, buffer,
+            kBufferSize, &type, &length) == B_OK) {
+        if (i++ > 0)
+            kprintf(", ");
+
+        // retrieve value and type identifier
+
+        uint64 value;
+        bool valueKnown = false;
+
+        switch (type) {
+            case B_INT64_TYPE:
+                value = read_function_argument_value<int64>(arg, valueKnown);
+                if (valueKnown)
+                    kprintf("int64: \33[34m%Ld\33[0m", value);
+                break;
+            case B_INT32_TYPE:
+                value = read_function_argument_value<int32>(arg, valueKnown);
+                if (valueKnown)
+                    kprintf("int32: \33[34m%ld\33[0m", (int32)value);
+                break;
+            case B_INT16_TYPE:
+                value = read_function_argument_value<int16>(arg, valueKnown);
+                if (valueKnown)
+                    kprintf("int16: \33[34m%d\33[0m", (int16)value);
+                break;
+            case B_INT8_TYPE:
+                value = read_function_argument_value<int8>(arg, valueKnown);
+                if (valueKnown)
+                    kprintf("int8: \33[34m%d\33[0m", (int8)value);
+                break;
+            case B_UINT64_TYPE:
+                value = read_function_argument_value<uint64>(arg, valueKnown);
+                if (valueKnown) {
+                    kprintf("uint64: \33[34m%#Lx\33[0m", value);
+                    if (value < 0x100000)
+                        kprintf(" (\33[34m%Lu\33[0m)", value);
+                }
+                break;
+            case B_UINT32_TYPE:
+                value = read_function_argument_value<uint32>(arg, valueKnown);
+                if (valueKnown) {
+                    kprintf("uint32: \33[34m%#lx\33[0m", (uint32)value);
+                    if (value < 0x100000)
+                        kprintf(" (\33[34m%lu\33[0m)", (uint32)value);
+                }
+                break;
+            case B_UINT16_TYPE:
+                value = read_function_argument_value<uint16>(arg, valueKnown);
+                if (valueKnown) {
+                    kprintf("uint16: \33[34m%#x\33[0m (\33[34m%u\33[0m)",
+                        (uint16)value, (uint16)value);
+                }
+                break;
+            case B_UINT8_TYPE:
+                value = read_function_argument_value<uint8>(arg, valueKnown);
+                if (valueKnown) {
+                    kprintf("uint8: \33[34m%#x\33[0m (\33[34m%u\33[0m)",
+                        (uint8)value, (uint8)value);
+                }
+                break;
+            case B_BOOL_TYPE:
+                value = read_function_argument_value<uint8>(arg, valueKnown);
+                if (valueKnown)
+                    kprintf("\33[34m%s\33[0m", value ? "true" : "false");
+                break;
+            default:
+                if (buffer[0])
+                    kprintf("%s: ", buffer);
+
+                if (length == 4) {
+                    value = read_function_argument_value<uint32>(arg,
+                        valueKnown);
+                    if (valueKnown) {
+                        if (value == 0
+                            && (type == B_POINTER_TYPE || type == B_REF_TYPE))
+                            kprintf("NULL");
+                        else
+                            kprintf("\33[34m%#lx\33[0m", (uint32)value);
+                    }
+                    break;
+                }
+
+
+                if (length == 8) {
+                    value = read_function_argument_value<uint64>(arg,
+                        valueKnown);
+                } else
+                    value = (uint64)arg;
+
+                if (valueKnown)
+                    kprintf("\33[34m%#Lx\33[0m", value);
+                break;
+        }
+
+        if (!valueKnown)
+            kprintf("???");
+
+        if (valueKnown && type == B_STRING_TYPE) {
+            if (value == 0)
+                kprintf(" \33[31m\"<NULL>\"\33[0m");
+            else if (debug_strlcpy(B_CURRENT_TEAM, buffer, (char*)value,
+                    kBufferSize) < B_OK) {
+                kprintf(" \33[31m\"<???>\"\33[0m");
+            } else
+                kprintf(" \33[36m\"%s\"\33[0m", buffer);
+        }
+
+        if (addDebugVariables)
+            set_debug_argument_variable(i, value);
+        arg = (uint64*)((uint8*)arg + length);
+    }
+
+    debug_free(buffer);
+
+    kprintf(")");
+    return B_OK;
+}
+
+
+static void
+print_call(struct thread *thread, addr_t rip, addr_t rbp, addr_t nextRbp,
+    int32 argCount)
+{
+    const char *symbol, *image;
+    addr_t baseAddress;
+    bool exactMatch;
+    status_t status;
+    bool demangled = false;
+    int64 *arg = (int64 *)(nextRbp + 8);
+
+    status = lookup_symbol(thread, rip, &baseAddress, &symbol, &image,
+        &exactMatch);
+
+    kprintf("%08lx %08lx   ", rbp, rip);
+
+    if (status == B_OK) {
+        if (symbol != NULL) {
+            if (exactMatch && (argCount == 0 || argCount == -1)) {
+                status = print_demangled_call(image, symbol, (addr_t)arg,
+                    argCount == -1, true);
+                if (status == B_OK)
+                    demangled = true;
+            }
+            if (!demangled) {
+                kprintf("<%s>:%s%s", image, symbol,
+                    exactMatch ? "" : " (nearest)");
+            }
+        } else {
+            kprintf("<%s@%p>:unknown + 0x%04lx", image,
+                (void *)baseAddress, rip - baseAddress);
+        }
+    } else {
+        VMArea *area = NULL;
+        if (thread->team->address_space != NULL)
+            area = thread->team->address_space->LookupArea(rip);
+        if (area != NULL) {
+            kprintf("%ld:%s@%p + %#lx", area->id, area->name,
+                (void *)area->Base(), rip - area->Base());
+        }
+    }
+
+    if (!demangled) {
+        kprintf("(");
+
+        for (int32 i = 0; i < argCount; i++) {
+            if (i > 0)
+                kprintf(", ");
+            kprintf("%#llx", *arg);
+            if (*arg > -0x10000 && *arg < 0x10000)
+                kprintf(" (%lld)", *arg);
+
+            set_debug_argument_variable(i + 1, *(uint32 *)arg);
+            arg++;
+        }
+
+        kprintf(")\n");
+    } else
+        kprintf("\n");
+
+    set_debug_variable("_frame", nextRbp);
+}
+
+
+static int
+dump_iframes(int argc, char **argv)
+{
+    static const char* usage = "usage: %s [ <thread id> ]\n"
+        "Prints the iframe stack for the current, respectively the specified\n"
+        "thread.\n"
+        "  <thread id>  -  The ID of the thread for which to print the iframe\n"
+        "                  stack.\n";
+    if (argc == 2 && strcmp(argv[1], "--help") == 0) {
+        kprintf(usage, argv[0]);
+        return 0;
+    }
+
+    struct thread *thread = NULL;
+
+    if (argc < 2) {
+        thread = thread_get_current_thread();
+    } else if (argc == 2) {
+        thread_id id = strtoul(argv[1], NULL, 0);
+        thread = thread_get_thread_struct_locked(id);
+        if (thread == NULL) {
+            kprintf("could not find thread %ld\n", id);
+            return 0;
+        }
+    } else if (argc > 2) {
+        kprintf(usage, argv[0]);
+        return 0;
+    }
+
+    if (thread != NULL)
+        kprintf("iframes for thread %ld \"%s\"\n", thread->id, thread->name);
+
+    DebuggedThreadSetter threadSetter(thread);
+
+    struct iframe* frame = find_previous_iframe(thread, x86_64_read_rbp());
+    while (frame != NULL) {
+        print_iframe(frame);
+        frame = get_previous_iframe(thread, frame);
+    }
+
+    return 0;
+}
+
+
+static status_t
+get_next_frame_debugger(addr_t rbp, addr_t *_next, addr_t *_rip)
+{
+    stack_frame frame;
+    if (debug_memcpy(B_CURRENT_TEAM, &frame, (void*)rbp, sizeof(frame)) != B_OK)
+        return B_BAD_ADDRESS;
+
+    *_rip = frame.return_address;
+    *_next = (addr_t)frame.previous;
+
+    return B_OK;
+}
+
+
+static int
+show_call(int argc, char **argv)
+{
+    static const char* usage
+        = "usage: %s [ <thread id> ] <call index> [ -<arg count> ]\n"
+        "Prints a function call with parameters of the current, respectively\n"
+        "the specified thread.\n"
+        "  <thread id>   -  The ID of the thread for which to print the call.\n"
+        "  <call index>  -  The index of the call in the stack trace.\n"
+        "  <arg count>   -  The number of call arguments to print (use 'c' to\n"
+        "                   force the C++ demangler to use class methods,\n"
+        "                   use 'd' to disable demangling).\n";
+    if (argc == 2 && strcmp(argv[1], "--help") == 0) {
+        kprintf(usage, argv[0]);
+        return 0;
+    }
+
+    struct thread *thread = NULL;
+    uint64 oldPageDirectory = 0;
+    uint64 rbp = x86_64_read_rbp();
+    int32 argCount = 0;
+
+    if (argc >= 2 && argv[argc - 1][0] == '-') {
+        if (argv[argc - 1][1] == 'c')
+            argCount = -1;
+        else if (argv[argc - 1][1] == 'd')
+            argCount = -2;
+        else
+            argCount = strtoul(argv[argc - 1] + 1, NULL, 0);
+
+        if (argCount < -2 || argCount > 16) {
+            kprintf("Invalid argument count \"%ld\".\n", argCount);
+            return 0;
+        }
+        argc--;
+    }
+
+    if (argc < 2 || argc > 3) {
+        kprintf(usage, argv[0]);
+        return 0;
+    }
+
+    if (!setup_for_thread(argc == 3 ? argv[1] : NULL, &thread, &rbp,
+            &oldPageDirectory))
+        return 0;
+
+    DebuggedThreadSetter threadSetter(thread);
+
+    int32 callIndex = strtoul(argv[argc == 3 ? 2 : 1], NULL, 0);
+
+    if (thread != NULL)
+        kprintf("thread %ld, %s\n", thread->id, thread->name);
+
+    bool onKernelStack = true;
+
+    for (int32 index = 0; index <= callIndex; index++) {
+        onKernelStack = onKernelStack
+            && is_kernel_stack_address(thread, rbp);
+
+        if (onKernelStack && is_iframe(thread, rbp)) {
+            struct iframe *frame = (struct iframe *)rbp;
+
+            if (index == callIndex)
+                print_call(thread, frame->rip, rbp, frame->rbp, argCount);
+
+            rbp = frame->rbp;
+        } else {
+            addr_t rip, nextRbp;
+
+            if (get_next_frame_debugger(rbp, &nextRbp, &rip) != B_OK) {
+                kprintf("%08llx -- read fault\n", rbp);
+                break;
+            }
+
+            if (rip == 0 || rbp == 0)
+                break;
+
+            if (index == callIndex)
+                print_call(thread, rip, rbp, nextRbp, argCount);
+
+            rbp = nextRbp;
+        }
+
+        if (rbp == 0)
+            break;
+    }
+
+    if (oldPageDirectory != 0) {
+        // switch back to the previous page directory to not cause any troubles
+        write_cr3(oldPageDirectory);
+    }
+
+    return 0;
+}
+
+
+static void
+print_stack_frame(struct thread *thread, addr_t rip, addr_t rbp, addr_t nextRbp,
+    int32 callIndex, bool demangle)
+{
+    const char *symbol, *image;
+    addr_t baseAddress;
+    bool exactMatch;
+    status_t status;
+    addr_t diff;
+
+    diff = nextRbp - rbp;
+
+    // kernel space/user space switch
+    if (diff & 0x80000000)
+        diff = 0;
+
+    status = lookup_symbol(thread, rip, &baseAddress, &symbol, &image,
+        &exactMatch);
+
+    kprintf("%2ld %08lx (+%4ld) %08lx   ", callIndex, rbp, diff, rip);
+
+    if (status == B_OK) {
+        if (exactMatch && demangle) {
+            status = print_demangled_call(image, symbol, nextRbp + 8, false,
+                false);
+        }
+
+        if (!exactMatch || !demangle || status != B_OK) {
+            if (symbol != NULL) {
+                kprintf("<%s>:%s%s", image, symbol,
+                    exactMatch ? "" : " (nearest)");
+            } else
+                kprintf("<%s@%p>:unknown", image, (void *)baseAddress);
+        }
+
+        kprintf(" + 0x%04lx\n", rip - baseAddress);
+    } else {
+        VMArea *area = NULL;
+        if (thread != NULL && thread->team != NULL
+            && thread->team->address_space != NULL) {
+            area = thread->team->address_space->LookupArea(rip);
+        }
+        if (area != NULL) {
+            kprintf("%ld:%s@%p + %#lx\n", area->id, area->name,
+                (void*)area->Base(), rip - area->Base());
+        } else
+            kprintf("\n");
+    }
+}
+
+
+static bool
+already_visited(uint64 *visited, int32 *_last, int32 *_num, uint64 rbp)
+{
+    int32 last = *_last;
+    int32 num = *_num;
+    int32 i;
+
+    for (i = 0; i < num; i++) {
+        if (visited[(NUM_PREVIOUS_LOCATIONS + last - i) % NUM_PREVIOUS_LOCATIONS] == rbp)
+            return true;
+    }
+
+    *_last = last = (last + 1) % NUM_PREVIOUS_LOCATIONS;
+    visited[last] = rbp;
+
+    if (num < NUM_PREVIOUS_LOCATIONS)
+        *_num = num + 1;
+
+    return false;
+}
+
+
+static int
+stack_trace(int argc, char **argv)
+{
+    static const char* usage = "usage: %s [-d] [ <thread id> ]\n"
+        "Prints a stack trace for the current, respectively the specified\n"
+        "thread.\n"
+        "  -d           -  Disables the demangling of the symbols.\n"
+        "  <thread id>  -  The ID of the thread for which to print the stack\n"
+        "                  trace.\n";
+    bool demangle = true;
+    int32 threadIndex = 1;
+    if (argc > 1 && !strcmp(argv[1], "-d")) {
+        demangle = false;
+        threadIndex++;
+    }
+
+    if (argc > threadIndex + 1
+        || (argc == 2 && strcmp(argv[1], "--help") == 0)) {
+        kprintf(usage, argv[0]);
+        return 0;
+    }
+
+    uint64 previousLocations[NUM_PREVIOUS_LOCATIONS];
+    struct thread *thread = NULL;
+    uint64 oldPageDirectory = 0;
+    uint64 rbp = x86_64_read_rbp();
+    int32 num = 0, last = 0;
+
+    if (!setup_for_thread(argc == threadIndex + 1 ? argv[threadIndex] : NULL,
+            &thread, &rbp, &oldPageDirectory))
+        return 0;
+
+    DebuggedThreadSetter threadSetter(thread);
+
+    if (thread != NULL) {
+        kprintf("stack trace for thread %ld \"%s\"\n", thread->id,
+            thread->name);
+
+        kprintf("    kernel stack: %p to %p\n",
+            (void *)thread->kernel_stack_base,
+            (void *)(thread->kernel_stack_top));
+        if (thread->user_stack_base != 0) {
+            kprintf("      user stack: %p to %p\n",
+                (void *)thread->user_stack_base,
+                (void *)(thread->user_stack_base + thread->user_stack_size));
+        }
+    }
+
+    kprintf("frame               caller     <image>:function + offset\n");
+
+    bool onKernelStack = true;
+
+    for (int32 callIndex = 0;; callIndex++) {
+        onKernelStack = onKernelStack
+            && is_kernel_stack_address(thread, rbp);
+
+        if (onKernelStack && is_iframe(thread, rbp)) {
+            struct iframe *frame = (struct iframe *)rbp;
+
+            print_iframe(frame);
+            print_stack_frame(thread, frame->rip, rbp, frame->rbp, callIndex,
+                demangle);
+
+            rbp = frame->rbp;
+        } else {
+            addr_t rip, nextRbp;
+
+            if (get_next_frame_debugger(rbp, &nextRbp, &rip) != B_OK) {
+                kprintf("%08llx -- read fault\n", rbp);
+                break;
+            }
+
+            if (rip == 0 || rbp == 0)
+                break;
+
+            print_stack_frame(thread, rip, rbp, nextRbp, callIndex, demangle);
+            rbp = nextRbp;
+        }
+
+        if (already_visited(previousLocations, &last, &num, rbp)) {
+            kprintf("circular stack frame: %p!\n", (void *)rbp);
+            break;
+        }
+        if (rbp == 0)
+            break;
+    }
+
+    if (oldPageDirectory != 0) {
+        // switch back to the previous page directory to no cause any troubles
+        write_cr3(oldPageDirectory);
+    }
+
+    return 0;
+}
+
+
+void
+arch_debug_stack_trace(void)
+{
+    stack_trace(0, NULL);
+}
+
+
+static int
+cmd_in_context(int argc, char** argv)
+{
+    if (argc != 2) {
+        print_debugger_command_usage(argv[0]);
+        return 0;
+    }
+
+    // get the thread ID
+    const char* commandLine = argv[1];
+    char threadIDString[16];
+    if (parse_next_debug_command_argument(&commandLine, threadIDString,
+            sizeof(threadIDString)) != B_OK) {
+        kprintf("Failed to parse thread ID.\n");
+        return 0;
+    }
+
+    if (commandLine == NULL) {
+        print_debugger_command_usage(argv[0]);
+        return 0;
+    }
+
+    uint64 threadID;
+    if (!evaluate_debug_expression(threadIDString, &threadID, false))
+        return 0;
+
+    // get the thread
+    struct thread* thread = thread_get_thread_struct_locked(threadID);
+    if (thread == NULL) {
+        kprintf("Could not find thread with ID \"%s\".\n", threadIDString);
+        return 0;
+    }
+
+    // switch the page directory, if necessary
+    uint32 oldPageDirectory = 0;
+    if (thread != thread_get_current_thread()) {
+        uint32 newPageDirectory = x86_64_next_page_directory(
+            thread_get_current_thread(), thread);
+
+        if (newPageDirectory != 0) {
+            read_cr3(oldPageDirectory);
+            write_cr3(newPageDirectory);
+        }
+    }
+
+    // execute the command
+    {
+        DebuggedThreadSetter threadSetter(thread);
+        evaluate_debug_command(commandLine);
+    }
+
+    // reset the page directory
+    if (oldPageDirectory)
+        write_cr3(oldPageDirectory);
+
+    return 0;
+}
+
+
+status_t
+arch_debug_init(kernel_args *args)
+{
+    // at this stage, the debugger command system is alive
+
+    add_debugger_command("where", &stack_trace, "Same as \"sc\"");
+    add_debugger_command("bt", &stack_trace, "Same as \"sc\" (as in gdb)");
+    add_debugger_command("sc", &stack_trace,
+        "Stack crawl for current thread (or any other)");
+    add_debugger_command("iframe", &dump_iframes,
+        "Dump iframes for the specified thread");
+    add_debugger_command("call", &show_call, "Show call with arguments");
+    add_debugger_command_etc("in_context", &cmd_in_context,
+        "Executes a command in the context of a given thread",
+        "<thread ID> <command> ...\n"
+        "Executes a command in the context of a given thread.\n",
+        B_KDEBUG_DONT_PARSE_ARGUMENTS);
+
+    return B_NO_ERROR;
+}
+
+int32
+arch_debug_get_stack_trace(addr_t* returnAddresses, int32 maxCount,
+    int32 skipIframes, int32 skipFrames, uint32 flags)
+{
+    // Keep skipping normal stack frames until we've skipped the iframes we're
+    // supposed to skip.
+    if (skipIframes > 0)
+        skipFrames = INT_MAX;
+
+    struct thread* thread = thread_get_current_thread();
+    int32 count = 0;
+    addr_t rbp = x86_64_read_rbp();
+    bool onKernelStack = true;
+
+    while (rbp != 0 && count < maxCount) {
+        onKernelStack = onKernelStack
+            && is_kernel_stack_address(thread, rbp);
+        if (!onKernelStack && (flags & STACK_TRACE_USER) == 0)
+            break;
+
+        addr_t rip;
+        addr_t nextRbp;
+
+        if (onKernelStack && is_iframe(thread, rbp)) {
+            struct iframe *frame = (struct iframe*)rbp;
+            rip = frame->rip;
+            nextRbp = frame->rbp;
+
+            if (skipIframes > 0) {
+                if (--skipIframes == 0)
+                    skipFrames = 0;
+            }
+        } else {
+            if (get_next_frame_no_debugger(rbp, &nextRbp, &rip) != B_OK)
+                break;
+        }
+
+        if (skipFrames <= 0
+            && ((flags & STACK_TRACE_KERNEL) != 0 || onKernelStack)) {
+            returnAddresses[count++] = rip;
+        } else
+            skipFrames--;
+
+        rbp = nextRbp;
+    }
+
+    return count;
+}

src/system/kernel/arch/x86_64/arch_cpu.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2007, François Revol, revol@free.fr.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+
+#include <cpu.h>
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <ACPI.h>
+
+#include <boot_device.h>
+#include <commpage.h>
+#include <debug.h>
+#include <elf.h>
+#include <smp.h>
+#include <tls.h>
+#include <vm/vm.h>
+#include <vm/vm_types.h>
+#include <vm/VMAddressSpace.h>
+
+#include <arch_system_info.h>
+//#include <arch/x86/selector.h>
+#include <boot/kernel_args.h>
+
+#include "interrupts.h"
+
+
+#define CR0_CACHE_DISABLE       (1UL << 30)
+#define CR0_NOT_WRITE_THROUGH       (1UL << 29)
+#define CR0_FPU_EMULATION       (1UL << 2)
+#define CR0_MONITOR_FPU         (1UL << 1)
+
+#define CR4_OS_FXSR         (1UL << 9)
+#define CR4_OS_XMM_EXCEPTION        (1UL << 10)
+
+struct set_mtrrs_parameter {
+    const x86_64_mtrr_info*     infos;
+    uint32              count;
+    uint8               defaultType;
+};
+
+
+void (*gX86SwapFPUFunc)(void *oldState, const void *newState);
+
+segment_descriptor *gGDT = NULL;
+
+static uint32 sCpuRendezvous;
+static uint32 sCpuRendezvous2;
+static uint32 sCpuRendezvous3;
+
+/* Some specials for the double fault handler */
+static uint8* sDoubleFaultStacks;
+static const size_t kDoubleFaultStackSize = 4096;   // size per CPU
+
+static x86_64_cpu_module_info *sCpuModule;
+
+
+extern "C" void memcpy_generic(void* dest, const void* source, size_t count);
+extern int memcpy_generic_end;
+extern "C" void memset_generic(void* dest, int value, size_t count);
+extern int memset_generic_end;
+
+
+x86_64_optimized_functions gOptimizedFunctions = {
+    memcpy_generic,
+    &memcpy_generic_end,
+    memset_generic,
+    &memset_generic_end
+};
+
+
+void*
+x86_64_get_double_fault_stack(int32 cpu, size_t* _size)
+{
+    *_size = kDoubleFaultStackSize;
+    return sDoubleFaultStacks + kDoubleFaultStackSize * cpu;
+}
+
+
+/*! Returns the index of the current CPU. Can only be called from the double
+    fault handler.
+*/
+int64
+x86_64_double_fault_get_cpu(void)
+{
+    uint64 stack = x86_64_read_rbp();
+    return (stack - (uint64)sDoubleFaultStacks) / kDoubleFaultStackSize;
+}
+
+
+void*
+x86_64_set_tss_and_kstack(addr_t kstack)
+{
+    get_cpu_struct()->arch.tss.sp0 = kstack;
+    return (void*)kstack;
+}
+
+
+/* TODO: Check code. Fix eax names and bitwidths  */
+static int
+detect_cpu(int currentCPU)
+{
+    cpu_ent *cpu = get_cpu_struct();
+    char vendorString[17];
+    cpuid_info cpuid;
+
+    // clear out the cpu info data
+    cpu->arch.vendor = VENDOR_UNKNOWN;
+    cpu->arch.vendor_name = "UNKNOWN VENDOR";
+    cpu->arch.feature[FEATURE_COMMON] = 0;
+    cpu->arch.feature[FEATURE_EXT] = 0;
+    cpu->arch.feature[FEATURE_EXT_AMD] = 0;
+    cpu->arch.model_name[0] = 0;
+
+    // print some fun data
+    get_current_cpuid(&cpuid, 0);
+
+    // build the vendor string
+    memset(vendorString, 0, sizeof(vendorString));
+    memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id));
+
+    // get the family, model, stepping
+    get_current_cpuid(&cpuid, 1);
+    cpu->arch.type = cpuid.eax_1.type;
+    cpu->arch.family = cpuid.eax_1.family;
+    cpu->arch.extended_family = cpuid.eax_1.extended_family;
+    cpu->arch.model = cpuid.eax_1.model;
+    cpu->arch.extended_model = cpuid.eax_1.extended_model;
+    cpu->arch.stepping = cpuid.eax_1.stepping;
+    dprintf("CPU %d: type %d family %d extended_family %d model %d "
+        "extended_model %d stepping %d, string '%s'\n",
+        currentCPU, cpu->arch.type, cpu->arch.family,
+        cpu->arch.extended_family, cpu->arch.model,
+        cpu->arch.extended_model, cpu->arch.stepping, vendorString);
+
+    // figure out what vendor we have here
+
+    for (int32 i = 0; i < VENDOR_NUM; i++) {
+        if (vendor_info[i].ident_string[0]
+            && !strcmp(vendorString, vendor_info[i].ident_string[0])) {
+            cpu->arch.vendor = (x86_vendors)i;
+            cpu->arch.vendor_name = vendor_info[i].vendor;
+            break;
+        }
+        if (vendor_info[i].ident_string[1]
+            && !strcmp(vendorString, vendor_info[i].ident_string[1])) {
+            cpu->arch.vendor = (x86_vendors)i;
+            cpu->arch.vendor_name = vendor_info[i].vendor;
+            break;
+        }
+    }
+
+    // see if we can get the model name
+    get_current_cpuid(&cpuid, 0x80000000);
+    if (cpuid.eax_0.max_eax >= 0x80000004) {
+        // build the model string (need to swap ecx/edx data before copying)
+        unsigned int temp;
+        memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name));
+
+        get_current_cpuid(&cpuid, 0x80000002);
+        temp = cpuid.regs.edx;
+        cpuid.regs.edx = cpuid.regs.ecx;
+        cpuid.regs.ecx = temp;
+        memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars));
+
+        get_current_cpuid(&cpuid, 0x80000003);
+        temp = cpuid.regs.edx;
+        cpuid.regs.edx = cpuid.regs.ecx;
+        cpuid.regs.ecx = temp;
+        memcpy(cpu->arch.model_name + 16, cpuid.as_chars,
+            sizeof(cpuid.as_chars));
+
+        get_current_cpuid(&cpuid, 0x80000004);
+        temp = cpuid.regs.edx;
+        cpuid.regs.edx = cpuid.regs.ecx;
+        cpuid.regs.ecx = temp;
+        memcpy(cpu->arch.model_name + 32, cpuid.as_chars,
+            sizeof(cpuid.as_chars));
+
+        // some cpus return a right-justified string
+        int32 i = 0;
+        while (cpu->arch.model_name[i] == ' ')
+            i++;
+        if (i > 0) {
+            memmove(cpu->arch.model_name, &cpu->arch.model_name[i],
+                strlen(&cpu->arch.model_name[i]) + 1);
+        }
+
+        dprintf("CPU %d: vendor '%s' model name '%s'\n",
+            currentCPU, cpu->arch.vendor_name, cpu->arch.model_name);
+    } else {
+        strcpy(cpu->arch.model_name, "unknown");
+    }
+
+    // load feature bits
+    get_current_cpuid(&cpuid, 1);
+    cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx
+    cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx
+    if (cpu->arch.vendor == VENDOR_AMD) {
+        get_current_cpuid(&cpuid, 0x80000001);
+        cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx
+    }
+
+#if DUMP_FEATURE_STRING
+    dump_feature_string(currentCPU, cpu);
+#endif
+
+    return 0;
+}
+
+/*
+static void
+init_double_fault(int cpuNum)
+{
+    // set up the double fault TSS
+    struct tss *tss = &gCPU[cpuNum].arch.double_fault_tss;
+
+    memset(tss, 0, sizeof(struct tss));
+    size_t stackSize;
+    tss->sp0 = (uint64)x86_64_get_double_fault_stack(cpuNum, &stackSize);
+    tss->sp0 += stackSize;
+    tss->ss0 = KERNEL_DATA_SEG;
+    read_cr3(tss->cr3);
+        // copy the current cr3 to the double fault cr3
+    tss->rip = (uint64)&double_fault;
+    tss->es = KERNEL_DATA_SEG;
+    tss->cs = KERNEL_CODE_SEG;
+    tss->ss = KERNEL_DATA_SEG;
+    tss->rsp = tss->sp0;
+    tss->ds = KERNEL_DATA_SEG;
+    tss->fs = KERNEL_DATA_SEG;
+    tss->gs = KERNEL_DATA_SEG;
+    tss->ldt_seg_selector = 0;
+    tss->io_map_base = sizeof(struct tss);
+
+    // add TSS descriptor for this new TSS
+    uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum;
+    set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex],
+        (addr_t)tss, sizeof(struct tss));
+
+    x86_64_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3);
+}
+*/
+
+status_t 
+arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu)
+{
+    x86_64_write_cr0(x86_64_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
+    gX86SwapFPUFunc = x86_64_fnsave_swap;
+#if 0   
+/* TODO: Once x86_64 kernel is more stable enable SMP code */
+    // On SMP system we want to synchronize the CPUs' TSCs, so system_time()
+    // will return consistent values.
+    if (smp_get_num_cpus() > 1) {
+        // let the first CPU prepare the rendezvous point
+        if (cpu == 0)
+            sTSCSyncRendezvous = smp_get_num_cpus() - 1;
+
+        // One CPU after the other will drop out of this loop and be caught by
+        // the loop below, until the last CPU (0) gets there. Save for +/- a few
+        // cycles the CPUs should pass the second loop at the same time.
+        while (sTSCSyncRendezvous != cpu) {
+        }
+
+        sTSCSyncRendezvous = cpu - 1;
+
+        while (sTSCSyncRendezvous != -1) {
+        }
+
+        // reset TSC to 0
+        x86_write_msr(IA32_MSR_TSC, 0);
+    }
+#endif
+    return B_OK;
+}
+
+
+/*! Disable CPU caches, and invalidate them. */
+static void
+disable_caches()
+{
+    x86_64_write_cr0((x86_64_read_cr0() | CR0_CACHE_DISABLE)
+        & ~CR0_NOT_WRITE_THROUGH);
+    wbinvd();
+    arch_cpu_global_TLB_invalidate();
+}
+
+
+/*! Invalidate CPU caches, and enable them. */
+static void
+enable_caches()
+{
+    wbinvd();
+    arch_cpu_global_TLB_invalidate();
+    x86_64_write_cr0(x86_64_read_cr0()
+        & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH));
+}
+
+
+
+static void
+init_mtrrs(void *_unused, int cpu)
+{
+    // wait until all CPUs have arrived here
+    smp_cpu_rendezvous(&sCpuRendezvous, cpu);
+
+    // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
+    // that initiated the call_all_cpus() from doing that again and clearing
+    // sCpuRendezvous2 before the last CPU has actually left the loop in
+    // smp_cpu_rendezvous();
+    if (cpu == 0)
+        atomic_set((vint32*)&sCpuRendezvous3, 0);
+
+    disable_caches();
+
+    sCpuModule->init_mtrrs();
+
+    enable_caches();
+
+    // wait until all CPUs have arrived here
+    smp_cpu_rendezvous(&sCpuRendezvous2, cpu);
+    smp_cpu_rendezvous(&sCpuRendezvous3, cpu);
+}
+
+
+uint32
+x86_64_count_mtrrs(void)
+{
+    if (sCpuModule == NULL)
+        return 0;
+
+    return sCpuModule->count_mtrrs();
+}
+
+
+extern "C" void
+init_sse(void)
+{
+    // enable OS support for SSE
+    x86_64_write_cr4(x86_64_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION);
+    x86_64_write_cr0(x86_64_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
+
+    gX86SwapFPUFunc = x86_64_fxsave_swap;
+}
+
+
+static void
+load_tss(int cpu)
+{
+    short seg = ((TSS_BASE_SEGMENT + cpu) << 3) | DPL_KERNEL;
+    asm("movw  %0, %%ax;"
+        "ltr %%ax;" : : "r" (seg) : "rax");
+}
+
+
+status_t 
+arch_cpu_init_percpu(kernel_args *args, int cpuid)
+{
+    detect_cpu(cpuid);
+
+    // load the TSS for this cpu
+    // note the main cpu gets initialized in arch_cpu_init_post_vm()
+    if (cpuid != 0) {
+        load_tss(cpuid);
+
+        // set the IDT
+        struct {
+            uint16  limit;
+            void*   address;    //TODO: Should GDT/IDT have 64bit address or 32bit address in longmode
+        } _PACKED descriptor = {
+            256 * 8 - 1,    // 256 descriptors, 8 bytes each (-1 for "limit")
+            x86_64_get_idt(cpuid)
+        };
+
+        asm volatile("lidt  %0" : : "m"(descriptor));
+    }
+
+    return 0;
+}
+
+
+status_t
+arch_cpu_init(kernel_args *args)
+{
+    // init the TSC -> system_time() conversion factors
+
+    uint32 conversionFactor = args->arch_args.system_time_cv_factor;
+    uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000;
+
+
+    if (conversionFactorNsecs >> 32 != 0) {
+        // the TSC frequency is < 1 GHz, which forces us to shift the factor
+        __x86_64_setup_system_time(conversionFactor, conversionFactorNsecs >> 16,
+            true);
+    } else {
+        // the TSC frequency is >= 1 GHz
+        __x86_64_setup_system_time(conversionFactor, conversionFactorNsecs, false);
+    }
+
+    return B_OK;
+}
+
+
+status_t
+arch_cpu_init_post_vm(kernel_args *args)
+{
+/*  uint32 i;
+
+    // account for the segment descriptors
+    gGDT = (segment_descriptor *)args->arch_args.vir_gdt;
+    create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE,
+        B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
+
+    // currently taken out of the build, because it's not yet used (and assumes
+    // (a fixed number of used GDT entries)
+    //i386_selector_init(gGDT);  // pass the new gdt
+
+    // allocate an area for the double fault stacks
+    virtual_address_restrictions virtualRestrictions = {};
+    virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
+    physical_address_restrictions physicalRestrictions = {};
+    create_area_etc(B_SYSTEM_TEAM, "double fault stacks",
+        kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK,
+        B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT,
+        &virtualRestrictions, &physicalRestrictions,
+        (void**)&sDoubleFaultStacks);
+
+    X86PagingStructures* kernelPagingStructures
+        = static_cast<X86VMTranslationMap*>(
+            VMAddressSpace::Kernel()->TranslationMap())->PagingStructures();
+
+    // setup task-state segments
+    for (i = 0; i < args->num_cpus; i++) {
+        // initialize the regular and double fault tss stored in the per-cpu
+        // structure
+        memset(&gCPU[i].arch.tss, 0, sizeof(struct tss));
+        gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG;
+        gCPU[i].arch.tss.io_map_base = sizeof(struct tss);
+
+        // add TSS descriptor for this new TSS
+        set_tss_descriptor(&gGDT[TSS_BASE_SEGMENT + i],
+            (addr_t)&gCPU[i].arch.tss, sizeof(struct tss));
+
+        // initialize the double fault tss
+        init_double_fault(i);
+
+        // init active translation map
+        gCPU[i].arch.active_paging_structures = kernelPagingStructures;
+        kernelPagingStructures->AddReference();
+    }
+
+    // set the current hardware task on cpu 0
+    load_tss(0);
+
+    // setup TLS descriptors (one for every CPU)
+
+    for (i = 0; i < args->num_cpus; i++) {
+        set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE,
+            DT_DATA_WRITEABLE, DPL_USER);
+    }
+
+    // setup SSE2/3 support
+    init_sse();
+
+*/  return B_OK;
+}
+
+status_t
+arch_cpu_init_post_modules(kernel_args *args)
+{
+    // initialize CPU module
+
+    void *cookie = open_module_list("cpu");
+
+    while (true) {
+        char name[B_FILE_NAME_LENGTH];
+        size_t nameLength = sizeof(name);
+
+        if (read_next_module_name(cookie, name, &nameLength) != B_OK
+            || get_module(name, (module_info **)&sCpuModule) == B_OK)
+            break;
+    }
+
+    close_module_list(cookie);
+
+    // initialize MTRRs if available
+    if (x86_64_count_mtrrs() > 0) {
+        sCpuRendezvous = sCpuRendezvous2 = 0;
+        call_all_cpus(&init_mtrrs, NULL);
+    }
+
+    // get optimized functions from the CPU module
+    if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) {
+        x86_64_optimized_functions functions;
+        memset(&functions, 0, sizeof(functions));
+
+        sCpuModule->get_optimized_functions(&functions);
+
+        if (functions.memcpy != NULL) {
+            gOptimizedFunctions.memcpy = functions.memcpy;
+            gOptimizedFunctions.memcpy_end = functions.memcpy_end;
+        }
+
+        if (functions.memset != NULL) {
+            gOptimizedFunctions.memset = functions.memset;
+            gOptimizedFunctions.memset_end = functions.memset_end;
+        }
+    }
+
+    // put the optimized functions into the commpage
+    size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end
+        - (addr_t)gOptimizedFunctions.memcpy;
+    fill_commpage_entry(COMMPAGE_ENTRY_X86_64_MEMCPY,
+        (const void*)gOptimizedFunctions.memcpy, memcpyLen);
+    size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end
+        - (addr_t)gOptimizedFunctions.memset;
+    fill_commpage_entry(COMMPAGE_ENTRY_X86_64_MEMSET,
+        (const void*)gOptimizedFunctions.memset, memsetLen);
+
+    // add the functions to the commpage image
+    image_id image = get_commpage_image();
+    elf_add_memory_image_symbol(image, "commpage_memcpy",
+        ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_64_MEMCPY], memcpyLen,
+        B_SYMBOL_TYPE_TEXT);
+    elf_add_memory_image_symbol(image, "commpage_memset",
+        ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_64_MEMSET], memsetLen,
+        B_SYMBOL_TYPE_TEXT);
+
+    return B_OK;
+}
+
+
+void 
+arch_cpu_sync_icache(void *address, size_t len)
+{
+    // instruction cache is always consistent on x86 and x86_64
+}
+
+
+void
+arch_cpu_memory_read_barrier(void)
+{
+    asm volatile ("lock;" : : : "memory");
+    asm volatile ("add $0, 0(%%rsp);" : : : "memory");
+}
+
+
+void
+arch_cpu_memory_write_barrier(void)
+{
+    asm volatile ("lock;" : : : "memory");
+    asm volatile ("add $0, 0(%%rsp);" : : : "memory");
+}
+
+
+void 
+arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
+{
+    int64 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE;
+    while (num_pages-- >= 0) {
+        invalidate_TLB(start);
+        start += B_PAGE_SIZE;
+    }
+}
+
+
+void 
+arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
+{
+    int i;
+    for (i = 0; i < num_pages; i++) {
+        invalidate_TLB(pages[i]);
+    }
+}
+
+
+void 
+arch_cpu_global_TLB_invalidate(void)
+{
+    uint32 flags = x86_64_read_cr4();
+
+    if (flags & IA32_CR4_GLOBAL_PAGES) {
+        // disable and reenable the global pages to flush all TLBs regardless
+        // of the global page bit
+        x86_64_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES);
+        x86_64_write_cr4(flags | IA32_CR4_GLOBAL_PAGES);
+    } else {
+        cpu_status state = disable_interrupts();
+        arch_cpu_user_TLB_invalidate();
+        restore_interrupts(state);
+    }
+}
+
+
+ssize_t
+arch_cpu_user_strlcpy(char *to, const char *from, size_t size, addr_t *faultHandler)
+{
+    int fromLength = 0;
+    addr_t oldFaultHandler = *faultHandler;
+
+    // this check is to trick the gcc4 compiler and have it keep the error label
+    if (to == NULL && size > 0)
+        goto error;
+
+    *faultHandler = (addr_t)&&error;
+
+    if (size > 0) {
+        to[--size] = '\0';
+        // copy
+        for ( ; size; size--, fromLength++, to++, from++) {
+            if ((*to = *from) == '\0')
+                break;
+        }
+    }
+    // count any leftover from chars
+    while (*from++ != '\0') {
+        fromLength++;
+    }
+
+    *faultHandler = oldFaultHandler;
+    return fromLength;
+
+error:
+    *faultHandler = oldFaultHandler;
+    return B_BAD_ADDRESS;
+}
+
+
+status_t
+arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler)
+{
+    char *xs = (char *)s;
+    addr_t oldFaultHandler = *faultHandler;
+
+    // this check is to trick the gcc4 compiler and have it keep the error label
+    if (s == NULL)
+        goto error;
+
+    *faultHandler = (addr_t)&&error;
+
+    while (count--)
+        *xs++ = c;
+
+    *faultHandler = oldFaultHandler;
+    return 0;
+
+error:
+    *faultHandler = oldFaultHandler;
+    return B_BAD_ADDRESS;
+}
+
+
+static status_t
+acpi_shutdown(bool reboot)
+{
+    if (debug_debugger_running() || !are_interrupts_enabled())
+        return B_ERROR;
+
+    acpi_module_info* acpi;
+    if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK)
+        return B_NOT_SUPPORTED;
+
+    status_t status;
+    if (reboot) {
+        status = acpi->reboot();
+    } else {
+        status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0);
+        if (status == B_OK) {
+            //cpu_status state = disable_interrupts();
+            status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF);
+            //restore_interrupts(state);
+        }
+    }
+
+    put_module(B_ACPI_MODULE_NAME);
+    return status;
+}
+
+
+status_t
+arch_cpu_shutdown(bool reboot)
+{
+    // Make sure we run on the boot CPU (apparently needed for ACPI)
+    _user_set_cpu_enabled(0, true);
+    for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) {
+        _user_set_cpu_enabled(cpu, false);
+    }
+    thread_yield(true);
+
+    if (acpi_shutdown(reboot) == B_OK)
+        return B_OK;
+
+    /* TODO: once APM code is in place shutdown apm here if !reboot */
+
+    cpu_status state = disable_interrupts();
+
+    // try to reset the system using the keyboard controller
+    out8(0xfe, 0x64);
+
+    // Give some time to the controller to do its job (0.5s)
+    snooze(500000);
+
+    // if that didn't help, try it this way
+//  reboot();   /* TODO: write a function for arch_x86_64.S */
+
+    restore_interrupts(state);
+    return B_ERROR;
+}
+
+
+void
+arch_cpu_idle(void)
+{
+    asm("hlt");
+}
+
+
+static void
+set_mtrrs(void* _parameter, int cpu)
+{
+    set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter;
+
+    // wait until all CPUs have arrived here
+    smp_cpu_rendezvous(&sCpuRendezvous, cpu);
+
+    // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
+    // that initiated the call_all_cpus() from doing that again and clearing
+    // sCpuRendezvous2 before the last CPU has actually left the loop in
+    // smp_cpu_rendezvous();
+    if (cpu == 0)
+        atomic_set((vint32*)&sCpuRendezvous3, 0);
+
+    disable_caches();
+
+    sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos,
+        parameter->count);
+
+    enable_caches();
+
+    // wait until all CPUs have arrived here
+    smp_cpu_rendezvous(&sCpuRendezvous2, cpu);
+    smp_cpu_rendezvous(&sCpuRendezvous3, cpu);
+}
+
+
+void
+x86_64_set_mtrrs(uint8 defaultType, const x86_64_mtrr_info* infos, uint32 count)
+{
+    if (sCpuModule == NULL)
+        return;
+
+    struct set_mtrrs_parameter parameter;
+    parameter.defaultType = defaultType;
+    parameter.infos = infos;
+    parameter.count = count;
+
+    sCpuRendezvous = sCpuRendezvous2 = 0;
+    call_all_cpus(&set_mtrrs, &parameter);
+}

src/system/kernel/arch/x86_64/interrupts.h

@@ +1 @@
+/*
+ * Copyright 2005-2006, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+#ifndef _KERNEL_ARCH_x86_64_INTERRUPTS_H
+#define _KERNEL_ARCH_x86_64_INTERRUPTS_H
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void trap0();void trap1();void trap2();void trap3();void trap4();void trap5();
+void trap6();void trap7();void trap9();void trap10();void trap11();
+void trap12();void trap13();void trap14();void trap16();void trap17();
+void trap18();void trap19();
+
+void trap32();void trap33();void trap34();void trap35();void trap36();
+void trap37();void trap38();void trap39();void trap40();void trap41();
+void trap42();void trap43();void trap44();void trap45();void trap46();
+void trap47();void trap48();void trap49();void trap50();void trap51();
+void trap52();void trap53();void trap54();void trap55();void trap56();
+void trap57();void trap58();void trap59();void trap60();void trap61();
+void trap62();void trap63();void trap64();void trap65();void trap66();
+void trap67();void trap68();void trap69();void trap70();void trap71();
+void trap72();void trap73();void trap74();void trap75();void trap76();
+void trap77();void trap78();void trap79();void trap80();void trap81();
+void trap82();void trap83();void trap84();void trap85();void trap86();
+void trap87();void trap88();void trap89();void trap90();void trap91();
+void trap92();void trap93();void trap94();void trap95();void trap96();
+void trap97();
+
+void double_fault();    // int 8
+void trap14_double_fault();
+
+void trap98();
+void trap99();
+
+void trap100();void trap101();void trap102();void trap103();void trap104();
+void trap105();void trap106();void trap107();void trap108();void trap109();
+void trap110();void trap111();void trap112();void trap113();void trap114();
+void trap115();void trap116();void trap117();void trap118();void trap119();
+void trap120();void trap121();void trap122();void trap123();void trap124();
+void trap125();void trap126();void trap127();void trap128();void trap129();
+void trap130();void trap131();void trap132();void trap133();void trap134();
+void trap135();void trap136();void trap137();void trap138();void trap139();
+void trap140();void trap141();void trap142();void trap143();void trap144();
+void trap145();void trap146();void trap147();void trap148();void trap149();
+void trap150();void trap151();void trap152();void trap153();void trap154();
+void trap155();void trap156();void trap157();void trap158();void trap159();
+void trap160();void trap161();void trap162();void trap163();void trap164();
+void trap165();void trap166();void trap167();void trap168();void trap169();
+void trap170();void trap171();void trap172();void trap173();void trap174();
+void trap175();void trap176();void trap177();void trap178();void trap179();
+void trap180();void trap181();void trap182();void trap183();void trap184();
+void trap185();void trap186();void trap187();void trap188();void trap189();
+void trap190();void trap191();void trap192();void trap193();void trap194();
+void trap195();void trap196();void trap197();void trap198();void trap199();
+void trap200();void trap201();void trap202();void trap203();void trap204();
+void trap205();void trap206();void trap207();void trap208();void trap209();
+void trap210();void trap211();void trap212();void trap213();void trap214();
+void trap215();void trap216();void trap217();void trap218();void trap219();
+void trap220();void trap221();void trap222();void trap223();void trap224();
+void trap225();void trap226();void trap227();void trap228();void trap229();
+void trap230();void trap231();void trap232();void trap233();void trap234();
+void trap235();void trap236();void trap237();void trap238();void trap239();
+void trap240();void trap241();void trap242();void trap243();void trap244();
+void trap245();void trap246();void trap247();void trap248();void trap249();
+void trap250();
+
+void trap251();void trap252();void trap253();void trap254();void trap255();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  /* _KERNEL_ARCH_x86_64_INTERRUPTS_H */

src/system/kernel/arch/x86_64/paging/X86_64VMTranslationMap.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+
+#include "paging/X86_64VMTranslationMap.h"
+
+#include <int.h>
+#include <slab/Slab.h>
+#include <thread.h>
+#include <util/AutoLock.h>
+#include <vm/vm_page.h>
+#include <vm/VMAddressSpace.h>
+#include <vm/VMCache.h>
+
+#include "paging/X86_64PagingMethod.h"
+#include "paging/X86_64PagingStructures.h"
+#include "paging/x86_64_physical_page_mapper.h"
+
+
+//#define TRACE_X86_64_VM_TRANSLATION_MAP
+#ifdef TRACE_X86_64_VM_TRANSLATION_MAP
+#   define TRACE(x...) dprintf(x)
+#else
+#   define TRACE(x...) ;
+#endif
+
+
+X86_64VMTranslationMap::X86_64VMTranslationMap()
+    :
+    fPagingStructures(NULL)
+{
+}
+
+
+X86_64VMTranslationMap::~X86_64VMTranslationMap()
+{
+    if (fPagingStructures == NULL)
+        return;
+
+    if (fPageMapper != NULL)
+        fPageMapper->Delete();
+
+    // cycle through and free all of the user space page tables
+
+    STATIC_ASSERT(KERNEL_BASE == 0x80000000 && KERNEL_SIZE == 0x80000000);
+        // assuming 1-1 split of the address space
+
+    for (uint32 k = 0; k < 2; k++) {
+        page_directory_entry* pageDir
+            = fPagingStructures->VirtualPageDirs()[k];
+        if (pageDir == NULL)
+            continue;
+
+        for (uint32 i = 0; i < kPageDirEntryCount; i++) {
+            if ((pageDir[i] & X86_64_PDE_PRESENT) != 0) {
+                phys_addr_t address = pageDir[i] & X86_64_PDE_ADDRESS_MASK;
+                vm_page* page = vm_lookup_page(address / B_PAGE_SIZE);
+                if (page == NULL)
+                    panic("X86_64VMTranslationMap::~X86_64VMTranslationMap: "
+                        "didn't find page table page: page address: %#"
+                        B_PRIxPHYSADDR ", virtual base: %#" B_PRIxADDR "\n",
+                        address,
+                        (k * kPageDirEntryCount + i) * kPageTableRange);
+                DEBUG_PAGE_ACCESS_START(page);
+                vm_page_set_state(page, PAGE_STATE_FREE);
+            }
+        }
+    }
+
+    fPagingStructures->RemoveReference();
+}
+
+
+status_t
+X86_64VMTranslationMap::Init(bool kernel)
+{
+    TRACE("X86_64VMTranslationMap::Init()\n");
+
+    X86_64VMTranslationMap::Init(kernel);
+
+    fPagingStructures = new(std::nothrow) X86_64PagingStructures;
+    if (fPagingStructures == NULL)
+        return B_NO_MEMORY;
+
+    X86_64PagingMethod* method = X86_64PagingMethod::Method();
+
+    if (kernel) {
+        // kernel
+        // get the physical page mapper
+        fPageMapper = method->KernelPhysicalPageMapper();
+
+        // we already know the kernel pgdir mapping
+        fPagingStructures->Init(method->KernelVirtualPageDirPointerTable(),
+            method->KernelPhysicalPageDirPointerTable(), NULL,
+            method->KernelVirtualPageDirs(), method->KernelPhysicalPageDirs());
+    } else {
+        // user
+        // allocate a physical page mapper
+        status_t error = method->PhysicalPageMapper()
+            ->CreateTranslationMapPhysicalPageMapper(&fPageMapper);
+        if (error != B_OK)
+            return error;
+
+        // The following code assumes that the kernel address space occupies the
+        // upper half of the virtual address space. This simplifies things a
+        // lot, since it allows us to just use the upper two page directories
+        // of the kernel and create two new lower page directories for the
+        // userland.
+        STATIC_ASSERT(KERNEL_BASE == 0x80000000 && KERNEL_SIZE == 0x80000000);
+
+        // allocate the page directories (both at once)
+        page_directory_entry* virtualPageDirs[4];
+        phys_addr_t physicalPageDirs[4];
+        virtualPageDirs[0] = (page_directory_entry*)memalign(B_PAGE_SIZE,
+            2 * B_PAGE_SIZE);
+        if (virtualPageDirs[0] == NULL)
+            return B_NO_MEMORY;
+        virtualPageDirs[1] = virtualPageDirs[0] + kPageTableEntryCount;
+
+        // clear the userland page directories
+        memset(virtualPageDirs[0], 0, 2 * B_PAGE_SIZE);
+
+        // use the upper two kernel page directories
+        for (int32 i = 2; i < 4; i++) {
+            virtualPageDirs[i] = method->KernelVirtualPageDirs()[i];
+            physicalPageDirs[i] = method->KernelPhysicalPageDirs()[i];
+        }
+
+        // look up the page directories' physical addresses
+        for (int32 i = 0; i < 2; i++) {
+            vm_get_page_mapping(VMAddressSpace::KernelID(),
+                (addr_t)virtualPageDirs[i], &physicalPageDirs[i]);
+        }
+
+        // allocate the PDPT -- needs to have a 32 bit physical address
+        phys_addr_t physicalPDPT;
+        void* pdptHandle;
+        page_directory_pointer_table_entry* pdpt
+            = (page_directory_pointer_table_entry*)
+                method->Allocate32BitPage(physicalPDPT, pdptHandle);
+        if (pdpt == NULL)
+            free(virtualPageDirs[0]);
+
+        // init the PDPT entries
+        for (int32 i = 0; i < 4; i++) {
+            pdpt[i] = (physicalPageDirs[i] & X86_64_PDPTE_ADDRESS_MASK)
+                | X86_64_PDPTE_PRESENT;
+        }
+
+        // init the paging structures
+        fPagingStructures->Init(pdpt, physicalPDPT, pdptHandle, virtualPageDirs,
+            physicalPageDirs);
+    }
+
+    return B_OK;
+}
+
+
+size_t
+X86_64VMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const
+{
+    // If start == 0, the actual base address is not yet known to the caller and
+    // we shall assume the worst case.
+    if (start == 0) {
+        // offset the range so it has the worst possible alignment
+        start = kPageTableRange - B_PAGE_SIZE;
+        end += kPageTableRange - B_PAGE_SIZE;
+    }
+
+    return end / kPageTableRange + 1 - start / kPageTableRange;
+}
+
+
+status_t
+X86_64VMTranslationMap::Map(addr_t virtualAddress, phys_addr_t physicalAddress,
+    uint32 attributes, uint32 memoryType, vm_page_reservation* reservation)
+{
+    TRACE("X86_64VMTranslationMap::Map(): %#" B_PRIxADDR " -> %#" B_PRIxPHYSADDR
+        "\n", virtualAddress, physicalAddress);
+
+    // check to see if a page table exists for this range
+    page_directory_entry* pageDirEntry
+        = X86_64PagingMethod::PageDirEntryForAddress(
+            fPagingStructures->VirtualPageDirs(), virtualAddress);
+    if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+        // we need to allocate a page table
+        vm_page *page = vm_page_allocate_page(reservation,
+            PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
+
+        DEBUG_PAGE_ACCESS_END(page);
+
+        phys_addr_t physicalPageTable
+            = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
+
+        TRACE("X86_64VMTranslationMap::Map(): asked for free page for "
+            "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable);
+
+        // put it in the page dir
+        X86_64PagingMethod::PutPageTableInPageDir(pageDirEntry,
+            physicalPageTable,
+            attributes
+                | ((attributes & B_USER_PROTECTION) != 0
+                        ? B_WRITE_AREA : B_KERNEL_WRITE_AREA));
+
+        fMapCount++;
+    }
+
+    // now, fill in the page table entry
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner pinner(thread);
+
+    page_table_entry* pageTable
+        = (page_table_entry*)fPageMapper->GetPageTableAt(
+            *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+    page_table_entry* entry = pageTable
+        + virtualAddress / B_PAGE_SIZE % kPageTableEntryCount;
+
+    ASSERT_PRINT((*entry & X86_64_PTE_PRESENT) == 0,
+        "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64,
+        virtualAddress, *entry);
+
+    X86_64PagingMethod::PutPageTableEntryInTable(entry, physicalAddress,
+        attributes, memoryType, fIsKernelMap);
+
+    pinner.Unlock();
+
+    // Note: We don't need to invalidate the TLB for this address, as previously
+    // the entry was not present and the TLB doesn't cache those entries.
+
+    fMapCount++;
+
+    return 0;
+}
+
+
+status_t
+X86_64VMTranslationMap::Unmap(addr_t start, addr_t end)
+{
+    start = ROUNDDOWN(start, B_PAGE_SIZE);
+    if (start >= end)
+        return B_OK;
+
+    TRACE("X86_64VMTranslationMap::Unmap(): %#" B_PRIxADDR " - %#" B_PRIxADDR
+        "\n", start, end);
+
+    do {
+        page_directory_entry* pageDirEntry
+            = X86_64PagingMethod::PageDirEntryForAddress(
+                fPagingStructures->VirtualPageDirs(), start);
+        if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+            // no page table here, move the start up to access the next page
+            // table
+            start = ROUNDUP(start + 1, kPageTableRange);
+            continue;
+        }
+
+        struct thread* thread = thread_get_current_thread();
+        ThreadCPUPinner pinner(thread);
+
+        page_table_entry* pageTable
+            = (page_table_entry*)fPageMapper->GetPageTableAt(
+                *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+
+        uint32 index = start / B_PAGE_SIZE % kPageTableEntryCount;
+        for (; index < kPageTableEntryCount && start < end;
+                index++, start += B_PAGE_SIZE) {
+            if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) {
+                // page mapping not valid
+                continue;
+            }
+
+            TRACE("X86_64VMTranslationMap::Unmap(): removing page %#"
+                B_PRIxADDR "\n", start);
+
+            page_table_entry oldEntry
+                = X86_64PagingMethod::ClearPageTableEntryFlags(
+                    &pageTable[index], X86_64_PTE_PRESENT);
+            fMapCount--;
+
+            if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
+                // Note, that we only need to invalidate the address, if the
+                // accessed flags was set, since only then the entry could have
+                // been in any TLB.
+                InvalidatePage(start);
+            }
+        }
+    } while (start != 0 && start < end);
+
+    return B_OK;
+}
+
+
+/*! Caller must have locked the cache of the page to be unmapped.
+    This object shouldn't be locked.
+*/
+status_t
+X86_64VMTranslationMap::UnmapPage(VMArea* area, addr_t address,
+    bool updatePageQueue)
+{
+    ASSERT(address % B_PAGE_SIZE == 0);
+
+    page_directory_entry* pageDirEntry
+        = X86_64PagingMethod::PageDirEntryForAddress(
+            fPagingStructures->VirtualPageDirs(), address);
+
+    TRACE("X86_64VMTranslationMap::UnmapPage(%#" B_PRIxADDR ")\n", address);
+
+    RecursiveLocker locker(fLock);
+
+    if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0)
+        return B_ENTRY_NOT_FOUND;
+
+    ThreadCPUPinner pinner(thread_get_current_thread());
+
+    page_table_entry* pageTable
+        = (page_table_entry*)fPageMapper->GetPageTableAt(
+            *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+
+    page_table_entry oldEntry = X86_64PagingMethod::ClearPageTableEntry(
+        &pageTable[address / B_PAGE_SIZE % kPageTableEntryCount]);
+
+    pinner.Unlock();
+
+    if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
+        // page mapping not valid
+        return B_ENTRY_NOT_FOUND;
+    }
+
+    fMapCount--;
+
+    if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
+        // Note, that we only need to invalidate the address, if the
+        // accessed flags was set, since only then the entry could have been
+        // in any TLB.
+        InvalidatePage(address);
+
+        Flush();
+
+        // NOTE: Between clearing the page table entry and Flush() other
+        // processors (actually even this processor with another thread of the
+        // same team) could still access the page in question via their cached
+        // entry. We can obviously lose a modified flag in this case, with the
+        // effect that the page looks unmodified (and might thus be recycled),
+        // but is actually modified.
+        // In most cases this is harmless, but for vm_remove_all_page_mappings()
+        // this is actually a problem.
+        // Interestingly FreeBSD seems to ignore this problem as well
+        // (cf. pmap_remove_all()), unless I've missed something.
+    }
+
+    locker.Detach();
+        // PageUnmapped() will unlock for us
+
+    PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE,
+        (oldEntry & X86_64_PTE_ACCESSED) != 0,
+        (oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue);
+
+    return B_OK;
+}
+
+
+void
+X86_64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size,
+    bool updatePageQueue)
+{
+    if (size == 0)
+        return;
+
+    addr_t start = base;
+    addr_t end = base + size - 1;
+
+    TRACE("X86_64VMTranslationMap::UnmapPages(%p, %#" B_PRIxADDR ", %#"
+        B_PRIxADDR ")\n", area, start, end);
+
+    VMAreaMappings queue;
+
+    RecursiveLocker locker(fLock);
+
+    do {
+        page_directory_entry* pageDirEntry
+            = X86_64PagingMethod::PageDirEntryForAddress(
+                fPagingStructures->VirtualPageDirs(), start);
+        if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+            // no page table here, move the start up to access the next page
+            // table
+            start = ROUNDUP(start + 1, kPageTableRange);
+            continue;
+        }
+
+        struct thread* thread = thread_get_current_thread();
+        ThreadCPUPinner pinner(thread);
+
+        page_table_entry* pageTable
+            = (page_table_entry*)fPageMapper->GetPageTableAt(
+                *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+
+        uint32 index = start / B_PAGE_SIZE % kPageTableEntryCount;
+        for (; index < kPageTableEntryCount && start < end;
+                index++, start += B_PAGE_SIZE) {
+            page_table_entry oldEntry
+                = X86_64PagingMethod::ClearPageTableEntry(&pageTable[index]);
+            if ((oldEntry & X86_64_PTE_PRESENT) == 0)
+                continue;
+
+            fMapCount--;
+
+            if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
+                // Note, that we only need to invalidate the address, if the
+                // accessed flags was set, since only then the entry could have
+                // been in any TLB.
+                InvalidatePage(start);
+            }
+
+            if (area->cache_type != CACHE_TYPE_DEVICE) {
+                // get the page
+                vm_page* page = vm_lookup_page(
+                    (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
+                ASSERT(page != NULL);
+
+                DEBUG_PAGE_ACCESS_START(page);
+
+                // transfer the accessed/dirty flags to the page
+                if ((oldEntry & X86_64_PTE_ACCESSED) != 0)
+                    page->accessed = true;
+                if ((oldEntry & X86_64_PTE_DIRTY) != 0)
+                    page->modified = true;
+
+                // remove the mapping object/decrement the wired_count of the
+                // page
+                if (area->wiring == B_NO_LOCK) {
+                    vm_page_mapping* mapping = NULL;
+                    vm_page_mappings::Iterator iterator
+                        = page->mappings.GetIterator();
+                    while ((mapping = iterator.Next()) != NULL) {
+                        if (mapping->area == area)
+                            break;
+                    }
+
+                    ASSERT(mapping != NULL);
+
+                    area->mappings.Remove(mapping);
+                    page->mappings.Remove(mapping);
+                    queue.Add(mapping);
+                } else
+                    page->DecrementWiredCount();
+
+                if (!page->IsMapped()) {
+                    atomic_add(&gMappedPagesCount, -1);
+
+                    if (updatePageQueue) {
+                        if (page->Cache()->temporary)
+                            vm_page_set_state(page, PAGE_STATE_INACTIVE);
+                        else if (page->modified)
+                            vm_page_set_state(page, PAGE_STATE_MODIFIED);
+                        else
+                            vm_page_set_state(page, PAGE_STATE_CACHED);
+                    }
+                }
+
+                DEBUG_PAGE_ACCESS_END(page);
+            }
+        }
+
+        Flush();
+            // flush explicitly, since we directly use the lock
+    } while (start != 0 && start < end);
+
+    // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
+    // really critical here, as in all cases this method is used, the unmapped
+    // area range is unmapped for good (resized/cut) and the pages will likely
+    // be freed.
+
+    locker.Unlock();
+
+    // free removed mappings
+    bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
+    uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
+        | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
+    while (vm_page_mapping* mapping = queue.RemoveHead())
+        object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+}
+
+
+void
+X86_64VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace,
+    bool ignoreTopCachePageFlags)
+{
+    if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
+        X86_64VMTranslationMap::UnmapPages(area, area->Base(), area->Size(),
+            true);
+        return;
+    }
+
+    bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
+
+    RecursiveLocker locker(fLock);
+
+    VMAreaMappings mappings;
+    mappings.MoveFrom(&area->mappings);
+
+    for (VMAreaMappings::Iterator it = mappings.GetIterator();
+            vm_page_mapping* mapping = it.Next();) {
+        vm_page* page = mapping->page;
+        page->mappings.Remove(mapping);
+
+        VMCache* cache = page->Cache();
+
+        bool pageFullyUnmapped = false;
+        if (!page->IsMapped()) {
+            atomic_add(&gMappedPagesCount, -1);
+            pageFullyUnmapped = true;
+        }
+
+        if (unmapPages || cache != area->cache) {
+            addr_t address = area->Base()
+                + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
+
+            page_directory_entry* pageDirEntry
+                = X86_64PagingMethod::PageDirEntryForAddress(
+                    fPagingStructures->VirtualPageDirs(), address);
+            if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+                panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
+                    "has no page dir entry", page, area, address);
+                continue;
+            }
+
+            ThreadCPUPinner pinner(thread_get_current_thread());
+
+            page_table_entry* pageTable
+                = (page_table_entry*)fPageMapper->GetPageTableAt(
+                    *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+            page_table_entry oldEntry
+                = X86_64PagingMethod::ClearPageTableEntry(
+                    &pageTable[address / B_PAGE_SIZE
+                        % kPageTableEntryCount]);
+
+            pinner.Unlock();
+
+            if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
+                panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
+                    "has no page table entry", page, area, address);
+                continue;
+            }
+
+            // transfer the accessed/dirty flags to the page and invalidate
+            // the mapping, if necessary
+            if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
+                page->accessed = true;
+
+                if (!deletingAddressSpace)
+                    InvalidatePage(address);
+            }
+
+            if ((oldEntry & X86_64_PTE_DIRTY) != 0)
+                page->modified = true;
+
+            if (pageFullyUnmapped) {
+                DEBUG_PAGE_ACCESS_START(page);
+
+                if (cache->temporary)
+                    vm_page_set_state(page, PAGE_STATE_INACTIVE);
+                else if (page->modified)
+                    vm_page_set_state(page, PAGE_STATE_MODIFIED);
+                else
+                    vm_page_set_state(page, PAGE_STATE_CACHED);
+
+                DEBUG_PAGE_ACCESS_END(page);
+            }
+        }
+
+        fMapCount--;
+    }
+
+    Flush();
+        // flush explicitely, since we directly use the lock
+
+    locker.Unlock();
+
+    bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
+    uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
+        | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
+    while (vm_page_mapping* mapping = mappings.RemoveHead())
+        object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+}
+
+
+status_t
+X86_64VMTranslationMap::Query(addr_t virtualAddress,
+    phys_addr_t* _physicalAddress, uint32* _flags)
+{
+    // default the flags to not present
+    *_flags = 0;
+    *_physicalAddress = 0;
+
+    // get the page directory entry
+    page_directory_entry* pageDirEntry
+        = X86_64PagingMethod::PageDirEntryForAddress(
+            fPagingStructures->VirtualPageDirs(), virtualAddress);
+    if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+        // no pagetable here
+        return B_OK;
+    }
+
+    // get the page table entry
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner pinner(thread);
+
+    page_table_entry* pageTable
+        = (page_table_entry*)fPageMapper->GetPageTableAt(
+            *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+    page_table_entry entry
+        = pageTable[virtualAddress / B_PAGE_SIZE % kPageTableEntryCount];
+
+    pinner.Unlock();
+
+    *_physicalAddress = entry & X86_64_PTE_ADDRESS_MASK;
+
+    // translate the page state flags
+    if ((entry & X86_64_PTE_USER) != 0) {
+        *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0)
+            | B_READ_AREA;
+    }
+
+    *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0)
+        | B_KERNEL_READ_AREA
+        | ((entry & X86_64_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0)
+        | ((entry & X86_64_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0)
+        | ((entry & X86_64_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0);
+
+    TRACE("X86_64VMTranslationMap::Query(%#" B_PRIxADDR ") -> %#"
+        B_PRIxPHYSADDR ":\n", *_physicalAddress, virtualAddress);
+
+    return B_OK;
+}
+
+
+status_t
+X86_64VMTranslationMap::QueryInterrupt(addr_t virtualAddress,
+    phys_addr_t* _physicalAddress, uint32* _flags)
+{
+    // default the flags to not present
+    *_flags = 0;
+    *_physicalAddress = 0;
+
+    // get the page directory entry
+    page_directory_entry* pageDirEntry
+        = X86_64PagingMethod::PageDirEntryForAddress(
+            fPagingStructures->VirtualPageDirs(), virtualAddress);
+    if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+        // no pagetable here
+        return B_OK;
+    }
+
+    // get the page table entry
+    page_table_entry* pageTable
+        = (page_table_entry*)X86_64PagingMethod::Method()
+            ->PhysicalPageMapper()->InterruptGetPageTableAt(
+                *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+    page_table_entry entry
+        = pageTable[virtualAddress / B_PAGE_SIZE % kPageTableEntryCount];
+
+    *_physicalAddress = entry & X86_64_PTE_ADDRESS_MASK;
+
+    // translate the page state flags
+    if ((entry & X86_64_PTE_USER) != 0) {
+        *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0)
+            | B_READ_AREA;
+    }
+
+    *_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0)
+        | B_KERNEL_READ_AREA
+        | ((entry & X86_64_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0)
+        | ((entry & X86_64_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0)
+        | ((entry & X86_64_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0);
+
+    TRACE("X86_64VMTranslationMap::Query(%#" B_PRIxADDR ") -> %#"
+        B_PRIxPHYSADDR ":\n", *_physicalAddress, virtualAddress);
+
+    return B_OK;
+}
+
+
+status_t
+X86_64VMTranslationMap::Protect(addr_t start, addr_t end, uint32 attributes,
+    uint32 memoryType)
+{
+    start = ROUNDDOWN(start, B_PAGE_SIZE);
+    if (start >= end)
+        return B_OK;
+
+    TRACE("X86_64VMTranslationMap::Protect(): %#" B_PRIxADDR " - %#" B_PRIxADDR
+        ", attributes: %#" B_PRIx32 "\n", start, end, attributes);
+
+    // compute protection flags
+    uint64 newProtectionFlags = 0;
+    if ((attributes & B_USER_PROTECTION) != 0) {
+        newProtectionFlags = X86_64_PTE_USER;
+        if ((attributes & B_WRITE_AREA) != 0)
+            newProtectionFlags |= X86_64_PTE_WRITABLE;
+    } else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
+        newProtectionFlags = X86_64_PTE_WRITABLE;
+
+    do {
+        page_directory_entry* pageDirEntry
+            = X86_64PagingMethod::PageDirEntryForAddress(
+                fPagingStructures->VirtualPageDirs(), start);
+        if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+            // no page table here, move the start up to access the next page
+            // table
+            start = ROUNDUP(start + 1, kPageTableRange);
+            continue;
+        }
+
+        struct thread* thread = thread_get_current_thread();
+        ThreadCPUPinner pinner(thread);
+
+        page_table_entry* pageTable
+            = (page_table_entry*)fPageMapper->GetPageTableAt(
+                *pageDirEntry & X86_64_PDE_ADDRESS_MASK);
+
+        uint32 index = start / B_PAGE_SIZE % kPageTableEntryCount;
+        for (; index < kPageTableEntryCount && start < end;
+                index++, start += B_PAGE_SIZE) {
+            page_table_entry entry = pageTable[index];
+            if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) {
+                // page mapping not valid
+                continue;
+            }
+
+            TRACE("X86_64VMTranslationMap::Protect(): protect page %#"
+                B_PRIxADDR "\n", start);
+
+            // set the new protection flags -- we want to do that atomically,
+            // without changing the accessed or dirty flag
+            page_table_entry oldEntry;
+            while (true) {
+                oldEntry = X86_64PagingMethod::TestAndSetPageTableEntry(
+                    &pageTable[index],
+                    (entry & ~(X86_64_PTE_PROTECTION_MASK
+                            | X86_64_PTE_MEMORY_TYPE_MASK))
+                        | newProtectionFlags
+                        | X86_64PagingMethod::MemoryTypeToPageTableEntryFlags(
+                            memoryType),
+                    entry);
+                if (oldEntry == entry)
+                    break;
+                entry = oldEntry;
+            }
+
+            if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
+                // Note, that we only need to invalidate the address, if the
+                // accessed flag was set, since only then the entry could have been
+                // in any TLB.
+                InvalidatePage(start);
+            }
+        }
+    } while (start != 0 && start < end);
+
+    return B_OK;
+}
+
+
+status_t
+X86_64VMTranslationMap::ClearFlags(addr_t address, uint32 flags)
+{
+    page_directory_entry* pageDirEntry
+        = X86_64PagingMethod::PageDirEntryForAddress(
+            fPagingStructures->VirtualPageDirs(), address);
+    if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0) {
+        // no pagetable here
+        return B_OK;
+    }
+
+    uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_64_PTE_DIRTY : 0)
+        | ((flags & PAGE_ACCESSED) ? X86_64_PTE_ACCESSED : 0);
+
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner pinner(thread);
+
+    page_table_entry* entry
+        = (page_table_entry*)fPageMapper->GetPageTableAt(
+                *pageDirEntry & X86_64_PDE_ADDRESS_MASK)
+            + address / B_PAGE_SIZE % kPageTableEntryCount;
+
+    // clear out the flags we've been requested to clear
+    page_table_entry oldEntry
+        = X86_64PagingMethod::ClearPageTableEntryFlags(entry, flagsToClear);
+
+    pinner.Unlock();
+
+    if ((oldEntry & flagsToClear) != 0)
+        InvalidatePage(address);
+
+    return B_OK;
+}
+
+
+bool
+X86_64VMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address,
+    bool unmapIfUnaccessed, bool& _modified)
+{
+    ASSERT(address % B_PAGE_SIZE == 0);
+
+    TRACE("X86_64VMTranslationMap32Bit::ClearAccessedAndModified(%#" B_PRIxADDR
+        ")\n", address);
+
+    page_directory_entry* pageDirEntry
+        = X86_64PagingMethod::PageDirEntryForAddress(
+            fPagingStructures->VirtualPageDirs(), address);
+
+    RecursiveLocker locker(fLock);
+
+    if ((*pageDirEntry & X86_64_PDE_PRESENT) == 0)
+        return false;
+
+    ThreadCPUPinner pinner(thread_get_current_thread());
+
+    page_table_entry* entry
+        = (page_table_entry*)fPageMapper->GetPageTableAt(
+                *pageDirEntry & X86_64_PDE_ADDRESS_MASK)
+            + address / B_PAGE_SIZE % kPageTableEntryCount;
+
+    // perform the deed
+    page_table_entry oldEntry;
+
+    if (unmapIfUnaccessed) {
+        while (true) {
+            oldEntry = *entry;
+            if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
+                // page mapping not valid
+                return false;
+            }
+
+            if (oldEntry & X86_64_PTE_ACCESSED) {
+                // page was accessed -- just clear the flags
+                oldEntry = X86_64PagingMethod::ClearPageTableEntryFlags(entry,
+                    X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY);
+                break;
+            }
+
+            // page hasn't been accessed -- unmap it
+            if (X86_64PagingMethod::TestAndSetPageTableEntry(entry, 0, oldEntry)
+                    == oldEntry) {
+                break;
+            }
+
+            // something changed -- check again
+        }
+    } else {
+        oldEntry = X86_64PagingMethod::ClearPageTableEntryFlags(entry,
+            X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY);
+    }
+
+    pinner.Unlock();
+
+    _modified = (oldEntry & X86_64_PTE_DIRTY) != 0;
+
+    if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
+        // Note, that we only need to invalidate the address, if the
+        // accessed flags was set, since only then the entry could have been
+        // in any TLB.
+        InvalidatePage(address);
+        Flush();
+
+        return true;
+    }
+
+    if (!unmapIfUnaccessed)
+        return false;
+
+    // We have unmapped the address. Do the "high level" stuff.
+
+    fMapCount--;
+
+    locker.Detach();
+        // UnaccessedPageUnmapped() will unlock for us
+
+    UnaccessedPageUnmapped(area,
+        (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
+
+    return false;
+}
+
+
+/*! Acquires the map's recursive lock, and resets the invalidate pages counter
+    in case it's the first locking recursion.
+*/
+bool
+X86_64VMTranslationMap::Lock()
+{
+    TRACE("%p->X86_64VMTranslationMap::Lock()\n", this);
+
+    recursive_lock_lock(&fLock);
+    if (recursive_lock_get_recursion(&fLock) == 1) {
+        // we were the first one to grab the lock
+        TRACE("clearing invalidated page count\n");
+        fInvalidPagesCount = 0;
+    }
+
+    return true;
+}
+
+
+/*! Unlocks the map, and, if we are actually losing the recursive lock,
+    flush all pending changes of this map (ie. flush TLB caches as
+    needed).
+*/
+void
+X86_64VMTranslationMap::Unlock()
+{
+    TRACE("%p->X86_64VMTranslationMap::Unlock()\n", this);
+
+    if (recursive_lock_get_recursion(&fLock) == 1) {
+        // we're about to release it for the last time
+        Flush();
+    }
+
+    recursive_lock_unlock(&fLock);
+}
+
+
+addr_t
+X86_64VMTranslationMap::MappedSize() const
+{
+    return fMapCount;
+}
+
+
+void
+X86_64VMTranslationMap::Flush()
+{
+    if (fInvalidPagesCount <= 0)
+        return;
+
+    struct thread* thread = thread_get_current_thread();
+    thread_pin_to_current_cpu(thread);
+
+    if (fInvalidPagesCount > PAGE_INVALIDATE_CACHE_SIZE) {
+        // invalidate all pages
+        TRACE("flush_tmap: %d pages to invalidate, invalidate all\n",
+            fInvalidPagesCount);
+
+        if (fIsKernelMap) {
+            arch_cpu_global_TLB_invalidate();
+            smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
+                NULL, SMP_MSG_FLAG_SYNC);
+        } else {
+            cpu_status state = disable_interrupts();
+            arch_cpu_user_TLB_invalidate();
+            restore_interrupts(state);
+
+            int cpu = smp_get_current_cpu();
+            uint32 cpuMask = PagingStructures()->active_on_cpus
+                & ~((uint32)1 << cpu);
+            if (cpuMask != 0) {
+                smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
+                    0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
+            }
+        }
+    } else {
+        TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
+            fInvalidPagesCount);
+
+        arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount);
+
+        if (fIsKernelMap) {
+            smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
+                (uint64)fInvalidPages, fInvalidPagesCount, 0, NULL,
+                SMP_MSG_FLAG_SYNC);
+        } else {
+            int cpu = smp_get_current_cpu();
+            uint32 cpuMask = PagingStructures()->active_on_cpus
+                & ~((uint32)1 << cpu);
+            if (cpuMask != 0) {
+                smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
+                    (uint64)fInvalidPages, fInvalidPagesCount, 0, NULL,
+                    SMP_MSG_FLAG_SYNC);
+            }
+        }
+    }
+    fInvalidPagesCount = 0;
+
+    thread_unpin_from_current_cpu(thread);
+}

src/system/kernel/arch/x86_64/paging/x86_64_physical_page_mapper.h

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef KERNEL_ARCH_X86_64_PAGING_X86_64_PHYSICAL_PAGE_MAPPER_H
+#define KERNEL_ARCH_X86_64_PAGING_X86_64_PHYSICAL_PAGE_MAPPER_H
+
+
+#include <vm/VMTranslationMap.h>
+
+
+struct kernel_args;
+struct vm_translation_map_ops;
+
+
+class TranslationMapPhysicalPageMapper {
+public:
+    virtual                     ~TranslationMapPhysicalPageMapper();
+
+    virtual void                Delete() = 0;
+
+    virtual void*               GetPageTableAt(phys_addr_t physicalAddress) = 0;
+        // Must be invoked with thread pinned to current CPU.
+};
+
+
+class X86_64PhysicalPageMapper : public VMPhysicalPageMapper {
+public:
+    virtual                     ~X86_64PhysicalPageMapper();
+
+    virtual status_t            CreateTranslationMapPhysicalPageMapper(
+                                    TranslationMapPhysicalPageMapper** _mapper)
+                                        = 0;
+
+    virtual void*               InterruptGetPageTableAt(
+                                    phys_addr_t physicalAddress) = 0;
+};
+
+
+#endif  // KERNEL_ARCH_X86_64_PAGING_X86_64_PHYSICAL_PAGE_MAPPER_H

src/system/kernel/arch/x86_64/paging/x86_64_physical_page_mapper_large_memory.h

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef KERNEL_ARCH_X86_64_PAGING_X86_64_PHYSICAL_PAGE_MAPPER_LARGE_MEMORY_H
+#define KERNEL_ARCH_X86_64_PAGING_X86_64_PHYSICAL_PAGE_MAPPER_LARGE_MEMORY_H
+
+
+#include <OS.h>
+
+#include <util/DoublyLinkedList.h>
+
+
+class TranslationMapPhysicalPageMapper;
+class X86_64PhysicalPageMapper;
+struct kernel_args;
+
+
+namespace X86_64LargePhysicalPageMapper {
+
+
+struct PhysicalPageSlotPool;
+
+
+struct PhysicalPageSlot {
+    PhysicalPageSlot*           next;
+    PhysicalPageSlotPool*           pool;
+    addr_t                  address;
+
+    inline  void                Map(phys_addr_t physicalAddress);
+};
+
+
+struct PhysicalPageSlotPool : DoublyLinkedListLinkImpl<PhysicalPageSlotPool> {
+
+    virtual                 ~PhysicalPageSlotPool();
+
+    inline  bool                IsEmpty() const;
+
+    inline  PhysicalPageSlot*       GetSlot();
+    inline  void                PutSlot(PhysicalPageSlot* slot);
+
+    virtual status_t            AllocatePool(PhysicalPageSlotPool*& _pool) = 0;
+    virtual void                Map(phys_addr_t physicalAddress,
+                                addr_t virtualAddress) = 0;
+
+protected:
+    PhysicalPageSlot*           fSlots;
+};
+
+
+}
+
+
+status_t large_memory_physical_page_ops_init(kernel_args* args,
+    X86_64LargePhysicalPageMapper::PhysicalPageSlotPool* initialPool,
+    X86_64PhysicalPageMapper*& _pageMapper,
+    TranslationMapPhysicalPageMapper*& _kernelPageMapper);
+
+
+#endif  // KERNEL_ARCH_X86_64_PAGING_X86_64_PHYSICAL_PAGE_MAPPER_LARGE_MEMORY_H

src/system/kernel/arch/x86_64/paging/X86_64VMTranslationMap.h

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef KERNEL_ARCH_X86_64_PAGING_X86_64_VM_TRANSLATION_MAP_H
+#define KERNEL_ARCH_X86_64_PAGING_X86_64_VM_TRANSLATION_MAP_H
+
+
+#include <vm/VMTranslationMap.h>
+
+
+#define PAGE_INVALIDATE_CACHE_SIZE 64
+
+
+struct X86_64PagingStructures;
+class TranslationMapPhysicalPageMapper;
+
+
+struct X86_64VMTranslationMap : VMTranslationMap {
+                                X86_64VMTranslationMap();
+    virtual                     ~X86_64VMTranslationMap();
+
+            status_t            Init(bool kernel);
+
+    virtual bool                Lock();
+    virtual void                Unlock();
+
+    virtual addr_t              MappedSize() const;
+
+    virtual void                Flush();
+
+    inline  void                InvalidatePage(addr_t address);
+
+    inline  X86_64PagingStructures*     PagingStructures();
+
+    virtual size_t              MaxPagesNeededToMap(addr_t start,
+                                    addr_t end) const;
+
+    virtual status_t            Map(addr_t virtualAddress,
+                                    phys_addr_t physicalAddress,
+                                    uint32 attributes, uint32 memoryType,
+                                    vm_page_reservation* reservation);
+    virtual status_t            Unmap(addr_t start, addr_t end);
+
+    virtual status_t            UnmapPage(VMArea* area, addr_t address,
+                                    bool updatePageQueue);
+    virtual void                UnmapPages(VMArea* area, addr_t base,
+                                    size_t size, bool updatePageQueue);
+    virtual void                UnmapArea(VMArea* area,
+                                    bool deletingAddressSpace,
+                                    bool ignoreTopCachePageFlags);
+
+    virtual status_t            Query(addr_t virtualAddress,
+                                    phys_addr_t* _physicalAddress,
+                                    uint32* _flags);
+    virtual status_t            QueryInterrupt(addr_t virtualAddress,
+                                    phys_addr_t* _physicalAddress,
+                                    uint32* _flags);
+
+    virtual status_t            Protect(addr_t base, addr_t top,
+                                    uint32 attributes, uint32 memoryType);
+
+    virtual status_t            ClearFlags(addr_t virtualAddress,
+                                    uint32 flags);
+
+    virtual bool                ClearAccessedAndModified(
+                                    VMArea* area, addr_t address,
+                                    bool unmapIfUnaccessed,
+                                    bool& _modified);
+
+protected:
+    TranslationMapPhysicalPageMapper*   fPageMapper;
+    int                 fInvalidPagesCount;
+    addr_t                  fInvalidPages[PAGE_INVALIDATE_CACHE_SIZE];
+    bool                    fIsKernelMap;
+
+    X86_64PagingStructures*         fPagingStructures;
+};
+
+
+void
+X86_64VMTranslationMap::InvalidatePage(addr_t address)
+{
+    if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE)
+        fInvalidPages[fInvalidPagesCount] = address;
+
+    fInvalidPagesCount++;
+}
+
+inline  X86_64PagingStructures* 
+    X86_64VMTranslationMap::PagingStructures()
+{
+    return fPagingStructures;
+}
+
+#endif  // KERNEL_ARCH_X86_64_PAGING_X86_64_VM_TRANSLATION_MAP_H

src/system/kernel/arch/x86_64/paging/X86_64PagingStructures.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include "paging/X86_64PagingStructures.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <KernelExport.h>
+
+#include <int.h>
+
+#include "paging/X86_64PagingMethod.h"
+
+
+X86_64PagingStructures::X86_64PagingStructures()
+{
+    fVirtualPageDirs[0] = NULL;
+}
+
+
+X86_64PagingStructures::~X86_64PagingStructures()
+{
+    // free the user page dirs
+    free(fVirtualPageDirs[0]);
+
+    // free the PDPT page
+    X86_64PagingMethod::Method()->Free32BitPage(fPageDirPointerTable,
+        pgdir_phys, fPageDirPointerTableHandle);
+}
+
+
+void
+X86_64PagingStructures::Init(
+    page_directory_pointer_table_entry* virtualPDPT,
+    phys_addr_t physicalPDPT, void* pdptHandle,
+    page_directory_entry* const* virtualPageDirs,
+    const phys_addr_t* physicalPageDirs)
+{
+    fPageDirPointerTable = virtualPDPT;
+    pgdir_phys = physicalPDPT;
+    fPageDirPointerTableHandle = pdptHandle;
+    memcpy(fVirtualPageDirs, virtualPageDirs, sizeof(fVirtualPageDirs));
+    memcpy(fPhysicalPageDirs, physicalPageDirs, sizeof(fPhysicalPageDirs));
+}
+
+
+void
+X86_64PagingStructures::Delete()
+{
+    if (are_interrupts_enabled())
+        delete this;
+    else
+        deferred_delete(this);
+}

src/system/kernel/arch/x86_64/paging/paging.h

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef KERNEL_ARCH_X86_64_PAGING_PAGING_H
+#define KERNEL_ARCH_X86_64_PAGING_PAGING_H
+
+
+#include <OS.h>
+
+
+// page directory pointer table entry bits
+#define X86_64_PDPTE_PRESENT            0x0000000000000001LL
+#define X86_64_PDPTE_WRITE_THROUGH      0x0000000000000008LL
+#define X86_64_PDPTE_CACHING_DISABLED   0x0000000000000010LL
+#define X86_64_PDPTE_ADDRESS_MASK       0x000ffffffffff000LL
+
+// page directory entry bits
+#define X86_64_PDE_PRESENT              0x0000000000000001LL
+#define X86_64_PDE_WRITABLE         0x0000000000000002LL
+#define X86_64_PDE_USER             0x0000000000000004LL
+#define X86_64_PDE_WRITE_THROUGH        0x0000000000000008LL
+#define X86_64_PDE_CACHING_DISABLED 0x0000000000000010LL
+#define X86_64_PDE_ACCESSED         0x0000000000000020LL
+#define X86_64_PDE_IGNORED1         0x0000000000000040LL
+#define X86_64_PDE_LARGE_PAGE           0x0000000000000080LL
+#define X86_64_PDE_IGNORED2         0x0000000000000100LL
+#define X86_64_PDE_IGNORED3         0x0000000000000200LL
+#define X86_64_PDE_IGNORED4         0x0000000000000400LL
+#define X86_64_PDE_IGNORED5         0x0000000000000800LL
+#define X86_64_PDE_ADDRESS_MASK     0x000ffffffffff000LL
+#define X86_64_PDE_NOT_EXECUTABLE       0x8000000000000000LL
+
+// page table entry bits
+#define X86_64_PTE_PRESENT              0x0000000000000001LL
+#define X86_64_PTE_WRITABLE         0x0000000000000002LL
+#define X86_64_PTE_USER             0x0000000000000004LL
+#define X86_64_PTE_WRITE_THROUGH        0x0000000000000008LL
+#define X86_64_PTE_CACHING_DISABLED 0x0000000000000010LL
+#define X86_64_PTE_ACCESSED         0x0000000000000020LL
+#define X86_64_PTE_DIRTY                0x0000000000000040LL
+#define X86_64_PTE_PAT                  0x0000000000000080LL
+#define X86_64_PTE_GLOBAL               0x0000000000000100LL
+#define X86_64_PTE_IGNORED1         0x0000000000000200LL
+#define X86_64_PTE_IGNORED2         0x0000000000000400LL
+#define X86_64_PTE_IGNORED3         0x0000000000000800LL
+#define X86_64_PTE_ADDRESS_MASK     0x000ffffffffff000LL
+#define X86_64_PTE_NOT_EXECUTABLE       0x8000000000000000LL
+#define X86_64_PTE_PROTECTION_MASK      (X86_64_PTE_WRITABLE \
+                                            | X86_64_PTE_USER)
+#define X86_64_PTE_MEMORY_TYPE_MASK (X86_64_PTE_WRITE_THROUGH \
+                                            | X86_64_PTE_CACHING_DISABLED)
+
+
+static const uint32 kPageDirEntryCount = 512;
+static const uint32 kPageTableEntryCount = 512;
+static const size_t kPageTableRange = kPageTableEntryCount * B_PAGE_SIZE;
+static const size_t kPageDirRange
+    = kPageDirEntryCount * kPageTableRange;
+
+
+typedef uint64 page_directory_pointer_table_entry;
+typedef uint64 page_directory_entry;
+typedef uint64 page_table_entry;
+
+
+#endif  // KERNEL_ARCH_X86_64_PAGING_PAGING_H

src/system/kernel/arch/x86_64/paging/x86_64_physical_page_mapper.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+#include "paging/x86_64_physical_page_mapper.h"
+
+
+TranslationMapPhysicalPageMapper::~TranslationMapPhysicalPageMapper()
+{
+}
+
+
+X86_64PhysicalPageMapper::~X86_64PhysicalPageMapper()
+{
+}

src/system/kernel/arch/x86_64/paging/x86_64_physical_page_mapper_large_memory.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+/*! Implementation of a physical page mapping strategy (PhysicalPageMapper,
+    TranslationMapPhysicalPageMapper) suitable for machines with a lot of
+    memory, i.e. more than we can afford to completely map into the kernel
+    address space.
+
+    We allocate a single page table (one page) that can map 1024 pages and
+    a corresponding virtual address space region (4 MB). Each of those 1024
+    slots can map a physical page. We reserve a fixed amount of slots per CPU.
+    They will be used for physical operations on that CPU (memset()/memcpy()
+    and {get,put}_physical_page_current_cpu()). A few slots we reserve for each
+    translation map (TranslationMapPhysicalPageMapper). Those will only be used
+    with the translation map locked, mapping a page table page. The remaining
+    slots remain in the global pool and are given out by get_physical_page().
+
+    When we run out of slots, we allocate another page table (and virtual
+    address space region).
+*/
+
+
+#include "paging/x86_64_physical_page_mapper_large_memory.h"
+
+#include <new>
+
+#include <AutoDeleter.h>
+
+#include <cpu.h>
+#include <lock.h>
+#include <smp.h>
+#include <util/AutoLock.h>
+#include <vm/vm.h>
+#include <vm/vm_types.h>
+#include <vm/VMAddressSpace.h>
+
+#include "paging/x86_64_physical_page_mapper.h"
+#include "paging/X86_64PagingStructures.h"
+#include "paging/X86_64VMTranslationMap.h"
+
+
+// The number of slots we reserve per translation map from mapping page tables.
+// One slot would suffice, since the map is locked while mapping a page table,
+// but we re-use several slots on a LRU-basis so that we can keep the mappings
+// a little longer, thus avoiding re-mapping.
+#define SLOTS_PER_TRANSLATION_MAP   4
+
+#define USER_SLOTS_PER_CPU  16
+#define KERNEL_SLOTS_PER_CPU    16
+#define TOTAL_SLOTS_PER_CPU (USER_SLOTS_PER_CPU + KERNEL_SLOTS_PER_CPU + 1)
+
+    // one slot is for use in interrupts
+
+
+using X86_64LargePhysicalPageMapper::PhysicalPageSlot;
+using X86_64LargePhysicalPageMapper::PhysicalPageSlotPool;
+
+
+class PhysicalPageSlotQueue {
+public:
+                    PhysicalPageSlotQueue();
+
+    inline  PhysicalPageSlot*   GetSlot();
+    inline  void            GetSlots(PhysicalPageSlot*& slot1,
+                            PhysicalPageSlot*& slot2);
+    inline  void            PutSlot(PhysicalPageSlot* slot);
+    inline  void            PutSlots(PhysicalPageSlot* slot1,
+                            PhysicalPageSlot* slot2);
+
+private:
+    PhysicalPageSlot*       fSlots;
+    ConditionVariable       fFreeSlotCondition;
+    ConditionVariable       fFreeSlotsCondition;
+};
+
+
+struct PhysicalPageOpsCPUData {
+    PhysicalPageSlotQueue       user;
+        // Used when copying from/to user memory. This can cause a page fault
+        // which might need to memcpy()/memset() a page when being handled.
+    PhysicalPageSlotQueue       kernel;
+        // Used when memset()ing or when memcpy()ing memory non-user memory.
+    PhysicalPageSlot*       interruptSlot;
+
+    void                Init();
+
+private:
+    static  PhysicalPageSlot*   _GetInitialSlot();
+};
+
+
+// #pragma mark -
+
+
+class LargeMemoryTranslationMapPhysicalPageMapper
+    : public TranslationMapPhysicalPageMapper {
+public:
+                    LargeMemoryTranslationMapPhysicalPageMapper();
+    virtual             ~LargeMemoryTranslationMapPhysicalPageMapper();
+
+    status_t            Init();
+
+    virtual void            Delete();
+
+    virtual void*           GetPageTableAt(phys_addr_t physicalAddress);
+
+private:
+    struct page_slot {
+        PhysicalPageSlot*   slot;
+        phys_addr_t     physicalAddress;
+        cpu_mask_t      valid;
+    };
+
+    page_slot           fSlots[SLOTS_PER_TRANSLATION_MAP];
+    int32               fSlotCount; // must be a power of 2
+    int32               fNextSlot;
+};
+
+
+class LargeMemoryPhysicalPageMapper : public X86_64PhysicalPageMapper {
+public:
+                    LargeMemoryPhysicalPageMapper();
+
+    status_t            Init(kernel_args* args,
+                         PhysicalPageSlotPool* initialPool,
+                         TranslationMapPhysicalPageMapper*&
+                        _kernelPageMapper);
+
+    virtual status_t        CreateTranslationMapPhysicalPageMapper(
+                        TranslationMapPhysicalPageMapper** _mapper);
+
+    virtual void*           InterruptGetPageTableAt(
+                        phys_addr_t physicalAddress);
+
+    virtual status_t        GetPage(phys_addr_t physicalAddress,
+                        addr_t* virtualAddress, void** handle);
+    virtual status_t        PutPage(addr_t virtualAddress, void* handle);
+
+    virtual status_t        GetPageCurrentCPU(phys_addr_t physicalAddress,
+                        addr_t* virtualAddress, void** handle);
+    virtual status_t        PutPageCurrentCPU(addr_t virtualAddress,
+                        void* handle);
+
+    virtual status_t        GetPageDebug(phys_addr_t physicalAddress,
+                        addr_t* virtualAddress, void** handle);
+    virtual status_t        PutPageDebug(addr_t virtualAddress,
+                        void* handle);
+
+    virtual status_t        MemsetPhysical(phys_addr_t address, int value,
+                        phys_size_t length);
+    virtual status_t        MemcpyFromPhysical(void* to, phys_addr_t from,
+                        size_t length, bool user);
+    virtual status_t        MemcpyToPhysical(phys_addr_t to,
+                        const void* from, size_t length, bool user);
+    virtual void            MemcpyPhysicalPage(phys_addr_t to,
+                        phys_addr_t from);
+
+    status_t            GetSlot(bool canWait,
+                        PhysicalPageSlot*& slot);
+    void                PutSlot(PhysicalPageSlot* slot);
+
+    inline  PhysicalPageSlotQueue*  GetSlotQueue(int32 cpu, bool user);
+
+private:
+    typedef DoublyLinkedList<PhysicalPageSlotPool> PoolList;
+
+    mutex               fLock;
+    PoolList            fEmptyPools;
+    PoolList            fNonEmptyPools;
+    PhysicalPageSlot* fDebugSlot;
+    PhysicalPageSlotPool* fInitialPool;
+    LargeMemoryTranslationMapPhysicalPageMapper fKernelMapper;
+    PhysicalPageOpsCPUData fPerCPUData[B_MAX_CPU_COUNT];
+};
+
+static LargeMemoryPhysicalPageMapper sPhysicalPageMapper;
+
+
+// #pragma mark - PhysicalPageSlot / PhysicalPageSlotPool
+
+
+inline void
+PhysicalPageSlot::Map(phys_addr_t physicalAddress)
+{
+    pool->Map(physicalAddress, address);
+}
+
+
+PhysicalPageSlotPool::~PhysicalPageSlotPool()
+{
+}
+
+
+inline bool
+PhysicalPageSlotPool::IsEmpty() const
+{
+    return fSlots == NULL;
+}
+
+
+inline PhysicalPageSlot*
+PhysicalPageSlotPool::GetSlot()
+{
+    PhysicalPageSlot* slot = fSlots;
+    fSlots = slot->next;
+    return slot;
+}
+
+
+inline void
+PhysicalPageSlotPool::PutSlot(PhysicalPageSlot* slot)
+{
+    slot->next = fSlots;
+    fSlots = slot;
+}
+
+
+// #pragma mark - PhysicalPageSlotQueue
+
+
+PhysicalPageSlotQueue::PhysicalPageSlotQueue()
+    :
+    fSlots(NULL)
+{
+    fFreeSlotCondition.Init(this, "physical page ops slot queue");
+    fFreeSlotsCondition.Init(this, "physical page ops slots queue");
+}
+
+
+PhysicalPageSlot*
+PhysicalPageSlotQueue::GetSlot()
+{
+    InterruptsLocker locker;
+
+    // wait for a free slot to turn up
+    while (fSlots == NULL) {
+        ConditionVariableEntry entry;
+        fFreeSlotCondition.Add(&entry);
+        locker.Unlock();
+        entry.Wait();
+        locker.Lock();
+    }
+
+    PhysicalPageSlot* slot = fSlots;
+    fSlots = slot->next;
+
+    return slot;
+}
+
+
+void
+PhysicalPageSlotQueue::GetSlots(PhysicalPageSlot*& slot1,
+    PhysicalPageSlot*& slot2)
+{
+    InterruptsLocker locker;
+
+    // wait for two free slot to turn up
+    while (fSlots == NULL || fSlots->next == NULL) {
+        ConditionVariableEntry entry;
+        fFreeSlotsCondition.Add(&entry);
+        locker.Unlock();
+        entry.Wait();
+        locker.Lock();
+    }
+
+    slot1 = fSlots;
+    slot2 = slot1->next;
+    fSlots = slot2->next;
+}
+
+
+void
+PhysicalPageSlotQueue::PutSlot(PhysicalPageSlot* slot)
+{
+    InterruptsLocker locker;
+
+    slot->next = fSlots;
+    fSlots = slot;
+
+    if (slot->next == NULL)
+        fFreeSlotCondition.NotifyAll();
+    else if (slot->next->next == NULL)
+        fFreeSlotCondition.NotifyAll();
+}
+
+
+void
+PhysicalPageSlotQueue::PutSlots(PhysicalPageSlot* slot1,
+    PhysicalPageSlot* slot2)
+{
+    InterruptsLocker locker;
+
+    slot1->next = slot2;
+    slot2->next = fSlots;
+    fSlots = slot1;
+
+    if (slot2->next == NULL)
+        fFreeSlotCondition.NotifyAll();
+    else if (slot2->next->next == NULL)
+        fFreeSlotCondition.NotifyAll();
+}
+
+
+// #pragma mark - PhysicalPageOpsCPUData
+
+
+void
+PhysicalPageOpsCPUData::Init()
+{
+    for (int32 i = 0; i < USER_SLOTS_PER_CPU; i++)
+        user.PutSlot(_GetInitialSlot());
+    for (int32 i = 0; i < KERNEL_SLOTS_PER_CPU; i++)
+        kernel.PutSlot(_GetInitialSlot());
+    interruptSlot = _GetInitialSlot();
+}
+
+
+/* static */ PhysicalPageSlot*
+PhysicalPageOpsCPUData::_GetInitialSlot()
+{
+    PhysicalPageSlot* slot;
+    status_t error = sPhysicalPageMapper.GetSlot(false, slot);
+    if (error != B_OK) {
+        panic("PhysicalPageOpsCPUData::Init(): Failed to get initial "
+            "physical page slots! Probably too many CPUs.");
+        return NULL;
+    }
+
+    return slot;
+}
+
+
+// #pragma mark - LargeMemoryTranslationMapPhysicalPageMapper
+
+
+LargeMemoryTranslationMapPhysicalPageMapper
+    ::LargeMemoryTranslationMapPhysicalPageMapper()
+    :
+    fSlotCount(sizeof(fSlots) / sizeof(page_slot)),
+    fNextSlot(0)
+{
+    memset(fSlots, 0, sizeof(fSlots));
+}
+
+
+LargeMemoryTranslationMapPhysicalPageMapper
+    ::~LargeMemoryTranslationMapPhysicalPageMapper()
+{
+    // put our slots back to the global pool
+    for (int32 i = 0; i < fSlotCount; i++) {
+        if (fSlots[i].slot != NULL)
+            sPhysicalPageMapper.PutSlot(fSlots[i].slot);
+    }
+}
+
+
+status_t
+LargeMemoryTranslationMapPhysicalPageMapper::Init()
+{
+    // get our slots from the global pool
+    for (int32 i = 0; i < fSlotCount; i++) {
+        status_t error = sPhysicalPageMapper.GetSlot(true, fSlots[i].slot);
+        if (error != B_OK)
+            return error;
+
+        // set to invalid physical address, so it won't be used accidentally
+        fSlots[i].physicalAddress = ~(phys_addr_t)0;
+    }
+
+    return B_OK;
+}
+
+
+void
+LargeMemoryTranslationMapPhysicalPageMapper::Delete()
+{
+    delete this;
+}
+
+
+void*
+LargeMemoryTranslationMapPhysicalPageMapper::GetPageTableAt(
+    phys_addr_t physicalAddress)
+{
+    ASSERT(physicalAddress % B_PAGE_SIZE == 0);
+
+    int32 currentCPU = smp_get_current_cpu();
+
+    // maybe the address is already mapped
+    for (int32 i = 0; i < fSlotCount; i++) {
+        page_slot& slot = fSlots[i];
+        if (slot.physicalAddress == physicalAddress) {
+            fNextSlot = (i + 1) & (fSlotCount - 1);
+            if ((slot.valid & (1 << currentCPU)) == 0) {
+                // not valid on this CPU -- invalidate the TLB entry
+                invalidate_TLB(slot.slot->address);
+                slot.valid |= 1 << currentCPU;
+            }
+            return (void*)slot.slot->address;
+        }
+    }
+
+    // not found -- need to map a fresh one
+    page_slot& slot = fSlots[fNextSlot];
+    fNextSlot = (fNextSlot + 1) & (fSlotCount - 1);
+
+    slot.physicalAddress = physicalAddress;
+    slot.slot->Map(physicalAddress);
+    slot.valid = 1 << currentCPU;
+
+    return (void*)slot.slot->address;
+}
+
+
+// #pragma mark - LargeMemoryPhysicalPageMapper
+
+
+LargeMemoryPhysicalPageMapper::LargeMemoryPhysicalPageMapper()
+    :
+    fInitialPool(NULL)
+{
+    mutex_init(&fLock, "large memory physical page mapper");
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::Init(kernel_args* args,
+    PhysicalPageSlotPool* initialPool,
+    TranslationMapPhysicalPageMapper*& _kernelPageMapper)
+{
+    fInitialPool = initialPool;
+    fNonEmptyPools.Add(fInitialPool);
+
+    // get the debug slot
+    GetSlot(true, fDebugSlot);
+
+    // init the kernel translation map physical page mapper
+    status_t error = fKernelMapper.Init();
+    if (error != B_OK) {
+        panic("LargeMemoryPhysicalPageMapper::Init(): Failed to init "
+            "kernel translation map physical page mapper!");
+        return error;
+    }
+    _kernelPageMapper = &fKernelMapper;
+
+    // init the per-CPU data
+    int32 cpuCount = smp_get_num_cpus();
+    for (int32 i = 0; i < cpuCount; i++)
+        fPerCPUData[i].Init();
+
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::CreateTranslationMapPhysicalPageMapper(
+    TranslationMapPhysicalPageMapper** _mapper)
+{
+    LargeMemoryTranslationMapPhysicalPageMapper* mapper
+        = new(std::nothrow) LargeMemoryTranslationMapPhysicalPageMapper;
+    if (mapper == NULL)
+        return B_NO_MEMORY;
+
+    status_t error = mapper->Init();
+    if (error != B_OK) {
+        delete mapper;
+        return error;
+    }
+
+    *_mapper = mapper;
+    return B_OK;
+}
+
+
+void*
+LargeMemoryPhysicalPageMapper::InterruptGetPageTableAt(
+    phys_addr_t physicalAddress)
+{
+    ASSERT(physicalAddress % B_PAGE_SIZE == 0);
+
+    PhysicalPageSlot* slot = fPerCPUData[smp_get_current_cpu()].interruptSlot;
+    slot->Map(physicalAddress);
+    return (void*)slot->address;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
+    addr_t* virtualAddress, void** handle)
+{
+    PhysicalPageSlot* slot;
+    status_t error = GetSlot(true, slot);
+    if (error != B_OK)
+        return error;
+
+    slot->Map(physicalAddress);
+
+    *handle = slot;
+    *virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
+
+    smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, *virtualAddress,
+        *virtualAddress, 0, NULL, SMP_MSG_FLAG_SYNC);
+
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::PutPage(addr_t virtualAddress, void* handle)
+{
+    PutSlot((PhysicalPageSlot*)handle);
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::GetPageCurrentCPU(phys_addr_t physicalAddress,
+    addr_t* virtualAddress, void** handle)
+{
+    // get a slot from the per-cpu user pool
+    PhysicalPageSlotQueue& slotQueue
+        = fPerCPUData[smp_get_current_cpu()].user;
+    PhysicalPageSlot* slot = slotQueue.GetSlot();
+    slot->Map(physicalAddress);
+
+    *virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
+    *handle = slot;
+
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::PutPageCurrentCPU(addr_t virtualAddress,
+    void* handle)
+{
+    // return the slot to the per-cpu user pool
+    PhysicalPageSlotQueue& slotQueue
+        = fPerCPUData[smp_get_current_cpu()].user;
+    slotQueue.PutSlot((PhysicalPageSlot*)handle);
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::GetPageDebug(phys_addr_t physicalAddress,
+    addr_t* virtualAddress, void** handle)
+{
+    fDebugSlot->Map(physicalAddress);
+
+    *handle = fDebugSlot;
+    *virtualAddress = fDebugSlot->address + physicalAddress % B_PAGE_SIZE;
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::PutPageDebug(addr_t virtualAddress, void* handle)
+{
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::MemsetPhysical(phys_addr_t address, int value,
+    phys_size_t length)
+{
+    phys_addr_t pageOffset = address % B_PAGE_SIZE;
+
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner _(thread);
+
+    PhysicalPageSlotQueue* slotQueue = GetSlotQueue(thread->cpu->cpu_num,
+        false);
+    PhysicalPageSlot* slot = slotQueue->GetSlot();
+
+    while (length > 0) {
+        slot->Map(address - pageOffset);
+
+        size_t toSet = min_c(length, B_PAGE_SIZE - pageOffset);
+        memset((void*)(slot->address + pageOffset), value, toSet);
+
+        length -= toSet;
+        address += toSet;
+        pageOffset = 0;
+    }
+
+    slotQueue->PutSlot(slot);
+
+    return B_OK;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::MemcpyFromPhysical(void* _to, phys_addr_t from,
+    size_t length, bool user)
+{
+    uint8* to = (uint8*)_to;
+    phys_addr_t pageOffset = from % B_PAGE_SIZE;
+
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner _(thread);
+
+    PhysicalPageSlotQueue* slotQueue = GetSlotQueue(thread->cpu->cpu_num, user);
+    PhysicalPageSlot* slot = slotQueue->GetSlot();
+
+    status_t error = B_OK;
+
+    while (length > 0) {
+        size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
+
+        slot->Map(from - pageOffset);
+
+        if (user) {
+            error = user_memcpy(to, (void*)(slot->address + pageOffset),
+                toCopy);
+            if (error != B_OK)
+                break;
+        } else
+            memcpy(to, (void*)(slot->address + pageOffset), toCopy);
+
+        to += toCopy;
+        from += toCopy;
+        length -= toCopy;
+        pageOffset = 0;
+    }
+
+    slotQueue->PutSlot(slot);
+
+    return error;
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::MemcpyToPhysical(phys_addr_t to,
+    const void* _from, size_t length, bool user)
+{
+    const uint8* from = (const uint8*)_from;
+    phys_addr_t pageOffset = to % B_PAGE_SIZE;
+
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner _(thread);
+
+    PhysicalPageSlotQueue* slotQueue = GetSlotQueue(thread->cpu->cpu_num, user);
+    PhysicalPageSlot* slot = slotQueue->GetSlot();
+
+    status_t error = B_OK;
+
+    while (length > 0) {
+        size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
+
+        slot->Map(to - pageOffset);
+
+        if (user) {
+            error = user_memcpy((void*)(slot->address + pageOffset), from,
+                toCopy);
+            if (error != B_OK)
+                break;
+        } else
+            memcpy((void*)(slot->address + pageOffset), from, toCopy);
+
+        to += toCopy;
+        from += toCopy;
+        length -= toCopy;
+        pageOffset = 0;
+    }
+
+    slotQueue->PutSlot(slot);
+
+    return error;
+}
+
+
+void
+LargeMemoryPhysicalPageMapper::MemcpyPhysicalPage(phys_addr_t to,
+    phys_addr_t from)
+{
+    struct thread* thread = thread_get_current_thread();
+    ThreadCPUPinner _(thread);
+
+    PhysicalPageSlotQueue* slotQueue = GetSlotQueue(thread->cpu->cpu_num,
+        false);
+    PhysicalPageSlot* fromSlot;
+    PhysicalPageSlot* toSlot;
+    slotQueue->GetSlots(fromSlot, toSlot);
+
+    fromSlot->Map(from);
+    toSlot->Map(to);
+
+    memcpy((void*)toSlot->address, (void*)fromSlot->address, B_PAGE_SIZE);
+
+    slotQueue->PutSlots(fromSlot, toSlot);
+}
+
+
+status_t
+LargeMemoryPhysicalPageMapper::GetSlot(bool canWait, PhysicalPageSlot*& slot)
+{
+    MutexLocker locker(fLock);
+
+    PhysicalPageSlotPool* pool = fNonEmptyPools.Head();
+    if (pool == NULL) {
+        if (!canWait)
+            return B_WOULD_BLOCK;
+
+        // allocate new pool
+        locker.Unlock();
+        status_t error = fInitialPool->AllocatePool(pool);
+        if (error != B_OK)
+            return error;
+        locker.Lock();
+
+        fNonEmptyPools.Add(pool);
+        pool = fNonEmptyPools.Head();
+    }
+
+    slot = pool->GetSlot();
+
+    if (pool->IsEmpty()) {
+        fNonEmptyPools.Remove(pool);
+        fEmptyPools.Add(pool);
+    }
+
+    return B_OK;
+}
+
+
+void
+LargeMemoryPhysicalPageMapper::PutSlot(PhysicalPageSlot* slot)
+{
+    MutexLocker locker(fLock);
+
+    PhysicalPageSlotPool* pool = slot->pool;
+    if (pool->IsEmpty()) {
+        fEmptyPools.Remove(pool);
+        fNonEmptyPools.Add(pool);
+    }
+
+    pool->PutSlot(slot);
+}
+
+
+inline PhysicalPageSlotQueue*
+LargeMemoryPhysicalPageMapper::GetSlotQueue(int32 cpu, bool user)
+{
+    return user ? &fPerCPUData[cpu].user : &fPerCPUData[cpu].kernel;
+}
+
+
+// #pragma mark - Initialization
+
+
+status_t
+large_memory_physical_page_ops_init(kernel_args* args,
+    X86_64LargePhysicalPageMapper::PhysicalPageSlotPool* initialPool,
+    X86_64PhysicalPageMapper*& _pageMapper,
+    TranslationMapPhysicalPageMapper*& _kernelPageMapper)
+{
+    new(&sPhysicalPageMapper) LargeMemoryPhysicalPageMapper;
+    sPhysicalPageMapper.Init(args, initialPool, _kernelPageMapper);
+
+    _pageMapper = &sPhysicalPageMapper;
+    return B_OK;
+}

src/system/kernel/arch/x86_64/paging/X86_64PagingStructures.h

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2005-2009, Axel Dörfler, axeld@pinc-software.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+#ifndef KERNEL_ARCH_X86_64_PAGING_X86_64_PAGING_STRUCTURES_H
+#define KERNEL_ARCH_X86_64_PAGING_X86_64_PAGING_STRUCTURES_H
+
+
+#include <SupportDefs.h>
+
+#include <heap.h>
+#include "paging/paging.h"
+
+
+struct X86_64PagingStructures : DeferredDeletable {
+    uint64                  pgdir_phys;
+    vint32                  ref_count;
+    vint32                  active_on_cpus;
+        // mask indicating on which CPUs the map is currently used
+
+                        X86_64PagingStructures();
+    virtual                 ~X86_64PagingStructures();
+
+    inline  void                AddReference();
+    inline  void                RemoveReference();
+
+    void                    Init(page_directory_pointer_table_entry*
+                            virtualPDPT, phys_addr_t physicalPDPT, void* pdptHandle,
+                            page_directory_entry* const* virtualPageDirs,
+                            const phys_addr_t* physicalPageDirs);
+
+    virtual void                Delete();
+
+    page_directory_entry* const*        VirtualPageDirs() const
+                            { return fVirtualPageDirs; }
+
+private:
+    page_directory_pointer_table_entry* fPageDirPointerTable;
+    void*                   fPageDirPointerTableHandle;
+    page_directory_entry*           fVirtualPageDirs[4];
+    phys_addr_t             fPhysicalPageDirs[4];
+};
+
+
+inline void
+X86_64PagingStructures::AddReference()
+{
+    atomic_add(&ref_count, 1);
+}
+
+
+inline void
+X86_64PagingStructures::RemoveReference()
+{
+    if (atomic_add(&ref_count, -1) == 1)
+        Delete();
+}
+
+
+#endif  // KERNEL_ARCH_X86_64_PAGING_X86_64_PAGING_STRUCTURES_H

src/system/kernel/arch/x86_64/arch_commpage.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2007, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <string.h>
+
+#include <KernelExport.h>
+
+#include <commpage.h>
+#include <cpu.h>
+#include <elf.h>
+#include <smp.h>
+
+#include <SupportDefs.h>
+
+
+extern "C" void x86_64_sysenter();
+
+void (*gX86_64SetSyscallStack)(addr_t stackTop) = NULL;
+
+extern "C" void _user_syscall_int(void);
+extern unsigned int _user_syscall_int_end;
+extern "C" void _user_syscall_sysenter(void);
+extern unsigned int _user_syscall_sysenter_end;
+
+
+void
+x86_64_set_syscall_stack(addr_t stackTop)
+{
+    if (gX86_64SetSyscallStack != NULL)
+        gX86_64SetSyscallStack(stackTop);
+}
+
+
+static void
+init_x86_64_syscall_registers(void* dummy, int cpuNum)
+{
+    x86_64_write_msr(IA32_MSR_SYSENTER_CS, KERNEL_CODE_SEG);
+    x86_64_write_msr(IA32_MSR_SYSENTER_ESP, 0);
+    x86_64_write_msr(IA32_MSR_SYSENTER_EIP, (addr_t)x86_64_sysenter);
+
+    gX86_64SetSyscallStack = &x86_64_set_syscall_stack;
+}
+
+
+static bool
+all_cpus_have_feature(enum x86_feature_type type, int feature)
+{
+    int i;
+    int cpuCount = smp_get_num_cpus();
+
+    for (i = 0; i < cpuCount; i++) {
+        if (!(gCPU[i].arch.feature[type] & feature))
+            return false;
+    }
+
+    return true;
+}
+
+
+status_t
+arch_commpage_init_post_cpus(void)
+{
+    void* syscallCode = (void *)&_user_syscall_int;
+    void* syscallCodeEnd = &_user_syscall_int_end;
+
+    // check syscall
+    if (all_cpus_have_feature(FEATURE_COMMON, IA32_FEATURE_SEP)
+        && !(gCPU[0].arch.family == 6 && gCPU[0].arch.model < 3
+            && gCPU[0].arch.stepping < 3)) {
+        // Intel sysenter/sysexit
+        dprintf("initialize_commpage_syscall(): sysenter/sysexit supported\n");
+
+        // the code to be used in userland
+        syscallCode = (void *)&_user_syscall_sysenter;
+        syscallCodeEnd = &_user_syscall_sysenter_end;
+
+        // tell all CPUs to init their sysenter/sysexit related registers
+        call_all_cpus_sync(&init_x86_64_syscall_registers, NULL);
+    } else if (all_cpus_have_feature(FEATURE_EXT_AMD,
+            IA32_FEATURE_AMD_EXT_SYSCALL)) {
+        // AMD syscall/sysret
+        dprintf("initialize_commpage_syscall(): syscall/sysret supported "
+            "-- not yet by Haiku, though");
+    } else {
+        // no special syscall support
+        dprintf("initialize_commpage_syscall(): no special syscall support\n");
+    }
+
+    // fill in the table entry
+    size_t len = (size_t)((addr_t)syscallCodeEnd - (addr_t)syscallCode);
+    fill_commpage_entry(COMMPAGE_ENTRY_X86_64_SYSCALL, syscallCode, len);
+
+    // add syscall to the commpage image
+    image_id image = get_commpage_image();
+    elf_add_memory_image_symbol(image, "commpage_syscall",
+        ((addr_t*)USER_COMMPAGE_ADDR)[COMMPAGE_ENTRY_X86_64_SYSCALL], len,
+        B_SYMBOL_TYPE_TEXT);
+
+    return B_OK;
+}
+
+
+status_t
+arch_commpage_init(void)
+{
+    return B_OK;
+}

src/system/kernel/arch/x86_64/arch_int.cpp

@@ +1 @@
+/*
+ * Copyright 2010, Nathan Mentley, nathanmentley@gmail.com.
+ * Copyright 2010, Clemens Zeidler, haiku@clemens-zeidler.de.
+ * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Copyright 2001, Travis Geiselbrecht. All rights reserved.
+ * Distributed under the terms of the NewOS License.
+ */
+
+
+#include <cpu.h>
+#include <int.h>
+#include <kscheduler.h>
+#include <ksyscalls.h>
+#include <smp.h>
+#include <team.h>
+#include <thread.h>
+#include <vm/vm.h>
+#include <vm/vm_priv.h>
+
+#include <arch/cpu.h>
+#include <arch/int.h>
+#include <arch/smp.h>
+#include <arch/user_debugger.h>
+#include <arch/vm.h>
+
+#include <arch/x86_64/apic.h>
+#include <arch/x86_64/descriptors.h>
+
+#include "interrupts.h"
+#include "irq_routing_table.h"
+
+#include <ACPI.h>
+#include <AutoDeleter.h>
+#include <safemode.h>
+#include <string.h>
+#include <stdio.h>
+
+
+//#define TRACE_ARCH_INT
+#ifdef TRACE_ARCH_INT
+#   define TRACE(x) dprintf x
+#else
+#   define TRACE(x) ;
+#endif
+
+
+// Definitions for the PIC 8259 controller
+// (this is not a complete list, only what we're actually using)
+
+#define PIC_MASTER_CONTROL      0x20
+#define PIC_MASTER_MASK         0x21
+#define PIC_SLAVE_CONTROL       0xa0
+#define PIC_SLAVE_MASK          0xa1
+#define PIC_MASTER_INIT1        PIC_MASTER_CONTROL
+#define PIC_MASTER_INIT2        PIC_MASTER_MASK
+#define PIC_MASTER_INIT3        PIC_MASTER_MASK
+#define PIC_MASTER_INIT4        PIC_MASTER_MASK
+#define PIC_SLAVE_INIT1         PIC_SLAVE_CONTROL
+#define PIC_SLAVE_INIT2         PIC_SLAVE_MASK
+#define PIC_SLAVE_INIT3         PIC_SLAVE_MASK
+#define PIC_SLAVE_INIT4         PIC_SLAVE_MASK
+
+// the edge/level trigger control registers
+#define PIC_MASTER_TRIGGER_MODE 0x4d0
+#define PIC_SLAVE_TRIGGER_MODE  0x4d1
+
+#define PIC_INIT1               0x10
+#define PIC_INIT1_SEND_INIT4    0x01
+#define PIC_INIT3_IR2_IS_SLAVE  0x04
+#define PIC_INIT3_SLAVE_ID2     0x02
+#define PIC_INIT4_x86_MODE      0x01
+
+#define PIC_CONTROL3            0x08
+#define PIC_CONTROL3_READ_ISR   0x03
+#define PIC_CONTROL3_READ_IRR   0x02
+
+#define PIC_NON_SPECIFIC_EOI    0x20
+
+#define PIC_SLAVE_INT_BASE      8
+#define PIC_NUM_INTS            0x0f