Ticket #8007: Mutex-priority-inheritance_version3.patch

File Mutex-priority-inheritance_version3.patch, 18.7 KB (added by jua, 12 years ago)

Version 3 of patch implementing priority inheritance for mutexes and priority sorting for mutex waiters

  • headers/private/kernel/lock.h 

    From 46f10215ccd0d3fbc4e995522208bc88759a60e5 Mon Sep 17 00:00:00 2001
From: Yourself <user@shredder.(none)>
Date: Sat, 3 Mar 2012 00:24:56 +0100
Subject: [PATCH 4/4] Mutex priority inheritance.

---
 headers/private/kernel/lock.h         |   12 +-
 headers/private/kernel/thread.h       |    3 +
 headers/private/kernel/thread_types.h |    6 +
 src/system/kernel/locks/lock.cpp      |  208 ++++++++++++++++++++++++++++++---
 src/system/kernel/thread.cpp          |   52 ++++++++-
 5 files changed, 258 insertions(+), 23 deletions(-)

diff --git a/headers/private/kernel/lock.h b/headers/private/kernel/lock.h
index 4381d21..e9cec2a 100644
    a b typedef struct mutex {  
    2525    uint16                  ignore_unlock_count;
    2626#endif
    2727    uint8                   flags;
     28
     29    struct Thread          *holder_thread;
    2830} mutex;
    2931
    3032#define MUTEX_FLAG_CLONE_NAME   0x1
    … … typedef struct rw_lock {  
    8890
    8991// static initializers
    9092#if KDEBUG
    91 #   define MUTEX_INITIALIZER(name)          { name, NULL, -1, 0 }
     93#   define MUTEX_INITIALIZER(name)          { name, NULL, -1, 0, NULL }
    9294#   define RECURSIVE_LOCK_INITIALIZER(name) { MUTEX_INITIALIZER(name), 0 }
    9395#else
    94 #   define MUTEX_INITIALIZER(name)          { name, NULL, 0, 0, 0 }
     96#   define MUTEX_INITIALIZER(name)          { name, NULL, 0, 0, 0, NULL }
    9597#   define RECURSIVE_LOCK_INITIALIZER(name) { MUTEX_INITIALIZER(name), -1, 0 }
    9698#endif
    9799
    … … extern void _mutex_unlock(mutex* lock, bool schedulerLocked);  
    152154extern status_t _mutex_trylock(mutex* lock);
    153155extern status_t _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags,
    154156    bigtime_t timeout);
    155 
     157extern void _mutex_transfer_lock(mutex* lock, thread_id thread);
    156158
    157159static inline status_t
    158160rw_lock_read_lock(rw_lock* lock)
    … … mutex_unlock(mutex* lock)  
    268270static inline void
    269271mutex_transfer_lock(mutex* lock, thread_id thread)
    270272{
    271 #if KDEBUG
    272     lock->holder = thread;
    273 #endif
     273    _mutex_transfer_lock(lock, thread);
    274274}
    275275
    276276

  • headers/private/kernel/thread.h 

    diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h
index 8d1287e..e7e58a8 100644
    a b status_t thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum);  
    8989void thread_yield(bool force);
    9090void thread_exit(void);
    9191
     92void boost_thread_priority(Thread *thread, int32 boost_priority);
     93void unboost_thread_priority(Thread *thread);
     94
    9295int32 thread_max_threads(void);
    9396int32 thread_used_threads(void);
    9497

  • headers/private/kernel/thread_types.h 

    diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h
index 017da64..d023ddb 100644
    a b private:  
    409409            vint32              fUserDefinedTimerCount; // accessed atomically
    410410};
    411411
     412#define HELD_LOCKS_ARRAY_SIZE 32
    412413
    413414struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {
    414415    int32           flags;          // summary of events relevant in interrupt
    … … struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {  
    423424    int32           priority;       // protected by scheduler lock
    424425    int32           next_priority;  // protected by scheduler lock
    425426    int32           io_priority;    // protected by fLock
     427
     428    bool            boosted;
     429    int32           pre_boost_priority;
     430    mutex*          held_locks[HELD_LOCKS_ARRAY_SIZE];
     431
    426432    int32           state;          // protected by scheduler lock
    427433    int32           next_state;     // protected by scheduler lock
    428434    struct cpu_ent  *cpu;           // protected by scheduler lock

  • src/system/kernel/locks/lock.cpp 

    diff --git a/src/system/kernel/locks/lock.cpp b/src/system/kernel/locks/lock.cpp
index 1cc5c35..364b43c 100644
    a b dump_rw_lock_info(int argc, char** argv)  
    556556
    557557// #pragma mark -
    558558
     559static inline void
     560add_lock_to_held_locks(mutex *lock, Thread *thread)
     561{
     562    ASSERT(thread);
     563    ASSERT(lock);
     564
     565    // Search for first free entry in the array and use it
     566    for (int i = 0; i < HELD_LOCKS_ARRAY_SIZE; i++) {
     567        if (thread->held_locks[i] == NULL) {
     568            thread->held_locks[i] = lock;
     569            return;
     570        }
     571    }
     572}
     573
     574static inline void
     575remove_lock_from_held_locks(mutex *lock, Thread *thread)
     576{
     577    for (int i = 0; i < HELD_LOCKS_ARRAY_SIZE; i++) {
     578        if (thread->held_locks[i] == lock) {
     579            thread->held_locks[i] = NULL;
     580            return;
     581        }
     582    }
     583}
     584
     585static inline void
     586add_to_mutex_waiters_list(mutex *lock, mutex_waiter *waiter)
     587{
     588    if (lock->waiters != NULL) {
     589        if (waiter->thread->priority > lock->waiters->thread->priority) {
     590            // We have the highest priority of all
     591            // threads currently in the queue, prepend ourselves.
     592            waiter->next = lock->waiters;
     593            waiter->last = lock->waiters->last;
     594            lock->waiters = waiter;
     595        } else {
     596            // Search for the first waiter with lower (or equal) priority than ours.
     597            mutex_waiter *waiter_iterator = lock->waiters;
     598            mutex_waiter *previous_waiter = NULL;
     599
     600            while (waiter_iterator->thread->priority >= waiter->thread->priority
     601                && waiter_iterator->next != NULL) {
     602                previous_waiter = waiter_iterator;
     603                waiter_iterator = waiter_iterator->next;
     604            }
     605
     606            if (waiter_iterator->next == NULL) {
     607                // We are now the last in the queue, append and set 'last' pointer.
     608                waiter_iterator->next = waiter;
     609                lock->waiters->last = waiter;
     610            } else {
     611                // We belong somewhere in the middle of the queue, insert ourselves.
     612                waiter->next = waiter_iterator;
     613                previous_waiter->next = waiter;
     614            }
     615        }
     616    } else {
     617        // Nobody else waiting yet, set ourselves as first (and last) waiter.
     618        lock->waiters = waiter;
     619        lock->waiters->last = waiter;
     620    }
     621}
    559622
    560623void
    561624mutex_init(mutex* lock, const char *name)
    … … mutex_init(mutex* lock, const char *name)  
    569632    lock->ignore_unlock_count = 0;
    570633#endif
    571634    lock->flags = 0;
     635    lock->holder_thread = NULL;
    572636
    573637    T_SCHEDULING_ANALYSIS(InitMutex(lock, name));
    574638    NotifyWaitObjectListeners(&WaitObjectListener::MutexInitialized, lock);
    … … mutex_init_etc(mutex* lock, const char *name, uint32 flags)  
    587651    lock->ignore_unlock_count = 0;
    588652#endif
    589653    lock->flags = flags & MUTEX_FLAG_CLONE_NAME;
     654    lock->holder_thread = NULL;
    590655
    591656    T_SCHEDULING_ANALYSIS(InitMutex(lock, name));
    592657    NotifyWaitObjectListeners(&WaitObjectListener::MutexInitialized, lock);
    … … mutex_destroy(mutex* lock)  
    612677    }
    613678#endif
    614679
     680    // Remove the destroyed lock from held locks array of its holder.
     681    Thread *holder = lock->holder_thread;
     682    if (holder != NULL)
     683    {
     684        remove_lock_from_held_locks(lock, holder);
     685        unboost_thread_priority(holder);
     686    }
     687
    615688    while (mutex_waiter* waiter = lock->waiters) {
    616689        // dequeue
    617690        lock->waiters = waiter->next;
    … … _mutex_lock(mutex* lock, bool schedulerLocked)  
    677750#if KDEBUG
    678751    if (lock->holder < 0) {
    679752        lock->holder = thread_get_current_thread_id();
     753        lock->holder_thread = thread_get_current_thread();
     754
     755        ASSERT(lock->holder >= 0);
     756        ASSERT(lock->holder_thread);
     757
     758        // Add the lock we just acquired to the array of held locks for
     759        // this thread.
     760        add_lock_to_held_locks(lock, lock->holder_thread);
     761
    680762        return B_OK;
    681763    } else if (lock->holder == thread_get_current_thread_id()) {
    682764        panic("_mutex_lock(): double lock of %p by thread %ld", lock,
    … … _mutex_lock(mutex* lock, bool schedulerLocked)  
    686768#else
    687769    if ((lock->flags & MUTEX_FLAG_RELEASED) != 0) {
    688770        lock->flags &= ~MUTEX_FLAG_RELEASED;
     771
     772        // Add the lock we just acquired to the array of held locks for
     773        // this thread.
     774        add_lock_to_held_locks(lock, thread_get_current_thread());
     775
    689776        return B_OK;
    690777    }
    691778#endif
    … … _mutex_lock(mutex* lock, bool schedulerLocked)  
    695782    waiter.thread = thread_get_current_thread();
    696783    waiter.next = NULL;
    697784
    698     if (lock->waiters != NULL) {
    699         lock->waiters->last->next = &waiter;
    700     } else
    701         lock->waiters = &waiter;
     785    add_to_mutex_waiters_list(lock, &waiter);
    702786
    703     lock->waiters->last = &waiter;
     787    // The lock is already held by another thread. If this other thread has
     788    // a lower priority than ours, boost it so it can release the lock for
     789    // us more quickly.
     790    Thread *holder_thread  = lock->holder_thread;
     791    if (holder_thread != NULL) {
     792        if (waiter.thread->priority > holder_thread->priority)
     793            boost_thread_priority(holder_thread, waiter.thread->priority);
     794    }
    704795
    705796    // block
    706797    thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_MUTEX, lock);
    707798    status_t error = thread_block_locked(waiter.thread);
    708799
     800    if (error == B_OK) {
    709801#if KDEBUG
    710     if (error == B_OK)
    711802        lock->holder = waiter.thread->id;
    712803#endif
     804        lock->holder_thread = waiter.thread;
     805
     806        // Add the lock we just acquired to the array of held locks for
     807        // this thread.
     808        add_lock_to_held_locks(lock, waiter.thread);
     809    }
    713810
    714811    return error;
    715812}
    … … _mutex_unlock(mutex* lock, bool schedulerLocked)  
    751848        // cause a race condition, since another locker could think the lock
    752849        // is not held by anyone.
    753850        lock->holder = waiter->thread->id;
     851        lock->holder_thread = waiter->thread;
    754852#endif
    755853    } else {
    756854        // We've acquired the spinlock before the locker that is going to wait.
    757855        // Just mark the lock as released.
    758856#if KDEBUG
    759857        lock->holder = -1;
     858        lock->holder_thread = NULL;
    760859#else
    761860        lock->flags |= MUTEX_FLAG_RELEASED;
    762861#endif
    763862    }
    764 }
    765863
     864    // Remove the lock we just unlocked from held locks array and also search
     865    // for the maximum priority of all threads which are waiting for locks
     866    // we hold.
     867    Thread *current_thread = thread_get_current_thread();
     868    int32 maximum_priority = 0;
     869    for (int i = 0; i < HELD_LOCKS_ARRAY_SIZE; i++) {
     870        mutex *held_lock = current_thread->held_locks[i];
     871
     872        if (held_lock == lock) {
     873            // Remove from held locks array
     874            current_thread->held_locks[i] = NULL;
     875        } else if (held_lock != NULL && held_lock->waiters != NULL) {
     876            // The first thread in the waiters list has the highest priority
     877            int32 priority = held_lock->waiters->thread->priority;
     878            if (priority > maximum_priority)
     879                maximum_priority = priority;
     880        }
     881    }
     882
     883    // Find out whether we have to boost or unboost our priority.
     884    if (waiter == NULL) {
     885        // No waiters
     886        unboost_thread_priority(current_thread);
     887        return;
     888    }
     889
     890    if (waiter->thread->boosted) {
     891        if (maximum_priority > waiter->thread->pre_boost_priority) {
     892            // We are holding another lock which has a waiter with higher
     893            // priority than ours. Boost ourselves again so we can release
     894            // that lock as well quickly.
     895            boost_thread_priority(waiter->thread, maximum_priority);
     896        } else {
     897            // We are boosted but don't hold any other lock with waiters
     898            // with higher priority than ours. Restore our old pre-boost
     899            // priority.
     900            unboost_thread_priority(current_thread);
     901        }
     902    } else if (maximum_priority > waiter->thread->priority) {
     903        // We are not boosted anymore because our priority was changed
     904        // by someone else while we were boosted (doing that overwrites
     905        // the boosted priority and clears the boosted flag).
     906        // However, with our current priority, there is another thread
     907        // with higher priority than ours waiting for a lock we hold,
     908        // so boost ourselves again.
     909        boost_thread_priority(waiter->thread, maximum_priority);
     910    }
     911}
    766912
    767913status_t
    768914_mutex_trylock(mutex* lock)
    … … _mutex_trylock(mutex* lock)  
    772918
    773919    if (lock->holder <= 0) {
    774920        lock->holder = thread_get_current_thread_id();
     921        lock->holder_thread = thread_get_current_thread();
     922
     923        // Add the lock we just acquired to the array of held locks for
     924        // this thread.
     925        add_lock_to_held_locks(lock, lock->holder_thread);
     926
    775927        return B_OK;
    776928    }
    777929#endif
    … … _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout)  
    796948#if KDEBUG
    797949    if (lock->holder < 0) {
    798950        lock->holder = thread_get_current_thread_id();
     951        lock->holder_thread = thread_get_current_thread();
     952
     953        // Add the lock we just acquired to the array of held locks for
     954        // this thread.
     955        add_lock_to_held_locks(lock, lock->holder_thread);
     956
    799957        return B_OK;
    800958    } else if (lock->holder == thread_get_current_thread_id()) {
    801959        panic("_mutex_lock(): double lock of %p by thread %ld", lock,
    … … _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout)  
    805963#else
    806964    if ((lock->flags & MUTEX_FLAG_RELEASED) != 0) {
    807965        lock->flags &= ~MUTEX_FLAG_RELEASED;
     966
     967        lock->holder_thread = thread_get_current_thread();
     968        // Add the lock we just acquired to the array of held locks for
     969        // this thread.
     970        add_lock_to_held_locks(lock, thread_get_current_thread());
    808971        return B_OK;
    809972    }
    810973#endif
    … … _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout)  
    814977    waiter.thread = thread_get_current_thread();
    815978    waiter.next = NULL;
    816979
    817     if (lock->waiters != NULL) {
    818         lock->waiters->last->next = &waiter;
    819     } else
    820         lock->waiters = &waiter;
     980    add_to_mutex_waiters_list(lock, &waiter);
    821981
    822     lock->waiters->last = &waiter;
     982    // TODO: We could also do priority boosting here, but then
     983    //       have to unboost again on timeout.
    823984
    824985    // block
    825986    thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_MUTEX, lock);
    … … _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout)  
    828989    if (error == B_OK) {
    829990#if KDEBUG
    830991        lock->holder = waiter.thread->id;
     992        lock->holder_thread = waiter.thread;
    831993#endif
     994        // Add the lock we just acquired to the array of held locks for
     995        // this thread.
     996        add_lock_to_held_locks(lock, waiter.thread);
     997
    832998    } else {
    833999        // If the timeout occurred, we must remove our waiter structure from
    8341000        // the queue.
    … … _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout)  
    8691035    return error;
    8701036}
    8711037
     1038void
     1039_mutex_transfer_lock(mutex* lock, thread_id thread)
     1040{
     1041    InterruptsSpinLocker locker(gSchedulerLock);
     1042
     1043#if KDEBUG
     1044    remove_lock_from_held_locks(lock, lock->holder_thread);
     1045    unboost_thread_priority(lock->holder_thread);
     1046    lock->holder = thread;
     1047    lock->holder_thread = Thread::Get(thread);
     1048    add_lock_to_held_locks(lock, lock->holder_thread);
     1049#endif
     1050}
    8721051
    8731052static int
    8741053dump_mutex_info(int argc, char** argv)
    … … dump_mutex_info(int argc, char** argv)  
    8931072#else
    8941073    kprintf("  count:           %ld\n", lock->count);
    8951074#endif
     1075    kprintf("  holder_thread    %p\n", lock->holder_thread);
    8961076
    897     kprintf("  waiting threads:");
     1077    kprintf("  waiting threads [priority]:");
    8981078    mutex_waiter* waiter = lock->waiters;
    8991079    while (waiter != NULL) {
    900         kprintf(" %ld", waiter->thread->id);
     1080        kprintf(" %ld [%ld]", waiter->thread->id, waiter->thread->priority);
    9011081        waiter = waiter->next;
    9021082    }
    9031083    kputs("\n");

  • src/system/kernel/thread.cpp 

    diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp
index 6c95268..c2e2c6e 100644
    a b Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu)  
    169169    priority(-1),
    170170    next_priority(-1),
    171171    io_priority(-1),
     172    boosted(false),
     173    pre_boost_priority(-1),
    172174    cpu(cpu),
    173175    previous_cpu(NULL),
    174176    pinned_to_cpu(0),
    … … Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu)  
    198200    post_interrupt_callback(NULL),
    199201    post_interrupt_data(NULL)
    200202{
     203    for (int i = 0; i < HELD_LOCKS_ARRAY_SIZE; i++)
     204        held_locks[i] = NULL;
     205
    201206    id = threadID >= 0 ? threadID : allocate_thread_id();
    202207    visible = false;
    203208
    … … set_thread_prio(int argc, char **argv)  
    14411446        if (thread->id != id)
    14421447            continue;
    14431448        thread->priority = thread->next_priority = prio;
     1449        thread->boosted = false;
    14441450        kprintf("thread %ld set to priority %ld\n", id, prio);
    14451451        found = true;
    14461452        break;
    … … _dump_thread_info(Thread *thread, bool shortInfo)  
    16741680    kprintf("name:               \"%s\"\n", thread->name);
    16751681    kprintf("hash_next:          %p\nteam_next:          %p\nq_next:             %p\n",
    16761682        thread->hash_next, thread->team_next, thread->queue_next);
    1677     kprintf("priority:           %ld (next %ld, I/O: %ld)\n", thread->priority,
    1678         thread->next_priority, thread->io_priority);
     1683    kprintf("priority:           %ld (next %ld, I/O: %ld, boosted? %s)\n", thread->priority,
     1684        thread->next_priority, thread->io_priority, thread->boosted ? "yes" : "no");
     1685    if (thread->boosted)
     1686        kprintf("pre-boost priority: %ld\n", thread->pre_boost_priority);
    16791687    kprintf("state:              %s\n", state_to_text(thread, thread->state));
    16801688    kprintf("next_state:         %s\n", state_to_text(thread, thread->next_state));
    16811689    kprintf("cpu:                %p ", thread->cpu);
    … … _dump_thread_info(Thread *thread, bool shortInfo)  
    17341742        }
    17351743    }
    17361744
     1745    kprintf("held mutexes:       ");
     1746    for (int i = 0; i < HELD_LOCKS_ARRAY_SIZE; i++)
     1747        if (thread->held_locks[i] != NULL)
     1748            kprintf("%p ", thread->held_locks[i]);
     1749    kprintf("\n");
     1750
    17371751    kprintf("fault_handler:      %p\n", (void *)thread->fault_handler);
    17381752    kprintf("team:               %p, \"%s\"\n", thread->team,
    17391753        thread->team->Name());
    … … dump_thread_list(int argc, char **argv)  
    18111825{
    18121826    bool realTimeOnly = false;
    18131827    bool calling = false;
     1828    bool boosted = false;
    18141829    const char *callSymbol = NULL;
    18151830    addr_t callStart = 0;
    18161831    addr_t callEnd = 0;
    … … dump_thread_list(int argc, char **argv)  
    18431858            callSymbol = argv[1];
    18441859
    18451860        calling = true;
     1861    } else if (!strcmp(argv[0], "boosted")) {
     1862        boosted = true;
    18461863    } else if (argc > 1) {
    18471864        team = strtoul(argv[1], NULL, 0);
    18481865        if (team == 0)
    … … dump_thread_list(int argc, char **argv)  
    18591876                    callStart, callEnd))
    18601877            || (sem > 0 && get_thread_wait_sem(thread) != sem)
    18611878            || (team > 0 && thread->team->id != team)
    1862             || (realTimeOnly && thread->priority < B_REAL_TIME_DISPLAY_PRIORITY))
     1879            || (realTimeOnly && thread->priority < B_REAL_TIME_DISPLAY_PRIORITY)
     1880            || (boosted && !thread->boosted))
    18631881            continue;
    18641882
    18651883        _dump_thread_info(thread, true);
    … … thread_init(kernel_args *args)  
    27592777        "List all realtime threads",
    27602778        "\n"
    27612779        "Prints a list of all threads with realtime priority.\n", 0);
     2780    add_debugger_command_etc("boosted", &dump_thread_list,
     2781        "List all boosted threads",
     2782        "\n"
     2783        "Prints a list of all threads with currently boosted priority.\n", 0);
    27622784    add_debugger_command_etc("thread", &dump_thread_info,
    27632785        "Dump info about a particular thread",
    27642786        "[ -s ] ( <id> | <address> | <name> )*\n"
    … … thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum)  
    28162838    return B_OK;
    28172839}
    28182840
     2841void
     2842boost_thread_priority(Thread *thread, int32 boost_priority)
     2843{
     2844    ASSERT(!are_interrupts_enabled());
     2845    ASSERT(thread);
     2846
     2847    thread->pre_boost_priority = thread->priority;
     2848    scheduler_set_thread_priority(thread, boost_priority);
     2849    thread->boosted = true;
     2850}
     2851
     2852void
     2853unboost_thread_priority(Thread *thread)
     2854{
     2855    ASSERT(!are_interrupts_enabled());
     2856    ASSERT(thread);
     2857
     2858    if (!thread->boosted)
     2859        return;
     2860
     2861    thread->boosted = false;
     2862    thread->priority = thread->pre_boost_priority;
     2863}
    28192864
    28202865//  #pragma mark - thread blocking API
    28212866
    … … set_thread_priority(thread_id id, int32 priority)  
    31863231
    31873232    InterruptsSpinLocker schedulerLocker(gSchedulerLock);
    31883233
     3234    thread->boosted = false;
    31893235    if (thread == thread_get_current_thread()) {
    31903236        // It's ourself, so we know we aren't in the run queue, and we can
    31913237        // manipulate our structure directly.