Ticket #8798: 0001-pthread_rwlock-local-implementation-with-mutexes.patch

headers/posix/pthread.h

@@ -80 +80 @@
     { PTHREAD_MUTEX_RECURSIVE, 0, -42, -1, 0 }
 #define PTHREAD_COND_INITIALIZER    \
     { 0, -42, NULL, 0, 0 }
+#define PTHREAD_RWLOCK_INITIALIZER  \
+    { 0, -1, 0, 0, 0, 0, 0, 0 }
 
 /* mutex functions */
 extern int pthread_mutex_destroy(pthread_mutex_t *mutex);

headers/posix/sys/types.h

@@ -108 +108 @@
             __haiku_std_int32   sem;
         } shared;
         struct {
-            __haiku_std_int32   lock_sem;
-            __haiku_std_int32   lock_count;
+            __haiku_std_int32   mutex;
             __haiku_std_int32   reader_count;
-            __haiku_std_int32   writer_count;
-            void*               waiters[2];
+            __haiku_std_int32   write_waiter_count;
+            __haiku_std_int32   write_lock;
+            __haiku_std_int32   read_waiter_count;
+            __haiku_std_int32   read_lock;
+#ifdef B_HAIKU_64_BIT
+            __haiku_std_int32   unused[2];
+#endif
         } local;
     } u;
 };

src/system/libroot/posix/pthread/pthread_rwlock.cpp

@@ -16 +16 @@
 #include <util/DoublyLinkedList.h>
 
 #include "pthread_private.h"
+#include <user_mutex_defs.h>
 
 
 #define MAX_READER_COUNT    1000000
@@ -93 +94 @@
 struct LocalRWLock {
     uint32_t    flags;
     int32_t     owner;
-    int32_t     lock_sem;
-    int32_t     lock_count;
+    int32_t     mutex;
     int32_t     reader_count;
-    int32_t     writer_count;
-        // Note, that reader_count and writer_count are not used the same way.
-        // writer_count includes the write lock owner as well as waiting
-        // writers. reader_count includes read lock owners only.
-    WaiterList  waiters;
+    int32_t     write_waiter_count;
+    int32_t     write_lock;
+    int32_t     read_waiter_count;
+    int32_t     read_lock;
 
     status_t Init()
     {
         flags = 0;
         owner = -1;
-        lock_sem = create_sem(0, "pthread rwlock");
-        lock_count = 1;
+        mutex = 0;
         reader_count = 0;
-        writer_count = 0;
-        new(&waiters) WaiterList;
-
-        return lock_sem >= 0 ? B_OK : EAGAIN;
+        write_waiter_count = 0;
+        write_lock = 0;
+        read_waiter_count = 0;
+        read_lock = 0;
+        return B_OK;
     }
 
     status_t Destroy()
     {
-        if (lock_sem < 0)
-            return B_BAD_VALUE;
-        return delete_sem(lock_sem) == B_OK ? B_OK : B_BAD_VALUE;
+        Locker locker(this);
+        if (reader_count > 0 || owner != -1 || read_waiter_count > 0
+            || write_waiter_count > 0)
+            return EBUSY;
+        return 0;
     }
 
     bool StructureLock()
     {
-        if (atomic_add((int32*)&lock_count, -1) <= 0)
-            acquire_sem(lock_sem);
+        // Enter critical region: lock the mutex
+        int32 status = atomic_or((int32*)&mutex, B_USER_MUTEX_LOCKED);
+
+        // If already locked, call the kernel
+        if ((status & (B_USER_MUTEX_LOCKED | B_USER_MUTEX_WAITING)) != 0) {
+            do {
+                status = _kern_mutex_lock((int32*)&mutex, NULL, 0, 0);
+            } while (status == B_INTERRUPTED);
+
+            if (status != B_OK)
+                return false;
+        }
         return true;
     }
 
     void StructureUnlock()
     {
-        if (atomic_add((int32*)&lock_count, 1) < 0)
-            release_sem(lock_sem);
+        // Exit critical region: unlock the mutex
+        int32 status = atomic_and((int32*)&mutex,
+            ~(int32)B_USER_MUTEX_LOCKED);
+
+        if ((status & B_USER_MUTEX_WAITING) != 0)
+            _kern_mutex_unlock((int32*)&mutex, 0);
     }
 
     status_t ReadLock(bigtime_t timeout)
     {
         Locker locker(this);
 
-        if (writer_count == 0) {
-            reader_count++;
-            return B_OK;
+        status_t status = B_OK;
+        if (owner != -1 || write_waiter_count > 0) {
+            if (timeout == 0)
+                return B_TIMED_OUT;
+            read_waiter_count++;
+            do {
+                atomic_or((int32*)&read_lock, B_USER_MUTEX_LOCKED);
+                status = _kern_mutex_switch_lock((int32*)&mutex,
+                    (int32*)&read_lock, "pthread rwlock",
+                    timeout == B_INFINITE_TIMEOUT ? 0 : B_ABSOLUTE_REAL_TIME_TIMEOUT,
+                    timeout);
+                StructureLock();
+                if (status != 0)
+                    break;
+            } while (owner != -1 || write_waiter_count > 0);
+            read_waiter_count--;
         }
 
-        return _Wait(false, timeout);
+        if (status == 0)
+            reader_count++;
+
+        return status;
     }
 
     status_t WriteLock(bigtime_t timeout)
     {
         Locker locker(this);
 
-        if (reader_count == 0 && writer_count == 0) {
-            writer_count++;
+        status_t status = B_OK;
+        if (reader_count > 0 || owner != -1) {
+            if (timeout == 0)
+                return B_TIMED_OUT;
+            write_waiter_count++;
+            do {
+                atomic_or((int32*)&write_lock, B_USER_MUTEX_LOCKED);
+                status = _kern_mutex_switch_lock((int32*)&mutex,
+                    (int32*)&write_lock, "pthread rwlock",
+                    timeout == B_INFINITE_TIMEOUT ? 0 : B_ABSOLUTE_REAL_TIME_TIMEOUT,
+                    timeout);
+                StructureLock();
+                if (status != 0)
+                    break;
+            } while (reader_count > 0 || owner != -1);
+            write_waiter_count--;
+            if (status == B_TIMED_OUT && write_waiter_count == 0
+                && read_waiter_count > 0) {
+                _kern_mutex_unlock((int32*)&read_lock,
+                    B_USER_MUTEX_UNBLOCK_ALL);
+            }
+        }
+        if (status == 0) {
             owner = find_thread(NULL);
-            return B_OK;
         }
-
-        return _Wait(true, timeout);
+        return status;
     }
 
     status_t Unlock()
@@ -165 +215 @@
         Locker locker(this);
 
         if (find_thread(NULL) == owner) {
-            writer_count--;
             owner = -1;
-        } else
+            if (write_waiter_count > 0)
+                _kern_mutex_unlock((int32*)&write_lock, 0);
+            else if (read_waiter_count > 0)
+                _kern_mutex_unlock((int32*)&read_lock, B_USER_MUTEX_UNBLOCK_ALL);
+        } else {
             reader_count--;
-
-        _Unblock();
+            if (reader_count == 0 && write_waiter_count > 0)
+                _kern_mutex_unlock((int32*)&write_lock, 0);
+            else if (read_waiter_count > 0)
+                _kern_mutex_unlock((int32*)&read_lock, B_USER_MUTEX_UNBLOCK_ALL);
+        }
 
         return B_OK;
     }
 
 private:
-    status_t _Wait(bool writer, bigtime_t timeout)
-    {
-        if (timeout == 0)
-            return B_TIMED_OUT;
-
-        Waiter waiter(writer);
-        waiters.Add(&waiter);
-        waiter.queued = true;
-        waiter.userThread->wait_status = 1;
-
-        if (writer)
-            writer_count++;
-
-        StructureUnlock();
-        status_t error = _kern_block_thread(
-            timeout >= 0 ? B_ABSOLUTE_REAL_TIME_TIMEOUT : 0, timeout);
-        StructureLock();
-
-        if (!waiter.queued)
-            return waiter.status;
-
-        // we're still queued, which means an error (timeout, interrupt)
-        // occurred
-        waiters.Remove(&waiter);
-
-        if (writer)
-            writer_count--;
-
-        _Unblock();
-
-        return error;
-    }
-
-    void _Unblock()
-    {
-        // Check whether there any waiting threads at all and whether anyone
-        // has the write lock
-        Waiter* waiter = waiters.Head();
-        if (waiter == NULL || owner >= 0)
-            return;
-
-        // writer at head of queue?
-        if (waiter->writer) {
-            if (reader_count == 0) {
-                waiter->status = B_OK;
-                waiter->queued = false;
-                waiters.Remove(waiter);
-                owner = waiter->thread;
-
-                if (waiter->userThread->wait_status > 0)
-                    _kern_unblock_thread(waiter->thread, B_OK);
-            }
-            return;
-        }
-
-        // wake up one or more readers -- we unblock more than one reader at
-        // a time to save trips to the kernel
-        while (!waiters.IsEmpty() && !waiters.Head()->writer) {
-            static const int kMaxReaderUnblockCount = 128;
-            thread_id readers[kMaxReaderUnblockCount];
-            int readerCount = 0;
-
-            while (readerCount < kMaxReaderUnblockCount
-                    && (waiter = waiters.Head()) != NULL
-                    && !waiter->writer) {
-                waiter->status = B_OK;
-                waiter->queued = false;
-                waiters.Remove(waiter);
-
-                if (waiter->userThread->wait_status > 0) {
-                    readers[readerCount++] = waiter->thread;
-                    reader_count++;
-                }
-            }
-
-            if (readerCount > 0)
-                _kern_unblock_threads(readers, readerCount, B_OK);
-        }
-    }
-
 
     struct Locking {
         inline bool Lock(LocalRWLock* lockable)