Ticket #8661: 0001-Fix-8661-fcntl-fd-F_GETLK-.-violates-POSIX.patch

src/system/kernel/fs/vfs.cpp

@@ -1524 +1524 @@
 }
 
 
-/*! Retrieves the first lock that has been set by the current team.
+/*! Returns \c true when either \a flock is \c NULL or the \a flock intersects
+    with the advisory_lock \a lock.
+*/
+static bool
+advisory_lock_intersects(struct advisory_lock* lock, struct flock* flock)
+{
+    if (flock == NULL)
+        return true;
+
+    return lock->start <= flock->l_start - 1 + flock->l_len
+        && lock->end >= flock->l_start;
+}
+
+
+/*! Tests whether acquiring a lock would block.
 */
 static status_t
-get_advisory_lock(struct vnode* vnode, struct flock* flock)
+test_advisory_lock(struct vnode* vnode, struct flock* flock)
 {
+    flock->l_type = F_UNLCK;
+
     struct advisory_locking* locking = get_advisory_locking(vnode);
     if (locking == NULL)
-        return B_BAD_VALUE;
+        return B_OK;
 
-    // TODO: this should probably get the flock by its file descriptor!
     team_id team = team_get_current_team_id();
-    status_t status = B_BAD_VALUE;
 
+    // test for collisions
     LockList::Iterator iterator = locking->locks.GetIterator();
     while (iterator.HasNext()) {
         struct advisory_lock* lock = iterator.Next();
 
-        if (lock->team == team) {
-            flock->l_start = lock->start;
-            flock->l_len = lock->end - lock->start + 1;
-            status = B_OK;
-            break;
+        if (lock->team != team && advisory_lock_intersects(lock, flock)) {
+            // locks do overlap
+            if (flock->l_type != F_RDLCK || !lock->shared) {
+                // collision
+                flock->l_type = lock->shared ? F_RDLCK : F_WRLCK;
+                flock->l_whence = SEEK_SET;
+                flock->l_start = lock->start;
+                flock->l_len = lock->end - lock->start + 1;
+                flock->l_pid = lock->team;
+
+                break;
+            }
         }
     }
 
     put_advisory_locking(locking);
-    return status;
-}
-
-
-/*! Returns \c true when either \a flock is \c NULL or the \a flock intersects
-    with the advisory_lock \a lock.
-*/
-static bool
-advisory_lock_intersects(struct advisory_lock* lock, struct flock* flock)
-{
-    if (flock == NULL)
-        return true;
-
-    return lock->start <= flock->l_start - 1 + flock->l_len
-        && lock->end >= flock->l_start;
+    return B_OK;
 }
 
 
@@ -6006 +6014 @@
 
         case F_GETLK:
             if (vnode != NULL) {
-                status = get_advisory_lock(vnode, &flock);
+                struct flock tmp;
+                memcpy(&tmp, &flock, sizeof(struct flock));
+
+                status = normalize_flock(descriptor, &tmp);
+                if (status != B_OK)
+                    break;
+
+                status = test_advisory_lock(vnode, &tmp);
                 if (status == B_OK) {
                     // copy back flock structure
-                    status = user_memcpy((struct flock*)argument, &flock,
-                        sizeof(struct flock));
+                    if (tmp.l_type == F_UNLCK) {
+                        flock.l_type = F_UNLCK;
+
+                        status = user_memcpy((struct flock*)argument, &flock,
+                            sizeof(struct flock));
+                    } else {
+                        if (tmp.l_len == OFF_MAX)
+                            tmp.l_len = 0;
+
+                        status = user_memcpy((struct flock*)argument, &tmp,
+                            sizeof(struct flock));
+                    }
                 }
             } else
                 status = B_BAD_VALUE;