summaryrefslogtreecommitdiffstats
path: root/kernel/futex.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-06-24 14:46:01 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2015-06-24 14:46:01 -0700
commita262948335bc5359b82f0ed5ef35f6e82ca44d16 (patch)
treeed03ae6d9e95d0f317115906ec79e540806e98f4 /kernel/futex.c
parente3d8238d7f5c3f539a29f5ac596cd342d847e099 (diff)
parent9f40a51a35a0e1445cc4873251c3df2631eda294 (diff)
downloadlinux-stable-a262948335bc5359b82f0ed5ef35f6e82ca44d16.tar.gz
linux-stable-a262948335bc5359b82f0ed5ef35f6e82ca44d16.tar.bz2
linux-stable-a262948335bc5359b82f0ed5ef35f6e82ca44d16.zip
Merge branch 'sched-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Thomas Gleixner: "These locking updates depend on the alreay merged sched/core branch: - Lockless top waiter wakeup for rtmutex (Davidlohr) - Reduce hash bucket lock contention for PI futexes (Sebastian) - Documentation update (Davidlohr)" * 'sched-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/rtmutex: Update stale plist comments futex: Lower the lock contention on the HB lock during wake up locking/rtmutex: Implement lockless top-waiter wakeup
Diffstat (limited to 'kernel/futex.c')
-rw-r--r--kernel/futex.c32
1 files changed, 29 insertions, 3 deletions
diff --git a/kernel/futex.c b/kernel/futex.c
index ea6ca0bca525..c4a182f5357e 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1117,11 +1117,14 @@ static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q)
q->lock_ptr = NULL;
}
-static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
+static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,
+ struct futex_hash_bucket *hb)
{
struct task_struct *new_owner;
struct futex_pi_state *pi_state = this->pi_state;
u32 uninitialized_var(curval), newval;
+ WAKE_Q(wake_q);
+ bool deboost;
int ret = 0;
if (!pi_state)
@@ -1173,7 +1176,19 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
raw_spin_unlock_irq(&new_owner->pi_lock);
raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
- rt_mutex_unlock(&pi_state->pi_mutex);
+
+ deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
+
+ /*
+ * First unlock HB so the waiter does not spin on it once he got woken
+ * up. Second wake up the waiter before the priority is adjusted. If we
+ * deboost first (and lose our higher priority), then the task might get
+ * scheduled away before the wake up can take place.
+ */
+ spin_unlock(&hb->lock);
+ wake_up_q(&wake_q);
+ if (deboost)
+ rt_mutex_adjust_prio(current);
return 0;
}
@@ -2410,13 +2425,23 @@ retry:
*/
match = futex_top_waiter(hb, &key);
if (match) {
- ret = wake_futex_pi(uaddr, uval, match);
+ ret = wake_futex_pi(uaddr, uval, match, hb);
+ /*
+ * In case of success wake_futex_pi dropped the hash
+ * bucket lock.
+ */
+ if (!ret)
+ goto out_putkey;
/*
* The atomic access to the futex value generated a
* pagefault, so retry the user-access and the wakeup:
*/
if (ret == -EFAULT)
goto pi_faulted;
+ /*
+ * wake_futex_pi has detected invalid state. Tell user
+ * space.
+ */
goto out_unlock;
}
@@ -2437,6 +2462,7 @@ retry:
out_unlock:
spin_unlock(&hb->lock);
+out_putkey:
put_futex_key(&key);
return ret;