locking/rwsem: Always try to wake waiters in out_nolock path

jira LE-1907
Rebuild_History Non-Buildable kernel-rt-5.14.0-284.30.1.rt14.315.el9_2
commit-author Waiman Long <longman@redhat.com>
commit 1ee3261
Empty-Commit: Cherry-Pick Conflicts during history rebuild.
Will be included in final tarball splat. Ref for failed cherry-pick at:
ciq/ciq_backports/kernel-rt-5.14.0-284.30.1.rt14.315.el9_2/1ee32619.failed

For writers, the out_nolock path will always attempt to wake up waiters.
This may not be really necessary if the waiter to be removed is not the
first one.

For readers, no attempt to wake up waiter is being made. However, if
the HANDOFF bit is set and the reader to be removed is the first waiter,
the waiter behind it will inherit the HANDOFF bit and for a write lock
waiter waking it up will allow it to spin on the lock to acquire it
faster. So it can be beneficial to do a wakeup in this case.

Add a new rwsem_del_wake_waiter() helper function to do that consistently
for both reader and writer out_nolock paths.

	Signed-off-by: Waiman Long <longman@redhat.com>
	Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20220322152059.2182333-4-longman@redhat.com
(cherry picked from commit 1ee3261)
	Signed-off-by: Jonathan Maple <jmaple@ciq.com>

# Conflicts:
#	kernel/locking/rwsem.c

Author: PlaidCat

ciq/ciq_backports/kernel-rt-5.14.0-284.30.1.rt14.315.el9_2/1ee32619.failed

@@ -0,0 +1,115 @@
+locking/rwsem: Always try to wake waiters in out_nolock path
+
+jira LE-1907
+Rebuild_History Non-Buildable kernel-rt-5.14.0-284.30.1.rt14.315.el9_2
+commit-author Waiman Long <longman@redhat.com>
+commit 1ee326196c66583006b0c95356a4b7dc51bf3531
+Empty-Commit: Cherry-Pick Conflicts during history rebuild.
+Will be included in final tarball splat. Ref for failed cherry-pick at:
+ciq/ciq_backports/kernel-rt-5.14.0-284.30.1.rt14.315.el9_2/1ee32619.failed
+
+For writers, the out_nolock path will always attempt to wake up waiters.
+This may not be really necessary if the waiter to be removed is not the
+first one.
+
+For readers, no attempt to wake up waiter is being made. However, if
+the HANDOFF bit is set and the reader to be removed is the first waiter,
+the waiter behind it will inherit the HANDOFF bit and for a write lock
+waiter waking it up will allow it to spin on the lock to acquire it
+faster. So it can be beneficial to do a wakeup in this case.
+
+Add a new rwsem_del_wake_waiter() helper function to do that consistently
+for both reader and writer out_nolock paths.
+
+	Signed-off-by: Waiman Long <longman@redhat.com>
+	Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
+Link: https://lkml.kernel.org/r/20220322152059.2182333-4-longman@redhat.com
+(cherry picked from commit 1ee326196c66583006b0c95356a4b7dc51bf3531)
+	Signed-off-by: Jonathan Maple <jmaple@ciq.com>
+
+# Conflicts:
+#	kernel/locking/rwsem.c
+diff --cc kernel/locking/rwsem.c
+index 186ad9eda88d,16b532bb5b92..000000000000
+--- a/kernel/locking/rwsem.c
++++ b/kernel/locking/rwsem.c
+@@@ -373,6 -362,34 +373,37 @@@ enum writer_wait_state 
+   */
+  #define MAX_READERS_WAKEUP	0x100
+  
+++<<<<<<< HEAD
+++=======
++ static inline void
++ rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
++ {
++ 	lockdep_assert_held(&sem->wait_lock);
++ 	list_add_tail(&waiter->list, &sem->wait_list);
++ 	/* caller will set RWSEM_FLAG_WAITERS */
++ }
++ 
++ /*
++  * Remove a waiter from the wait_list and clear flags.
++  *
++  * Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of
++  * this function. Modify with care.
++  *
++  * Return: true if wait_list isn't empty and false otherwise
++  */
++ static inline bool
++ rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
++ {
++ 	lockdep_assert_held(&sem->wait_lock);
++ 	list_del(&waiter->list);
++ 	if (likely(!list_empty(&sem->wait_list)))
++ 		return true;
++ 
++ 	atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count);
++ 	return false;
++ }
++ 
+++>>>>>>> 1ee326196c66 (locking/rwsem: Always try to wake waiters in out_nolock path)
+  /*
+   * handle the lock release when processes blocked on it that can now run
+   * - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must
+@@@ -1013,12 -1080,7 +1071,16 @@@ queue
+  	return sem;
+  
+  out_nolock:
+++<<<<<<< HEAD
+ +	list_del(&waiter.list);
+ +	if (list_empty(&sem->wait_list)) {
+ +		atomic_long_andnot(RWSEM_FLAG_WAITERS|RWSEM_FLAG_HANDOFF,
+ +				   &sem->count);
+ +	}
+ +	raw_spin_unlock_irq(&sem->wait_lock);
+++=======
++ 	rwsem_del_wake_waiter(sem, &waiter, &wake_q);
+++>>>>>>> 1ee326196c66 (locking/rwsem: Always try to wake waiters in out_nolock path)
+  	__set_current_state(TASK_RUNNING);
+  	lockevent_inc(rwsem_rlock_fail);
+  	return ERR_PTR(-EINTR);
+@@@ -1151,19 -1176,8 +1212,23 @@@ trylock_again
+  out_nolock:
+  	__set_current_state(TASK_RUNNING);
+  	raw_spin_lock_irq(&sem->wait_lock);
+++<<<<<<< HEAD
+ +	list_del(&waiter.list);
+ +
+ +	if (unlikely(wstate == WRITER_HANDOFF))
+ +		atomic_long_add(-RWSEM_FLAG_HANDOFF,  &sem->count);
+ +
+ +	if (list_empty(&sem->wait_list))
+ +		atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count);
+ +	else
+ +		rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
+ +	raw_spin_unlock_irq(&sem->wait_lock);
+ +	wake_up_q(&wake_q);
+++=======
++ 	rwsem_del_wake_waiter(sem, &waiter, &wake_q);
+++>>>>>>> 1ee326196c66 (locking/rwsem: Always try to wake waiters in out_nolock path)
+  	lockevent_inc(rwsem_wlock_fail);
+ +
+  	return ERR_PTR(-EINTR);
+  }
+  
+* Unmerged path kernel/locking/rwsem.c