Allow concurrent modifications to the ActiveTaskStatus while another

thread has the task status record lock.

Today, if a thread is holding the StatusRecordLock, then no other
modification of the task status is possible - including a thread
starting to execute the task or stopping execution of the task.
However, the TaskStatusRecordLock is really about protecting the linked
list that is maintained in the ActiveTaskStatus. As such, other
operations which don't need to look at that linked list of us records
really shouldn't have to block on the StatusRecordLock.

This change allows for concurrent modification of the veTaskStatus while
the TaskStatusRecordLock is held. In particular, a task can cancelled,
escalated, start and stop running, all while another ad is holding onto
the task's StatusRecordLock. In the event of cancellation and
escalation, the task's StatusRecordLock must be n in order to propagate
cancellation and escalation to its child tasks is not needed to cancel
or escalate the task itself.

Radar-Id: rdar://problem/76127624

Author: rokhinip

include/swift/ABI/Task.h

@@ -297,7 +297,6 @@ class AsyncTask : public Job {
   /// Generally this should be done immediately after updating
   /// ActiveTask.
   void flagAsRunning();
-  void flagAsRunning_slow();
 
   /// Flag that this task is now suspended.  This can update the
   /// priority stored in the job flags if the priority hsa been
@@ -306,7 +305,6 @@ class AsyncTask : public Job {
   /// somewhere.  TODO: record where the task is enqueued if
   /// possible.
   void flagAsSuspended();
-  void flagAsSuspended_slow();
 
   /// Flag that this task is now completed. This normally does not do anything
   /// but can be used to locally insert logging.

stdlib/public/Concurrency/TaskPrivate.h

@@ -353,7 +353,7 @@ class alignas(sizeof(void*) * 2) ActiveTaskStatus {
 #endif
   }
 
-  /// Is there an active lock on the cancellation information?
+  /// Is there a lock on the linked list of status records?
   bool isStatusRecordLocked() const { return Flags & IsStatusRecordLocked; }
   ActiveTaskStatus withLockingRecord(TaskStatusRecord *lockRecord) const {
     assert(!isStatusRecordLocked());
@@ -365,6 +365,19 @@ class alignas(sizeof(void*) * 2) ActiveTaskStatus {
 #endif
   }
 
+  ActiveTaskStatus withoutLockingRecord() const {
+    assert(isStatusRecordLocked());
+    assert(Record->getKind() == TaskStatusRecordKind::Private_RecordLock);
+
+    // Remove the lock record, and put the next one as the head
+    auto newRecord = Record->Parent;
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveTaskStatus(newRecord, Flags & ~IsStatusRecordLocked, ExecutionLock);
+#else
+    return ActiveTaskStatus(newRecord, Flags & ~IsStatusRecordLocked);
+#endif
+  }
+
   JobPriority getStoredPriority() const {
     return JobPriority(Flags & PriorityMask);
   }
@@ -543,13 +556,6 @@ inline void AsyncTask::flagAsRunning() {
   auto oldStatus = _private().Status.load(std::memory_order_relaxed);
   while (true) {
     assert(!oldStatus.isRunning());
-    if (oldStatus.isStatusRecordLocked()) {
-      flagAsRunning_slow();
-      adoptTaskVoucher(this);
-      swift_task_enterThreadLocalContext(
-          (char *)&_private().ExclusivityAccessSet[0]);
-      return;
-    }
 
     auto newStatus = oldStatus.withRunning(true);
     if (newStatus.isStoredPriorityEscalated()) {
@@ -559,8 +565,8 @@ inline void AsyncTask::flagAsRunning() {
     }
 
     if (_private().Status.compare_exchange_weak(oldStatus, newStatus,
-                                                std::memory_order_relaxed,
-                                                std::memory_order_relaxed)) {
+             /* success */ std::memory_order_relaxed,
+             /* failure */ std::memory_order_relaxed)) {
       newStatus.traceStatusChanged(this);
       adoptTaskVoucher(this);
       swift_task_enterThreadLocalContext(
@@ -575,13 +581,6 @@ inline void AsyncTask::flagAsSuspended() {
   auto oldStatus = _private().Status.load(std::memory_order_relaxed);
   while (true) {
     assert(oldStatus.isRunning());
-    if (oldStatus.isStatusRecordLocked()) {
-      flagAsSuspended_slow();
-      swift_task_exitThreadLocalContext(
-          (char *)&_private().ExclusivityAccessSet[0]);
-      restoreTaskVoucher(this);
-      return;
-    }
 
     auto newStatus = oldStatus.withRunning(false);
     if (newStatus.isStoredPriorityEscalated()) {
@@ -591,15 +590,16 @@ inline void AsyncTask::flagAsSuspended() {
     }
 
     if (_private().Status.compare_exchange_weak(oldStatus, newStatus,
-                                                std::memory_order_relaxed,
-                                                std::memory_order_relaxed)) {
-      newStatus.traceStatusChanged(this);
-      swift_task_exitThreadLocalContext(
-          (char *)&_private().ExclusivityAccessSet[0]);
-      restoreTaskVoucher(this);
-      return;
+            /* success */std::memory_order_relaxed,
+            /* failure */std::memory_order_relaxed)) {
+      break;
     }
   }
+
+  newStatus.traceStatusChanged(this);
+  swift_task_exitThreadLocalContext((char *)&_private().ExclusivityAccessSet[0]);
+  restoreTaskVoucher(this);
+  return;
 }
 
 // READ ME: This is not a dead function! Do not remove it! This is a function