I211 priority scheduling for signal handler

kernel/app/init-example/app/init.cc

@@ -366,6 +366,82 @@ void test_ExecutionContext()
   MLOG_INFO(mlog::app, "End Test ExecutionContext");
 }
 
+std::atomic<unsigned> prio_flag;
+
+void* prio_main(void* ctx)
+{
+  if(ctx != nullptr){
+      MLOG_INFO(mlog::app, "hello priority thread!", DVAR(ctx));
+      ASSERT(prio_flag.load() == 1);
+      prio_flag++;
+      for (volatile int i=0; i<100000; i++) {
+        for (volatile int j=0; j<1000; j++) {}
+      }
+      MLOG_INFO(mlog::app, "priority thread finished", DVAR(ctx));
+  }else{
+      MLOG_INFO(mlog::app, "hello normal thread!", DVAR(ctx));
+      ASSERT(prio_flag.load() == 0);
+      prio_flag++;
+      while(prio_flag.load() < 3);
+      MLOG_INFO(mlog::app, "normal thread finished", DVAR(ctx));
+  }
+
+  prio_flag++;
+  return 0;
+}
+
+void test_PrioEC()
+{
+  MLOG_INFO(mlog::app, "Test ExecutionContext priority scheduling");
+  mythos::ExecutionContext ec1(capAlloc());
+  mythos::ExecutionContext ec2(capAlloc());
+
+  prio_flag.store(0);
+
+  {
+    MLOG_INFO(mlog::app, "test_EC: create ec");
+    mythos::PortalLock pl(portal); // future access will fail if the portal is in use already
+
+    // create new ec
+    auto tls1 = mythos::setupNewTLS();
+    auto tls2 = mythos::setupNewTLS();
+    ASSERT(tls1 != nullptr);
+    ASSERT(tls2 != nullptr);
+
+    auto sc = pa.alloc(pl).wait();
+    TEST(sc);
+
+    auto res = ec1.create(kmem).as(myAS).cs(myCS).sched(sc->cap)
+    .prepareStack(thread1stack_top).startFun(&prio_main, nullptr)
+    .suspended(true).fs(tls1)
+    .invokeVia(pl).wait();
+    TEST(res);
+
+    res = ec2.create(kmem).as(myAS).cs(myCS).sched(sc->cap)
+    .prepareStack(thread2stack_top).startFun(&prio_main, (void*)1)
+    .suspended(true).fs(tls2)
+    .invokeVia(pl).wait();
+    TEST(res);
+
+    ec2.setPriority(pl, true).wait();
+
+    ec1.resume(pl).wait();
+
+    MLOG_INFO(mlog::app, "wait for EC1 to start");
+    while(prio_flag.load() == 0);
+
+    ec2.resume(pl).wait();
+
+    MLOG_INFO(mlog::app, "wait for ECs to finish execution");
+    while(prio_flag < 4);
+
+    MLOG_INFO(mlog::app, "free ECs");
+    TEST(capAlloc.free(ec1, pl));
+    TEST(capAlloc.free(ec2, pl));
+  }
+  MLOG_INFO(mlog::app, "End Test ExecutionContext priority scheduling");
+}
+
 void test_InterruptControl() {
   MLOG_INFO(mlog::app, "test_InterruptControl start");
   mythos::InterruptControl ic(mythos::init::INTERRUPT_CONTROL_START);
@@ -549,6 +625,7 @@ int main()
   //test_InterruptControl();
   //test_HostChannel(portal, 24*1024*1024, 2*1024*1024);
   test_ExecutionContext();
+  test_PrioEC();
   test_pthreads();
   test_Rapl();
   test_processor_allocator();

kernel/mythos/invocation/mythos/protocol/ExecutionContext.hh

@@ -40,7 +40,8 @@ namespace mythos {
         WRITE_REGISTERS,
         SET_FSGS,
         RESUME,
-        SUSPEND
+        SUSPEND,
+        SET_PRIORITY
       };
 
       enum Signals : uint64_t {
@@ -125,6 +126,13 @@ namespace mythos {
         uintptr_t gs;
       };
 
+      struct SetPriority : public InvocationBase {
+        constexpr static uint16_t label = (proto<<8) + SET_PRIORITY;
+        SetPriority(bool priority)
+          : InvocationBase(label,getLength(this)), priority(priority) {}
+        bool priority;
+      };
+
       struct Resume : public InvocationBase {
         constexpr static uint16_t label = (proto<<8) + RESUME;
         Resume() : InvocationBase(label,getLength(this)) {}
@@ -156,6 +164,7 @@ namespace mythos {
         case READ_REGISTERS: return obj->invokeReadRegisters(args...);
         case WRITE_REGISTERS: return obj->invokeWriteRegisters(args...);
         case SET_FSGS: return obj->invokeSetFSGS(args...);
+        case SET_PRIORITY: return obj->invokeSetPriority(args...);
         case RESUME: return obj->invokeResume(args...);
         case SUSPEND: return obj->invokeSuspend(args...);
         default: return Error::NOT_IMPLEMENTED;

kernel/objects/common/objects/ISchedulable.hh

@@ -65,6 +65,8 @@ namespace mythos {
 
     virtual void saveState() = 0;
 
+    virtual bool hasPriority() { return false; } // no high priority per default
+
     /** informs the execution context that the user-mode code trapped and error handling is needed. */
     virtual void handleTrap() = 0;
 

kernel/objects/execution-context/objects/ExecutionContext.cc

@@ -350,6 +350,23 @@ namespace mythos {
       });
   }
 
+  Error ExecutionContext::invokeSetPriority(Tasklet*, Cap, IInvocation* msg)
+  {
+    auto const& data = *msg->getMessage()->cast<protocol::ExecutionContext::SetPriority>();
+    if (data.priority) {
+      setFlags(PRIORITY);
+    }else{
+      clearFlags(PRIORITY);
+    }
+
+    if (isReady()) {
+      auto sched = _sched.get();
+      if (sched) sched->ready(&ec_handle);
+    }
+
+    return Error::SUCCESS;
+  }
+
   Error ExecutionContext::invokeSetFSGS(Tasklet*, Cap, IInvocation* msg)
   {
     auto const& data = *msg->getMessage()->cast<protocol::ExecutionContext::SetFSGS>();

kernel/objects/execution-context/objects/ExecutionContext.hh

@@ -68,6 +68,7 @@ namespace mythos {
       NOT_LOADED   = 1<<8, // CPU state is not loaded
       DONT_PREEMPT  = 1<<9, // somebody else will send the preemption
       NOT_RUNNING  = 1<<10, // EC is not running
+      PRIORITY = 1<<11, // EC has high priority
       BLOCK_MASK = IS_WAITING | IS_TRAPPED | NO_AS | NO_SCHED | REGISTER_ACCESS
     };
 
@@ -99,6 +100,7 @@ namespace mythos {
 
   public: // ISchedulable interface
     bool isReady() const override { return !isBlocked(flags.load()); }
+    bool hasPriority() override { return (flags.load() & PRIORITY) != 0; }
     void resume() override;
     void handleTrap() override;
     void handleInterrupt() override { setFlags(NOT_RUNNING); }
@@ -149,6 +151,7 @@ namespace mythos {
     void suspendThread(Tasklet* t, optional<void>);
     Error getDebugInfo(Cap self, IInvocation* msg);
     Error invokeSetFSGS(Tasklet* t, Cap self, IInvocation* msg);
+    Error invokeSetPriority(Tasklet*, Cap, IInvocation* msg);
 
   protected:
     friend class CapRefBind;

kernel/objects/scheduling-context/objects/SchedulingContext.cc

@@ -45,8 +45,9 @@ namespace mythos {
         ASSERT(ec != nullptr);
         MLOG_INFO(mlog::sched, "unbind", DVAR(ec->get()));
         readyQueue.remove(ec);
+        priorityQueue.remove(ec);
         current_handle.store(nullptr);
-        if(readyQueue.empty()){
+        if(readyQueue.empty() && priorityQueue.empty()){
           MLOG_DETAIL(mlog::sched, "call idleSC");
           event::idleSC.emit(&paTask, home->getThreadID());
         }else{
@@ -67,11 +68,17 @@ namespace mythos {
 
         // add to the ready queue
         readyQueue.remove(ec); /// @todo do not need to remove if already on the queue, just do nothing then. This needs additional information in handle_t of LinkedList
-        readyQueue.push(ec);
+        priorityQueue.remove(ec); /// @todo do not need to remove if already on the queue, just do nothing then. This needs additional information in handle_t of LinkedList
+        if(ec->get()->hasPriority()){
+          priorityQueue.push(ec);
+          home->preempt(); // trigger reschedule to run priority ec 
+        }else{
+          readyQueue.push(ec);
 
-        // wake up the hardware thread if it has no execution context running
-	// or if if current ec got ready in case of race condition
-        if (current == nullptr || current == ec) home->preempt();
+          // wake up the hardware thread if it has no execution context running
+    // or if if current ec got ready in case of race condition
+          if (current == nullptr || current == ec) home->preempt();
+        }
     }
 
     void SchedulingContext::tryRunUser()
@@ -81,24 +88,38 @@ namespace mythos {
         while (true) {
             auto current = current_handle.load();
             if (current != nullptr) {
-                MLOG_DETAIL(mlog::sched, "try current", current, DVAR(current->get()));
-                auto loaded = current_ec->load();
-                if (loaded != current->get()) {
-                    if (loaded != nullptr) loaded->saveState();
-                    current->get()->loadState();
+                if(current->get()->hasPriority() || priorityQueue.empty()){
+                    MLOG_DETAIL(mlog::sched, "try current", current, DVAR(current->get()));
+                    auto loaded = current_ec->load();
+                    if (loaded != current->get()) {
+                        if (loaded != nullptr) loaded->saveState();
+                        current->get()->loadState();
+                    }
+                    current->get()->resume(); // if it returns, the ec is blocked
+                    current_handle.store(nullptr);
+                }else{
+                    MLOG_DETAIL(mlog::sched, "try switching to priority EC from non-prio EC ", current, DVAR(current->get()));
+                    readyQueue.push(current);
+                    current_handle.store(nullptr);
                 }
-                current->get()->resume(); // if it returns, the ec is blocked
-                current_handle.store(nullptr);
             }
-            MLOG_DETAIL(mlog::sched, "try from ready list");
-            // something on the ready list?
-            auto next = readyQueue.pull();
-            while (next != nullptr && !next->get()->isReady()) next = readyQueue.pull();
+
+            MLOG_DETAIL(mlog::sched, "try from priority list");
+            auto next = priorityQueue.pull();
+            while (next != nullptr && !next->get()->isReady()) next = priorityQueue.pull();
             if (next == nullptr) {
-                // go sleeping because we don't have anything to run
-                MLOG_DETAIL(mlog::sched, "empty ready list, going to sleep");
-                return;
-            }
+                MLOG_DETAIL(mlog::sched, "empty priority list, try ready list");
+                // something on the ready list?
+                next = readyQueue.pull();
+                while (next != nullptr && !next->get()->isReady()) next = readyQueue.pull();
+                if (next == nullptr) {
+                    // go sleeping because we don't have anything to run
+                    MLOG_DETAIL(mlog::sched, "empty ready list, going to sleep");
+                    return;
+                }
+            }else{
+                MLOG_DETAIL(mlog::sched, "found priority EC", next, DVAR(next->get()));
+            } 
             current_handle.store(next);
             // now retry
         }

kernel/objects/scheduling-context/objects/SchedulingContext.hh

@@ -97,6 +97,7 @@ namespace mythos {
   private:
     async::Place* home = nullptr;
     list_t readyQueue; //< the ready list of waiting execution contexts
+    list_t priorityQueue; //< the ready list of waiting execution contexts with high priority
     std::atomic<handle_t*> current_handle = {nullptr}; //< the currently selected execution context
 
     Tasklet paTask; //task for communication with processor allocator

kernel/runtime/kobject/runtime/ExecutionContext.hh

@@ -153,6 +153,9 @@ namespace mythos {
     PortalFuture<void> setFSGS(PortalLock pr, uintptr_t fs, uintptr_t gs) {
       return pr.invoke<protocol::ExecutionContext::SetFSGS>(_cap, fs,gs);
     }
+    PortalFuture<void> setPriority(PortalLock pr, bool priority) {
+      return pr.invoke<protocol::ExecutionContext::SetPriority>(_cap, priority);
+    }
     PortalFuture<void> resume(PortalLock pr) {
       return pr.invoke<protocol::ExecutionContext::Resume>(_cap);
     }