From e9c67348a790a092bf15ef521afa514f40474fc0 Mon Sep 17 00:00:00 2001 From: Philipp Gypser Date: Thu, 20 Jan 2022 10:40:17 +0100 Subject: [PATCH 1/3] EC priority scheduling --- .../mythos/protocol/ExecutionContext.hh | 11 +++- kernel/objects/common/objects/ISchedulable.hh | 2 + .../objects/ExecutionContext.cc | 17 ++++++ .../objects/ExecutionContext.hh | 3 + .../objects/SchedulingContext.cc | 61 +++++++++++++------ .../objects/SchedulingContext.hh | 1 + .../kobject/runtime/ExecutionContext.hh | 3 + 7 files changed, 77 insertions(+), 21 deletions(-) diff --git a/kernel/mythos/invocation/mythos/protocol/ExecutionContext.hh b/kernel/mythos/invocation/mythos/protocol/ExecutionContext.hh index 4089ce0a..1c9f39aa 100644 --- a/kernel/mythos/invocation/mythos/protocol/ExecutionContext.hh +++ b/kernel/mythos/invocation/mythos/protocol/ExecutionContext.hh @@ -40,7 +40,8 @@ namespace mythos { WRITE_REGISTERS, SET_FSGS, RESUME, - SUSPEND + SUSPEND, + SET_PRIORITY }; enum Signals : uint64_t { @@ -125,6 +126,13 @@ namespace mythos { uintptr_t gs; }; + struct SetPriority : public InvocationBase { + constexpr static uint16_t label = (proto<<8) + SET_PRIORITY; + SetPriority(bool priority) + : InvocationBase(label,getLength(this)), priority(priority) {} + bool priority; + }; + struct Resume : public InvocationBase { constexpr static uint16_t label = (proto<<8) + RESUME; Resume() : InvocationBase(label,getLength(this)) {} @@ -156,6 +164,7 @@ namespace mythos { case READ_REGISTERS: return obj->invokeReadRegisters(args...); case WRITE_REGISTERS: return obj->invokeWriteRegisters(args...); case SET_FSGS: return obj->invokeSetFSGS(args...); + case SET_PRIORITY: return obj->invokeSetPriority(args...); case RESUME: return obj->invokeResume(args...); case SUSPEND: return obj->invokeSuspend(args...); default: return Error::NOT_IMPLEMENTED; diff --git a/kernel/objects/common/objects/ISchedulable.hh b/kernel/objects/common/objects/ISchedulable.hh index 089933d3..9ccf05b0 100644 --- a/kernel/objects/common/objects/ISchedulable.hh +++ b/kernel/objects/common/objects/ISchedulable.hh @@ -65,6 +65,8 @@ namespace mythos { virtual void saveState() = 0; + virtual bool hasPriority() { return false; } + /** informs the execution context that the user-mode code trapped and error handling is needed. */ virtual void handleTrap() = 0; diff --git a/kernel/objects/execution-context/objects/ExecutionContext.cc b/kernel/objects/execution-context/objects/ExecutionContext.cc index 05270511..3d86ebe2 100644 --- a/kernel/objects/execution-context/objects/ExecutionContext.cc +++ b/kernel/objects/execution-context/objects/ExecutionContext.cc @@ -350,6 +350,23 @@ namespace mythos { }); } + Error ExecutionContext::invokeSetPriority(Tasklet*, Cap, IInvocation* msg) + { + auto const& data = *msg->getMessage()->cast(); + if (data.priority) { + setFlags(PRIORITY); + }else{ + clearFlags(PRIORITY); + } + + if (isReady()) { + auto sched = _sched.get(); + if (sched) sched->ready(&ec_handle); + } + + return Error::SUCCESS; + } + Error ExecutionContext::invokeSetFSGS(Tasklet*, Cap, IInvocation* msg) { auto const& data = *msg->getMessage()->cast(); diff --git a/kernel/objects/execution-context/objects/ExecutionContext.hh b/kernel/objects/execution-context/objects/ExecutionContext.hh index 5cfbd019..3b22277d 100644 --- a/kernel/objects/execution-context/objects/ExecutionContext.hh +++ b/kernel/objects/execution-context/objects/ExecutionContext.hh @@ -68,6 +68,7 @@ namespace mythos { NOT_LOADED = 1<<8, // CPU state is not loaded DONT_PREEMPT = 1<<9, // somebody else will send the preemption NOT_RUNNING = 1<<10, // EC is not running + PRIORITY = 1<<11, // EC has priority BLOCK_MASK = IS_WAITING | IS_TRAPPED | NO_AS | NO_SCHED | REGISTER_ACCESS }; @@ -99,6 +100,7 @@ namespace mythos { public: // ISchedulable interface bool isReady() const override { return !isBlocked(flags.load()); } + bool hasPriority() override { return (flags.load() & PRIORITY) != 0; } void resume() override; void handleTrap() override; void handleInterrupt() override { setFlags(NOT_RUNNING); } @@ -149,6 +151,7 @@ namespace mythos { void suspendThread(Tasklet* t, optional); Error getDebugInfo(Cap self, IInvocation* msg); Error invokeSetFSGS(Tasklet* t, Cap self, IInvocation* msg); + Error invokeSetPriority(Tasklet*, Cap, IInvocation* msg); protected: friend class CapRefBind; diff --git a/kernel/objects/scheduling-context/objects/SchedulingContext.cc b/kernel/objects/scheduling-context/objects/SchedulingContext.cc index 456dcbdc..f5cf1931 100644 --- a/kernel/objects/scheduling-context/objects/SchedulingContext.cc +++ b/kernel/objects/scheduling-context/objects/SchedulingContext.cc @@ -45,8 +45,9 @@ namespace mythos { ASSERT(ec != nullptr); MLOG_INFO(mlog::sched, "unbind", DVAR(ec->get())); readyQueue.remove(ec); + priorityQueue.remove(ec); current_handle.store(nullptr); - if(readyQueue.empty()){ + if(readyQueue.empty() && priorityQueue.empty()){ MLOG_DETAIL(mlog::sched, "call idleSC"); event::idleSC.emit(&paTask, home->getThreadID()); }else{ @@ -67,11 +68,17 @@ namespace mythos { // add to the ready queue readyQueue.remove(ec); /// @todo do not need to remove if already on the queue, just do nothing then. This needs additional information in handle_t of LinkedList - readyQueue.push(ec); + priorityQueue.remove(ec); /// @todo do not need to remove if already on the queue, just do nothing then. This needs additional information in handle_t of LinkedList + if(ec->get()->hasPriority()){ + priorityQueue.push(ec); + home->preempt(); // trigger reschedule to run priority ec + }else{ + readyQueue.push(ec); - // wake up the hardware thread if it has no execution context running - // or if if current ec got ready in case of race condition - if (current == nullptr || current == ec) home->preempt(); + // wake up the hardware thread if it has no execution context running + // or if if current ec got ready in case of race condition + if (current == nullptr || current == ec) home->preempt(); + } } void SchedulingContext::tryRunUser() @@ -81,24 +88,38 @@ namespace mythos { while (true) { auto current = current_handle.load(); if (current != nullptr) { - MLOG_DETAIL(mlog::sched, "try current", current, DVAR(current->get())); - auto loaded = current_ec->load(); - if (loaded != current->get()) { - if (loaded != nullptr) loaded->saveState(); - current->get()->loadState(); + if(current->get()->hasPriority() || priorityQueue.empty()){ + MLOG_DETAIL(mlog::sched, "try current", current, DVAR(current->get())); + auto loaded = current_ec->load(); + if (loaded != current->get()) { + if (loaded != nullptr) loaded->saveState(); + current->get()->loadState(); + } + current->get()->resume(); // if it returns, the ec is blocked + current_handle.store(nullptr); + }else{ + MLOG_DETAIL(mlog::sched, "try switching to priority EC from non-prio EC ", current, DVAR(current->get())); + readyQueue.push(current); + current_handle.store(nullptr); } - current->get()->resume(); // if it returns, the ec is blocked - current_handle.store(nullptr); } - MLOG_DETAIL(mlog::sched, "try from ready list"); - // something on the ready list? - auto next = readyQueue.pull(); - while (next != nullptr && !next->get()->isReady()) next = readyQueue.pull(); + + MLOG_DETAIL(mlog::sched, "try from priority list"); + auto next = priorityQueue.pull(); + while (next != nullptr && !next->get()->isReady()) next = priorityQueue.pull(); if (next == nullptr) { - // go sleeping because we don't have anything to run - MLOG_DETAIL(mlog::sched, "empty ready list, going to sleep"); - return; - } + MLOG_DETAIL(mlog::sched, "empty priority list, try ready list"); + // something on the ready list? + next = readyQueue.pull(); + while (next != nullptr && !next->get()->isReady()) next = readyQueue.pull(); + if (next == nullptr) { + // go sleeping because we don't have anything to run + MLOG_DETAIL(mlog::sched, "empty ready list, going to sleep"); + return; + } + }else{ + MLOG_DETAIL(mlog::sched, "found priority EC", next, DVAR(next->get())); + } current_handle.store(next); // now retry } diff --git a/kernel/objects/scheduling-context/objects/SchedulingContext.hh b/kernel/objects/scheduling-context/objects/SchedulingContext.hh index 1f82f78e..e0067f25 100644 --- a/kernel/objects/scheduling-context/objects/SchedulingContext.hh +++ b/kernel/objects/scheduling-context/objects/SchedulingContext.hh @@ -97,6 +97,7 @@ namespace mythos { private: async::Place* home = nullptr; list_t readyQueue; //< the ready list of waiting execution contexts + list_t priorityQueue; //< the ready list of waiting execution contexts with high priority std::atomic current_handle = {nullptr}; //< the currently selected execution context Tasklet paTask; //task for communication with processor allocator diff --git a/kernel/runtime/kobject/runtime/ExecutionContext.hh b/kernel/runtime/kobject/runtime/ExecutionContext.hh index 2d0dba31..bb989e71 100644 --- a/kernel/runtime/kobject/runtime/ExecutionContext.hh +++ b/kernel/runtime/kobject/runtime/ExecutionContext.hh @@ -153,6 +153,9 @@ namespace mythos { PortalFuture setFSGS(PortalLock pr, uintptr_t fs, uintptr_t gs) { return pr.invoke(_cap, fs,gs); } + PortalFuture setPriority(PortalLock pr, bool priority) { + return pr.invoke(_cap, priority); + } PortalFuture resume(PortalLock pr) { return pr.invoke(_cap); } From 840bf6a9980e17d6154670f0cce7876abb56bd28 Mon Sep 17 00:00:00 2001 From: Philipp Gypser Date: Thu, 20 Jan 2022 10:40:28 +0100 Subject: [PATCH 2/3] test case for priority scheduling --- kernel/app/init-example/app/init.cc | 77 +++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/kernel/app/init-example/app/init.cc b/kernel/app/init-example/app/init.cc index 588c53d6..3eb1d4e5 100644 --- a/kernel/app/init-example/app/init.cc +++ b/kernel/app/init-example/app/init.cc @@ -366,6 +366,82 @@ void test_ExecutionContext() MLOG_INFO(mlog::app, "End Test ExecutionContext"); } +std::atomic prio_flag; + +void* prio_main(void* ctx) +{ + if(ctx != nullptr){ + MLOG_INFO(mlog::app, "hello priority thread!", DVAR(ctx)); + ASSERT(prio_flag.load() == 1); + prio_flag++; + for (volatile int i=0; i<100000; i++) { + for (volatile int j=0; j<1000; j++) {} + } + MLOG_INFO(mlog::app, "priority thread finished", DVAR(ctx)); + }else{ + MLOG_INFO(mlog::app, "hello normal thread!", DVAR(ctx)); + ASSERT(prio_flag.load() == 0); + prio_flag++; + while(prio_flag.load() < 3); + MLOG_INFO(mlog::app, "normal thread finished", DVAR(ctx)); + } + + prio_flag++; + return 0; +} + +void test_PrioEC() +{ + MLOG_INFO(mlog::app, "Test ExecutionContext priority scheduling"); + mythos::ExecutionContext ec1(capAlloc()); + mythos::ExecutionContext ec2(capAlloc()); + + prio_flag.store(0); + + { + MLOG_INFO(mlog::app, "test_EC: create ec"); + mythos::PortalLock pl(portal); // future access will fail if the portal is in use already + + // create new ec + auto tls1 = mythos::setupNewTLS(); + auto tls2 = mythos::setupNewTLS(); + ASSERT(tls1 != nullptr); + ASSERT(tls2 != nullptr); + + auto sc = pa.alloc(pl).wait(); + TEST(sc); + + auto res = ec1.create(kmem).as(myAS).cs(myCS).sched(sc->cap) + .prepareStack(thread1stack_top).startFun(&prio_main, nullptr) + .suspended(true).fs(tls1) + .invokeVia(pl).wait(); + TEST(res); + + res = ec2.create(kmem).as(myAS).cs(myCS).sched(sc->cap) + .prepareStack(thread2stack_top).startFun(&prio_main, (void*)1) + .suspended(true).fs(tls2) + .invokeVia(pl).wait(); + TEST(res); + + ec2.setPriority(pl, true).wait(); + + ec1.resume(pl).wait(); + + MLOG_INFO(mlog::app, "wait for EC1 to start"); + while(prio_flag.load() == 0); + + ec2.resume(pl).wait(); + + MLOG_INFO(mlog::app, "wait for ECs to finish execution"); + while(prio_flag < 4); + + MLOG_INFO(mlog::app, "free ECs"); + TEST(capAlloc.free(ec1, pl)); + TEST(capAlloc.free(ec2, pl)); + } + MLOG_INFO(mlog::app, "End Test ExecutionContext priority scheduling"); +} + void test_InterruptControl() { MLOG_INFO(mlog::app, "test_InterruptControl start"); mythos::InterruptControl ic(mythos::init::INTERRUPT_CONTROL_START); @@ -549,6 +625,7 @@ int main() //test_InterruptControl(); //test_HostChannel(portal, 24*1024*1024, 2*1024*1024); test_ExecutionContext(); + test_PrioEC(); test_pthreads(); test_Rapl(); test_processor_allocator(); From a5dc4aeec5e1fc9adb85f90222e3b00b6c858d49 Mon Sep 17 00:00:00 2001 From: Philipp Gypser Date: Thu, 20 Jan 2022 13:32:55 +0100 Subject: [PATCH 3/3] commects for priority scheduling --- kernel/objects/common/objects/ISchedulable.hh | 2 +- kernel/objects/execution-context/objects/ExecutionContext.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/objects/common/objects/ISchedulable.hh b/kernel/objects/common/objects/ISchedulable.hh index 9ccf05b0..08c40f2b 100644 --- a/kernel/objects/common/objects/ISchedulable.hh +++ b/kernel/objects/common/objects/ISchedulable.hh @@ -65,7 +65,7 @@ namespace mythos { virtual void saveState() = 0; - virtual bool hasPriority() { return false; } + virtual bool hasPriority() { return false; } // no high priority per default /** informs the execution context that the user-mode code trapped and error handling is needed. */ virtual void handleTrap() = 0; diff --git a/kernel/objects/execution-context/objects/ExecutionContext.hh b/kernel/objects/execution-context/objects/ExecutionContext.hh index 3b22277d..4d351651 100644 --- a/kernel/objects/execution-context/objects/ExecutionContext.hh +++ b/kernel/objects/execution-context/objects/ExecutionContext.hh @@ -68,7 +68,7 @@ namespace mythos { NOT_LOADED = 1<<8, // CPU state is not loaded DONT_PREEMPT = 1<<9, // somebody else will send the preemption NOT_RUNNING = 1<<10, // EC is not running - PRIORITY = 1<<11, // EC has priority + PRIORITY = 1<<11, // EC has high priority BLOCK_MASK = IS_WAITING | IS_TRAPPED | NO_AS | NO_SCHED | REGISTER_ACCESS };