diff --git a/Dockerfile b/Dockerfile index ce6607e..8786afe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.23-bookworm AS builder RUN apt-get update && \ - apt-get install -y sudo debootstrap schroot g++ && \ + apt-get install -y sudo debootstrap schroot && \ rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -17,7 +17,6 @@ FROM debian:bookworm-slim RUN apt-get update && \ apt-get install -y \ - g++ \ docker.io \ && rm -rf /var/lib/apt/lists/* diff --git a/cmd/main.go b/cmd/main.go index e14bf4d..b0f784b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,7 +8,6 @@ import ( "github.com/mini-maxit/worker/internal/rabbitmq/consumer" "github.com/mini-maxit/worker/internal/rabbitmq/responder" "github.com/mini-maxit/worker/internal/scheduler" - "github.com/mini-maxit/worker/internal/stages/compiler" "github.com/mini-maxit/worker/internal/stages/executor" "github.com/mini-maxit/worker/internal/stages/packager" "github.com/mini-maxit/worker/internal/stages/verifier" @@ -47,7 +46,6 @@ func main() { if err := fileCache.InitCache(); err != nil { logger.Fatalf("Failed to initialize file cache: %v", err) } - compiler := compiler.NewCompiler() packager := packager.NewPackager(storageService, fileCache) executor := executor.NewExecutor(dCli) verifier := verifier.NewVerifier(config.VerifierFlags) @@ -57,7 +55,7 @@ func main() { logger.Error("Failed to close responder", err) } }() - scheduler := scheduler.NewScheduler(config.MaxWorkers, compiler, packager, executor, verifier, responder) + scheduler := scheduler.NewScheduler(config.MaxWorkers, packager, executor, verifier, responder) consumer := consumer.NewConsumer(workerChannel, config.ConsumeQueueName, scheduler, responder) // Start listening for messages diff --git a/generate_mocks.sh b/generate_mocks.sh index 2431b88..fbd8bc5 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -12,7 +12,6 @@ INTERFACES=( "internal/rabbitmq/channel Channel" "internal/rabbitmq/responder Responder" "internal/scheduler Scheduler" - "internal/stages/compiler Compiler" "internal/stages/executor Executor" "internal/stages/packager Packager" "internal/stages/verifier Verifier" diff --git a/internal/docker/docker_client.go b/internal/docker/docker_client.go index f5576a4..c361844 100644 --- a/internal/docker/docker_client.go +++ b/internal/docker/docker_client.go @@ -31,6 +31,7 @@ type DockerClient interface { ctx context.Context, containerID, srcPath, dstPath string, allowedDirs []string, + alwaysCopyFiles []string, maxFileSize int64, maxFilesInDir int, ) error @@ -151,6 +152,7 @@ func (d *dockerClient) CopyFromContainerFiltered( ctx context.Context, containerID, srcPath, dstPath string, allowedDirs []string, + alwaysCopyFiles []string, maxFileSize int64, maxFilesInDir int, ) error { @@ -163,7 +165,7 @@ func (d *dockerClient) CopyFromContainerFiltered( defer reader.Close() // Extract tar archive with filtering to destination - err = utils.ExtractTarArchiveFiltered(reader, dstPath, allowedDirs, maxFileSize, maxFilesInDir) + err = utils.ExtractTarArchiveFiltered(reader, dstPath, allowedDirs, alwaysCopyFiles, maxFileSize, maxFilesInDir) if err != nil { d.logger.Errorf("Failed to extract filtered tar archive to %s: %s", dstPath, err) } diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index d3db195..9915786 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -1,17 +1,15 @@ package pipeline import ( - "errors" "fmt" + "os" "github.com/mini-maxit/worker/internal/logger" "github.com/mini-maxit/worker/internal/rabbitmq/responder" - "github.com/mini-maxit/worker/internal/stages/compiler" "github.com/mini-maxit/worker/internal/stages/executor" "github.com/mini-maxit/worker/internal/stages/packager" "github.com/mini-maxit/worker/internal/stages/verifier" "github.com/mini-maxit/worker/pkg/constants" - customErr "github.com/mini-maxit/worker/pkg/errors" "github.com/mini-maxit/worker/pkg/languages" "github.com/mini-maxit/worker/pkg/messages" "github.com/mini-maxit/worker/pkg/solution" @@ -37,7 +35,6 @@ type worker struct { id int state WorkerState responseQueue string - compiler compiler.Compiler packager packager.Packager executor executor.Executor verifier verifier.Verifier @@ -47,7 +44,6 @@ type worker struct { func NewWorker( id int, - compiler compiler.Compiler, packager packager.Packager, executor executor.Executor, verifier verifier.Verifier, @@ -58,7 +54,6 @@ func NewWorker( return &worker{ id: id, state: WorkerState{Status: constants.WorkerStatusIdle, ProcessingMessageID: ""}, - compiler: compiler, packager: packager, executor: executor, verifier: verifier, @@ -136,28 +131,7 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta } }() - err = ws.compiler.CompileSolutionIfNeeded( - langType, - task.LanguageVersion, - dc.UserSolutionPath, - dc.UserExecFilePath, - dc.CompileErrFilePath, - messageID) - - if err != nil { - if errors.Is(err, customErr.ErrCompilationFailed) { - ws.publishCompilationError(dc, task.TestCases) - return - } - - ws.responder.PublishTaskErrorToResponseQueue( - constants.QueueMessageTypeTask, - ws.state.ProcessingMessageID, - ws.responseQueue, - err, - ) - return - } + requiresCompilation := !langType.IsScriptingLanguage() limits := make([]solution.Limit, len(task.TestCases)) for i, tc := range task.TestCases { @@ -168,11 +142,13 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta } cfg := executor.CommandConfig{ - MessageID: messageID, - DirConfig: dc, - LanguageType: langType, - LanguageVersion: task.LanguageVersion, - TestCases: task.TestCases, + MessageID: messageID, + DirConfig: dc, + LanguageType: langType, + LanguageVersion: task.LanguageVersion, + TestCases: task.TestCases, + SourceFilePath: dc.UserSolutionPath, + RequiresCompiling: requiresCompilation, } err = ws.executor.ExecuteCommand(cfg) @@ -186,6 +162,16 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta return } + if requiresCompilation { + // Check for compilation error + fileInfo, statErr := os.Stat(dc.CompileErrFilePath) + + if statErr == nil && fileInfo.Size() > 0 { + ws.publishCompilationError(dc, task.TestCases) + return + } + } + solutionResult := ws.verifier.EvaluateAllTestCases(dc, task.TestCases, messageID, langType) err = ws.packager.SendSolutionPackage(dc, task.TestCases /*hasCompilationErr*/, false, messageID) @@ -209,6 +195,7 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta } func (ws *worker) publishCompilationError(dirConfig *packager.TaskDirConfig, testCases []messages.TestCase) { + ws.logger.Infof("Compilation error occurred for message ID: %s", ws.state.ProcessingMessageID) sendErr := ws.packager.SendSolutionPackage(dirConfig, testCases, true, ws.state.ProcessingMessageID) if sendErr != nil { ws.responder.PublishTaskErrorToResponseQueue( diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 2e963f5..6033a6d 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -2,13 +2,13 @@ package pipeline_test import ( "errors" + "os" "testing" "time" "github.com/mini-maxit/worker/internal/pipeline" "github.com/mini-maxit/worker/internal/stages/packager" "github.com/mini-maxit/worker/pkg/constants" - pkgerrors "github.com/mini-maxit/worker/pkg/errors" "github.com/mini-maxit/worker/pkg/messages" "github.com/mini-maxit/worker/pkg/solution" mocks "github.com/mini-maxit/worker/tests/mocks" @@ -18,24 +18,19 @@ import ( // setupSuccessfulPipelineMocks configures mocks for a successful task processing flow. func setupSuccessfulPipelineMocks( t *testing.T, - mockCompiler *mocks.MockCompiler, mockPackager *mocks.MockPackager, mockExecutor *mocks.MockExecutor, mockVerifier *mocks.MockVerifier, mockResponder *mocks.MockResponder, ) { + tmpDir := t.TempDir() dir := &packager.TaskDirConfig{ - PackageDirPath: t.TempDir(), + PackageDirPath: tmpDir, UserSolutionPath: "src", UserExecFilePath: "exec", CompileErrFilePath: "compile.err", } mockPackager.EXPECT().PrepareSolutionPackage(gomock.Any(), gomock.Any(), gomock.Any()).Return(dir, nil) - mockCompiler.EXPECT(). - CompileSolutionIfNeeded( - gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(nil) mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).Return(nil) mockVerifier.EXPECT(). EvaluateAllTestCases(dir, gomock.Any(), gomock.Any(), gomock.Any()). @@ -52,15 +47,14 @@ func TestProcessTask_SuccessFlow(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) - setupSuccessfulPipelineMocks(t, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + setupSuccessfulPipelineMocks(t, mockPackager, mockExecutor, mockVerifier, mockResponder) - w := pipeline.NewWorker(1, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(1, mockPackager, mockExecutor, mockVerifier, mockResponder) task := &messages.TaskQueueMessage{ LanguageType: "cpp", @@ -80,28 +74,32 @@ func TestProcessTask_CompilationErrorFlow(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) + tmpDir := t.TempDir() dir := &packager.TaskDirConfig{ - PackageDirPath: t.TempDir(), + PackageDirPath: tmpDir, UserSolutionPath: "src", UserExecFilePath: "exec", - CompileErrFilePath: "compile.err", + CompileErrFilePath: tmpDir + "/compile.err", } mockPackager.EXPECT().PrepareSolutionPackage(gomock.Any(), gomock.Any(), gomock.Any()).Return(dir, nil) - // Simulate compilation failure - mockCompiler.EXPECT(). - CompileSolutionIfNeeded( - gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(pkgerrors.ErrCompilationFailed) + // Executor returns nil (container exited with 0), but we have compilation errors in the file + mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).DoAndReturn( + func(_ interface{}) error { + // Simulate compilation error by writing to the error file + if err := os.WriteFile(dir.CompileErrFilePath, []byte("undefined reference to `main'"), 0644); err != nil { + t.Fatalf("failed to write compile error: %v", err) + } + return nil + }, + ) - // When compilation fails SendSolutionPackage should be called with hasCompilationErr=true + // When compilation error detected, SendSolutionPackage should be called with hasCompilationErr=true mockPackager.EXPECT().SendSolutionPackage(dir, gomock.Any(), true, gomock.Any()).Return(nil) // Expect PublishPayloadTaskRespond called with a compilation error result @@ -116,7 +114,7 @@ func TestProcessTask_CompilationErrorFlow(t *testing.T) { }, ) - w := pipeline.NewWorker(2, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(2, mockPackager, mockExecutor, mockVerifier, mockResponder) task := &messages.TaskQueueMessage{LanguageType: "cpp", LanguageVersion: "11", TestCases: nil} w.ProcessTask("msg-compile", "respQ", task) } @@ -125,7 +123,6 @@ func TestProcessTask_PreparePackageFails(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) @@ -138,7 +135,7 @@ func TestProcessTask_PreparePackageFails(t *testing.T) { ).Return(nil, errors.New("download failed")) mockResponder.EXPECT().PublishTaskErrorToResponseQueue(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - w := pipeline.NewWorker(3, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(3, mockPackager, mockExecutor, mockVerifier, mockResponder) task := &messages.TaskQueueMessage{LanguageType: "cpp"} w.ProcessTask("msg-dl", "respQ", task) } @@ -147,24 +144,19 @@ func TestProcessTask_SendPackageFailsAfterRun(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) + tmpDir := t.TempDir() dir := &packager.TaskDirConfig{ - PackageDirPath: t.TempDir(), + PackageDirPath: tmpDir, UserSolutionPath: "src", UserExecFilePath: "exec", - CompileErrFilePath: "compile.err", + CompileErrFilePath: tmpDir + "/compile.err", } mockPackager.EXPECT().PrepareSolutionPackage(gomock.Any(), gomock.Any(), gomock.Any()).Return(dir, nil) - mockCompiler.EXPECT(). - CompileSolutionIfNeeded( - gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(nil) mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).Return(nil) mockVerifier.EXPECT(). EvaluateAllTestCases(dir, gomock.Any(), gomock.Any(), gomock.Any()). @@ -177,7 +169,7 @@ func TestProcessTask_SendPackageFailsAfterRun(t *testing.T) { mockPackager.EXPECT().SendSolutionPackage(dir, gomock.Any(), false, gomock.Any()).Return(errors.New("upload failed")) mockResponder.EXPECT().PublishTaskErrorToResponseQueue(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - w := pipeline.NewWorker(4, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(4, mockPackager, mockExecutor, mockVerifier, mockResponder) task := &messages.TaskQueueMessage{LanguageType: "cpp"} w.ProcessTask("msg-upload", "respQ", task) } @@ -186,24 +178,19 @@ func TestProcessTask_VerifierPanicRecovered(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) + tmpDir := t.TempDir() dir := &packager.TaskDirConfig{ - PackageDirPath: t.TempDir(), + PackageDirPath: tmpDir, UserSolutionPath: "src", UserExecFilePath: "exec", - CompileErrFilePath: "compile.err", + CompileErrFilePath: tmpDir + "/compile.err", } mockPackager.EXPECT().PrepareSolutionPackage(gomock.Any(), gomock.Any(), gomock.Any()).Return(dir, nil) - mockCompiler.EXPECT(). - CompileSolutionIfNeeded( - gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(nil) mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).Return(nil) // Make verifier panic @@ -215,7 +202,7 @@ func TestProcessTask_VerifierPanicRecovered(t *testing.T) { mockResponder.EXPECT().PublishTaskErrorToResponseQueue(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - w := pipeline.NewWorker(5, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(5, mockPackager, mockExecutor, mockVerifier, mockResponder) task := &messages.TaskQueueMessage{LanguageType: "cpp"} w.ProcessTask("msg-panic", "respQ", task) } @@ -224,15 +211,14 @@ func TestProcessTask_PublishPayloadFails(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) - setupSuccessfulPipelineMocks(t, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + setupSuccessfulPipelineMocks(t, mockPackager, mockExecutor, mockVerifier, mockResponder) - w := pipeline.NewWorker(6, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(6, mockPackager, mockExecutor, mockVerifier, mockResponder) task := &messages.TaskQueueMessage{ LanguageType: "cpp", LanguageVersion: "11", @@ -251,13 +237,12 @@ func TestGetStatus(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) - w := pipeline.NewWorker(7, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(7, mockPackager, mockExecutor, mockVerifier, mockResponder) if status := w.GetState(); status.Status != constants.WorkerStatusIdle { t.Fatalf("expected initial status to be Idle, got %q", status) @@ -278,13 +263,12 @@ func TestGetProcessingMessageID(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockCompiler := mocks.NewMockCompiler(ctrl) mockPackager := mocks.NewMockPackager(ctrl) mockExecutor := mocks.NewMockExecutor(ctrl) mockVerifier := mocks.NewMockVerifier(ctrl) mockResponder := mocks.NewMockResponder(ctrl) - w := pipeline.NewWorker(8, mockCompiler, mockPackager, mockExecutor, mockVerifier, mockResponder) + w := pipeline.NewWorker(8, mockPackager, mockExecutor, mockVerifier, mockResponder) if msgID := w.GetProcessingMessageID(); msgID != "" { t.Fatalf("expected initial processingMessageID to be empty, got %q", msgID) @@ -312,11 +296,6 @@ func TestGetProcessingMessageID(t *testing.T) { ) // The rest of the pipeline should succeed quickly after release - mockCompiler.EXPECT(). - CompileSolutionIfNeeded( - gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(nil) mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).Return(nil) mockVerifier.EXPECT(). EvaluateAllTestCases(dir, gomock.Any(), gomock.Any(), gomock.Any()). @@ -359,3 +338,120 @@ func TestGetProcessingMessageID(t *testing.T) { } } } + +// TestProcessTask_ContainerCompilationSuccess tests compilation happening inside container. +func TestProcessTask_ContainerCompilationSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockPackager := mocks.NewMockPackager(ctrl) + mockExecutor := mocks.NewMockExecutor(ctrl) + mockVerifier := mocks.NewMockVerifier(ctrl) + mockResponder := mocks.NewMockResponder(ctrl) + + tmpDir := t.TempDir() + dir := &packager.TaskDirConfig{ + PackageDirPath: tmpDir, + UserSolutionPath: tmpDir + "/solution.cpp", + UserExecFilePath: tmpDir + "/solution", + CompileErrFilePath: tmpDir + "/compile.err", + } + + mockPackager.EXPECT().PrepareSolutionPackage(gomock.Any(), gomock.Any(), gomock.Any()).Return(dir, nil) + + // Executor returns nil (compilation successful in container, empty error file) + mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).DoAndReturn( + func(cfg interface{}) error { + // Write empty error file (successful compilation) + if err := os.WriteFile(dir.CompileErrFilePath, []byte{}, 0644); err != nil { + t.Fatalf("failed to write empty compile error file: %v", err) + } + return nil + }, + ) + + // Should proceed to verification + mockVerifier.EXPECT(). + EvaluateAllTestCases(dir, gomock.Any(), gomock.Any(), gomock.Any()). + Return(solution.Result{ + StatusCode: solution.Success, + Message: "OK", + }) + + mockPackager.EXPECT().SendSolutionPackage(dir, gomock.Any(), false, gomock.Any()).Return(nil) + mockResponder.EXPECT().PublishPayloadTaskRespond(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + + w := pipeline.NewWorker(9, mockPackager, mockExecutor, mockVerifier, mockResponder) + + task := &messages.TaskQueueMessage{ + LanguageType: "cpp", + LanguageVersion: "11", + TestCases: []messages.TestCase{ + {TimeLimitMs: 100, MemoryLimitKB: 65536}, + }, + } + + w.ProcessTask("msg-container-success", "respQ", task) + + if got := w.GetProcessingMessageID(); got != "" { + t.Fatalf("expected processingMessageID to be cleared, got %q", got) + } +} + +// TestProcessTask_ContainerCompilationErrorDetection tests compilation error detection from container output. +func TestProcessTask_ContainerCompilationErrorDetection(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockPackager := mocks.NewMockPackager(ctrl) + mockExecutor := mocks.NewMockExecutor(ctrl) + mockVerifier := mocks.NewMockVerifier(ctrl) + mockResponder := mocks.NewMockResponder(ctrl) + + tmpDir := t.TempDir() + dir := &packager.TaskDirConfig{ + PackageDirPath: tmpDir, + UserSolutionPath: tmpDir + "/solution.cpp", + UserExecFilePath: tmpDir + "/solution", + CompileErrFilePath: tmpDir + "/compile.err", + } + + mockPackager.EXPECT().PrepareSolutionPackage(gomock.Any(), gomock.Any(), gomock.Any()).Return(dir, nil) + + // Executor succeeds (exit code 0), but has compilation error output + mockExecutor.EXPECT().ExecuteCommand(gomock.Any()).DoAndReturn( + func(cfg interface{}) error { + // Write compilation error message + errMsg := "solution.cpp:5:1: error: 'main' function not defined\n" + if err := os.WriteFile(dir.CompileErrFilePath, []byte(errMsg), 0644); err != nil { + t.Fatalf("failed to write compile error file: %v", err) + } + return nil + }, + ) + + // Should NOT proceed to verification, should report compilation error + mockPackager.EXPECT().SendSolutionPackage(dir, gomock.Any(), true, gomock.Any()).Return(nil) + + mockResponder.EXPECT().PublishPayloadTaskRespond(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(messageType, messageID, responseQueue string, res solution.Result) { + if res.StatusCode != solution.CompilationError { + t.Fatalf("expected compilation error status, got %v", res.StatusCode) + } + }, + ) + + w := pipeline.NewWorker(10, mockPackager, mockExecutor, mockVerifier, mockResponder) + + task := &messages.TaskQueueMessage{ + LanguageType: "cpp", + LanguageVersion: "11", + TestCases: []messages.TestCase{{TimeLimitMs: 100, MemoryLimitKB: 65536}}, + } + + w.ProcessTask("msg-container-error", "respQ", task) + + if got := w.GetProcessingMessageID(); got != "" { + t.Fatalf("expected processingMessageID to be cleared, got %q", got) + } +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 633032d..0581335 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -6,7 +6,6 @@ import ( "github.com/mini-maxit/worker/internal/logger" "github.com/mini-maxit/worker/internal/pipeline" "github.com/mini-maxit/worker/internal/rabbitmq/responder" - "github.com/mini-maxit/worker/internal/stages/compiler" "github.com/mini-maxit/worker/internal/stages/executor" "github.com/mini-maxit/worker/internal/stages/packager" "github.com/mini-maxit/worker/internal/stages/verifier" @@ -30,14 +29,13 @@ type scheduler struct { func NewScheduler( maxWorkers int, - compiler compiler.Compiler, packager packager.Packager, executor executor.Executor, verifier verifier.Verifier, responder responder.Responder, ) Scheduler { - return NewSchedulerWithWorkers(maxWorkers, nil, compiler, packager, executor, verifier, responder) + return NewSchedulerWithWorkers(maxWorkers, nil, packager, executor, verifier, responder) } // NewSchedulerWithWorkers creates a Scheduler using the provided worker map. @@ -46,7 +44,6 @@ func NewScheduler( func NewSchedulerWithWorkers( maxWorkers int, workers map[int]pipeline.Worker, - compiler compiler.Compiler, packager packager.Packager, executor executor.Executor, verifier verifier.Verifier, @@ -55,7 +52,7 @@ func NewSchedulerWithWorkers( if workers == nil { workers = make(map[int]pipeline.Worker, maxWorkers) for i := range maxWorkers { - workers[i] = pipeline.NewWorker(i, compiler, packager, executor, verifier, responder) + workers[i] = pipeline.NewWorker(i, packager, executor, verifier, responder) } } diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 664e39c..fd97a1b 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -19,14 +19,13 @@ func TestNewScheduler(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - compiler := mocktests.NewMockCompiler(ctrl) packager := mocktests.NewMockPackager(ctrl) executor := mocktests.NewMockExecutor(ctrl) verifier := mocktests.NewMockVerifier(ctrl) responder := mocktests.NewMockResponder(ctrl) maxWorkers := 3 - s := NewScheduler(maxWorkers, compiler, packager, executor, verifier, responder) + s := NewScheduler(maxWorkers, packager, executor, verifier, responder) if s == nil { t.Fatalf("NewScheduler returned nil") } @@ -53,7 +52,7 @@ func TestGetWorkersStatus(t *testing.T) { w1.EXPECT().GetState().Return(pipeline.WorkerState{Status: constants.WorkerStatusIdle}).Times(1) w1.EXPECT().GetId().Return(1).Times(1) - s := NewSchedulerWithWorkers(2, map[int]pipeline.Worker{0: w0, 1: w1}, nil, nil, nil, nil, nil) + s := NewSchedulerWithWorkers(2, map[int]pipeline.Worker{0: w0, 1: w1}, nil, nil, nil, nil) st := s.GetWorkersStatus() if len(st.WorkerStatus) != 2 { @@ -111,7 +110,7 @@ func TestProcessTask_SuccessAndMarkIdle(t *testing.T) { // After processing, scheduler.markWorkerAsIdle should call UpdateStatus(Idle) w.EXPECT().UpdateStatus(constants.WorkerStatusIdle).Times(1) - s := NewSchedulerWithWorkers(1, map[int]pipeline.Worker{0: w}, nil, nil, nil, nil, nil) + s := NewSchedulerWithWorkers(1, map[int]pipeline.Worker{0: w}, nil, nil, nil, nil) if err := s.ProcessTask("resp", "msg-id-1", &messages.TaskQueueMessage{}); err != nil { t.Fatalf("unexpected error from ProcessTask: %v", err) @@ -137,7 +136,7 @@ func TestProcessTask_NoFreeWorker(t *testing.T) { // worker reports busy w.EXPECT().GetState().Return(pipeline.WorkerState{Status: constants.WorkerStatusBusy}).Times(1) - s := NewSchedulerWithWorkers(1, map[int]pipeline.Worker{0: w}, nil, nil, nil, nil, nil) + s := NewSchedulerWithWorkers(1, map[int]pipeline.Worker{0: w}, nil, nil, nil, nil) err := s.ProcessTask("resp", "msg-id-2", &messages.TaskQueueMessage{}) if err == nil { diff --git a/internal/stages/compiler/compiler.go b/internal/stages/compiler/compiler.go deleted file mode 100644 index 37f7fc0..0000000 --- a/internal/stages/compiler/compiler.go +++ /dev/null @@ -1,65 +0,0 @@ -package compiler - -import ( - customErr "github.com/mini-maxit/worker/pkg/errors" - "github.com/mini-maxit/worker/pkg/languages" -) - -type Compiler interface { - CompileSolutionIfNeeded( - langType languages.LanguageType, - langVersion string, - sourceFilePath string, - outFilePath string, - compErrFilePath string, - messageID string, - ) error -} - -type compiler struct { -} - -func NewCompiler() Compiler { - return &compiler{} -} - -// LanguageCompiler is a language-specific compiler invoked internally. -type LanguageCompiler interface { - Compile(sourceFilePath, outFilePath, compErrFilePath, messageID string) error -} - -func initializeSolutionCompiler( - langType languages.LanguageType, - langVersion string, - messageID string, -) (LanguageCompiler, error) { - switch langType { - case languages.CPP: - return NewCppCompiler(langVersion, messageID) - case languages.PYTHON: - return nil, customErr.ErrInvalidLanguageType // Make linter happy - default: - return nil, customErr.ErrInvalidLanguageType - } -} - -// get proper compiler and compile if needed. -func (c *compiler) CompileSolutionIfNeeded( - langType languages.LanguageType, - langVersion, - sourceFilePath, - execFilePath, - compErrFilePath, - messageID string, -) error { - if langType.IsScriptingLanguage() { - return nil - } - - compiler, err := initializeSolutionCompiler(langType, langVersion, messageID) - if err != nil { - return err - } - - return compiler.Compile(sourceFilePath, execFilePath, compErrFilePath, messageID) -} diff --git a/internal/stages/compiler/compiler_test.go b/internal/stages/compiler/compiler_test.go deleted file mode 100644 index 3d46ad9..0000000 --- a/internal/stages/compiler/compiler_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package compiler_test - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "errors" - - . "github.com/mini-maxit/worker/internal/stages/compiler" - pkgErr "github.com/mini-maxit/worker/pkg/errors" - "github.com/mini-maxit/worker/pkg/languages" - "github.com/mini-maxit/worker/tests" -) - -func TestNewCompiler(t *testing.T) { - c := NewCompiler() - if c == nil { - t.Fatalf("NewCompiler returned nil") - } -} - -func TestCompileSolutionIfNeeded_Success(t *testing.T) { - dir := t.TempDir() - src := tests.WriteFile(t, dir, "main.cpp", ` - - #include - - int main() { - std::cout << "hello" << std::endl; - return 0; - } - `) - - out := filepath.Join(dir, "outbin") - compErr := filepath.Join(dir, "compile.err") - - c := NewCompiler() - - if err := c.CompileSolutionIfNeeded(languages.CPP, "17", src, out, compErr, "msg-id"); err != nil { - t.Fatalf("expected compile to succeed, got: %v", err) - } - - if info, err := os.Stat(out); err != nil { - t.Fatalf("expected output binary to exist: %v", err) - } else if info.IsDir() { - t.Fatalf("expected output to be a file, got dir") - } - - // run the produced binary to ensure it runs and returns 0 - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, out) - if outb, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("running compiled binary failed: %v, output: %s", err, string(outb)) - } -} - -func TestCompileSolutionIfNeeded_InvalidLanguage(t *testing.T) { - dir := t.TempDir() - src := tests.WriteFile(t, dir, "main.txt", `dummy`) - out := filepath.Join(dir, "out") - compErr := filepath.Join(dir, "compile.err") - - c := NewCompiler() - - // use an invalid language value (0) - err := c.CompileSolutionIfNeeded(languages.LanguageType(0), "", src, out, compErr, "msg-id") - if err == nil { - t.Fatalf("expected error for invalid language") - } - if !errors.Is(err, pkgErr.ErrInvalidLanguageType) { - t.Fatalf("expected ErrInvalidLanguageType, got: %v", err) - } -} diff --git a/internal/stages/compiler/cpp_compiler.go b/internal/stages/compiler/cpp_compiler.go deleted file mode 100644 index 3ae7707..0000000 --- a/internal/stages/compiler/cpp_compiler.go +++ /dev/null @@ -1,72 +0,0 @@ -package compiler - -import ( - "bytes" - "context" - "os" - "os/exec" - - "github.com/mini-maxit/worker/internal/logger" - "github.com/mini-maxit/worker/pkg/errors" - "github.com/mini-maxit/worker/pkg/languages" - "go.uber.org/zap" -) - -type CppCompiler struct { - version string - logger *zap.SugaredLogger -} - -func (e *CppCompiler) RequiresCompilation() bool { - return true -} - -// For now compile allows only one file. -func (e *CppCompiler) Compile(sourceFilePath, execFilePath, compErrFilePath, messageID string) error { - e.logger.Infof("Compiling %s [MsgID: %s]", sourceFilePath, messageID) - // Correctly pass the command and its arguments as separate strings. - versionFlag := "-std=" + e.version - ctx := context.Background() // TODO: use a timeout context - cmd := exec.CommandContext(ctx, "g++", "-o", execFilePath, versionFlag, sourceFilePath) - - var stderr bytes.Buffer - cmd.Stderr = &stderr - - cmdErr := cmd.Run() - if cmdErr != nil { - // Save stderr to a file - e.logger.Errorf("Error during compilation. %s [MsgID: %s]", cmdErr.Error(), messageID) - file, err := os.Create(compErrFilePath) - if err != nil { - e.logger.Errorf("Could not create stderr file. %s [MsgID: %s]", err.Error(), messageID) - return err - } - - _, err = file.Write(stderr.Bytes()) - if err != nil { - e.logger.Errorf("Error writing to file. %s [MsgID: %s]", err.Error(), messageID) - return err - } - err = file.Close() - if err != nil { - e.logger.Errorf("Error closing file. %s [MsgID: %s]", err.Error(), messageID) - return err - } - e.logger.Infof("Compilation error saved to %s [MsgID: %s]", compErrFilePath, messageID) - return errors.ErrCompilationFailed - } - e.logger.Infof("Compilation successful [MsgID: %s]", messageID) - return nil -} - -func NewCppCompiler(version, messageID string) (*CppCompiler, error) { - logger := logger.NewNamedLogger("cpp-compiler") - versionFlag, err := languages.GetVersionFlag(languages.CPP, version) - if err != nil { - logger.Errorf("Failed to get version flag. %s [MsgID: %s]", err.Error(), messageID) - return nil, err - } - return &CppCompiler{ - version: versionFlag, - logger: logger}, nil -} diff --git a/internal/stages/compiler/cpp_compiler_test.go b/internal/stages/compiler/cpp_compiler_test.go deleted file mode 100644 index 702a588..0000000 --- a/internal/stages/compiler/cpp_compiler_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package compiler_test - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "errors" - - . "github.com/mini-maxit/worker/internal/stages/compiler" - pkgErr "github.com/mini-maxit/worker/pkg/errors" - "github.com/mini-maxit/worker/tests" -) - -func TestRequiresCompilation(t *testing.T) { - c, err := NewCppCompiler("17", "msg-test") - if err != nil { - t.Fatalf("NewCppCompiler returned error: %v", err) - } - if !c.RequiresCompilation() { - t.Fatalf("expected RequiresCompilation to be true") - } -} - -func TestCompileSuccess(t *testing.T) { - dir := t.TempDir() - src := tests.WriteFile(t, dir, "main.cpp", ` - - #include - - int main() { - std::cout << "hello" << std::endl; - return 0; - } - `) - - out := filepath.Join(dir, "outbin") - compErr := filepath.Join(dir, "compile.err") - - c, err := NewCppCompiler("17", "msg-test") - if err != nil { - t.Fatalf("NewCppCompiler returned error: %v", err) - } - - if err := c.Compile(src, out, compErr, "msg-id"); err != nil { - t.Fatalf("expected compile to succeed, got: %v", err) - } - - // binary should exist and be executable - if info, err := os.Stat(out); err != nil { - t.Fatalf("expected output binary to exist: %v", err) - } else if info.IsDir() { - t.Fatalf("expected output to be a file, got dir") - } - - // run the produced binary to ensure it runs and returns 0 - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, out) - if outb, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("running compiled binary failed: %v, output: %s", err, string(outb)) - } -} - -func TestCompileFailureProducesErrorFile(t *testing.T) { - dir := t.TempDir() - // intentionally broken C++ source - src := tests.WriteFile(t, dir, "bad.cpp", ` - - #include - - int main() { - this is not valid C++ - } - `) - - out := filepath.Join(dir, "outbad") - compErr := filepath.Join(dir, "compile.err") - - c, err := NewCppCompiler("17", "msg-id") - if err != nil { - t.Fatalf("NewCppCompiler returned error: %v", err) - } - - err = c.Compile(src, out, compErr, "msg-id") - if err == nil { - t.Fatalf("expected compile to fail for broken source") - } - if !errors.Is(err, pkgErr.ErrCompilationFailed) { - t.Fatalf("expected ErrCompilationFailed, got: %v", err) - } - - // error file should exist and contain some data - info, statErr := os.Stat(compErr) - if statErr != nil { - t.Fatalf("expected compile.err to exist, stat error: %v", statErr) - } - if info.Size() == 0 { - t.Fatalf("expected compile.err to contain compiler stderr") - } -} - -func TestNewCppCompilerIntegration(t *testing.T) { - // NewCppCompiler uses languages.GetVersionFlag; pass valid version '17' - cc, err := NewCppCompiler("17", "msg-test") - if err != nil { - t.Fatalf("NewCppCompiler returned error: %v", err) - } - - dir := t.TempDir() - src := tests.WriteFile(t, dir, "main2.cpp", ` - #include - - int main() { - printf("ok\n"); - return 0; - } - `) - out := filepath.Join(dir, "out2") - compErr := filepath.Join(dir, "compile2.err") - - if err := cc.Compile(src, out, compErr, "msg-test"); err != nil { - t.Fatalf("expected compile to succeed using NewCppCompiler, got: %v", err) - } -} diff --git a/internal/stages/executor/cpp/Dockerfile b/internal/stages/executor/cpp/Dockerfile index 3ff614c..53f991d 100644 --- a/internal/stages/executor/cpp/Dockerfile +++ b/internal/stages/executor/cpp/Dockerfile @@ -3,7 +3,7 @@ FROM debian:bookworm-slim ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - libstdc++6 ca-certificates time \ + libstdc++6 ca-certificates time g++\ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY run_tests.sh /usr/local/bin/run_tests.sh diff --git a/internal/stages/executor/executor.go b/internal/stages/executor/executor.go index 93c47ea..76c251a 100644 --- a/internal/stages/executor/executor.go +++ b/internal/stages/executor/executor.go @@ -27,11 +27,13 @@ import ( var containerNameRegex = regexp.MustCompile("[^a-zA-Z0-9_.-]") type CommandConfig struct { - MessageID string - DirConfig *packager.TaskDirConfig - LanguageType languages.LanguageType - LanguageVersion string - TestCases []messages.TestCase + MessageID string + DirConfig *packager.TaskDirConfig + LanguageType languages.LanguageType + LanguageVersion string + TestCases []messages.TestCase + SourceFilePath string + RequiresCompiling bool } type ExecutionResult struct { @@ -136,12 +138,18 @@ func (d *executor) ExecuteCommand( constants.UserDiffDirName, constants.UserExecResultDirName, } + + alwaysCopyFiles := []string{ + filepath.Base(cfg.DirConfig.CompileErrFilePath), + } + err = d.docker.CopyFromContainerFiltered( copyCtx, containerID, cfg.DirConfig.PackageDirPath, cfg.DirConfig.TmpDirPath, allowedDirs, + alwaysCopyFiles, constants.MaxContainerOutputFileSize, len(cfg.TestCases), ) @@ -181,6 +189,27 @@ func SanitizeContainerName(raw string) string { return "submission-" + cleaned } +func buildCompileCommand(cfg CommandConfig) ([]string, error) { + switch cfg.LanguageType { + case languages.CPP: + versionFlag, err := languages.GetVersionFlag(languages.CPP, cfg.LanguageVersion) + if err != nil { + return []string{}, err + } + return []string{ + "g++", + "-o", + filepath.Base(cfg.DirConfig.UserExecFilePath), + "-std=" + versionFlag, + filepath.Base(cfg.SourceFilePath), + }, nil + case languages.PYTHON: // make linter happy. + return []string{}, nil + default: + return []string{}, nil + } +} + func buildEnvironmentVariables(cfg CommandConfig) ([]string, error) { timeEnv := make([]string, len(cfg.TestCases)) memEnv := make([]string, len(cfg.TestCases)) @@ -220,7 +249,7 @@ func buildEnvironmentVariables(cfg CommandConfig) ([]string, error) { fmt.Sprintf("%d.%s", tc.Order, constants.ExecutionResultFileExt)) } - return []string{ + envVars := []string{ "RUN_CMD=" + utils.ShellQuoteSlice(runCmd), "TIME_LIMITS_MS=" + utils.ShellQuoteSlice(timeEnv), "MEM_LIMITS_KB=" + utils.ShellQuoteSlice(memEnv), @@ -228,7 +257,23 @@ func buildEnvironmentVariables(cfg CommandConfig) ([]string, error) { "USER_OUTPUT_FILES=" + utils.ShellQuoteSlice(userOutputFilePaths), "USER_ERROR_FILES=" + utils.ShellQuoteSlice(userErrorFilePaths), "USER_EXEC_RESULT_FILES=" + utils.ShellQuoteSlice(userExecResultFilePaths), - }, nil + } + + if cfg.RequiresCompiling { + compileCmd, err := buildCompileCommand(cfg) + if err != nil { + return nil, err + } + envVars = append(envVars, + "REQUIRES_COMPILATION=true", + "COMPILE_CMD="+utils.ShellQuoteSlice(compileCmd), + "SOURCE_FILE="+filepath.Base(cfg.SourceFilePath), + "EXEC_FILE="+filepath.Base(cfg.DirConfig.UserExecFilePath), + "COMPILE_ERR_FILE="+filepath.Base(cfg.DirConfig.CompileErrFilePath), + ) + } + + return envVars, nil } func buildContainerConfig( diff --git a/internal/stages/executor/executor_test.go b/internal/stages/executor/executor_test.go index fbbef7a..02cb478 100644 --- a/internal/stages/executor/executor_test.go +++ b/internal/stages/executor/executor_test.go @@ -78,7 +78,7 @@ func setupMockExpectations( gomock.Any(), containerID, gomock.Any(), ).Return(statusCode, nil).Times(1), mockDocker.EXPECT().CopyFromContainerFiltered( - gomock.Any(), containerID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), containerID, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), ).Return(nil).Times(1), mockDocker.EXPECT().ContainerRemove(gomock.Any(), containerID).Times(1), ) @@ -118,7 +118,7 @@ func TestExecuteCommand_Success(t *testing.T) { gomock.Any(), "cid123", gomock.Any(), ).Return(int64(0), nil).Times(1), mockDocker.EXPECT().CopyFromContainerFiltered( - gomock.Any(), "cid123", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), "cid123", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), ).Return(nil).Times(1), mockDocker.EXPECT().ContainerRemove(gomock.Any(), "cid123").Times(1), ) @@ -320,7 +320,9 @@ func TestExecuteCommand_CopyFromContainerFails(t *testing.T) { gomock.Any(), "cid-copyfrom-fail", gomock.Any(), ).Return(int64(0), nil).Times(1), mockDocker.EXPECT().CopyFromContainerFiltered( - gomock.Any(), "cid-copyfrom-fail", gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), + "cid-copyfrom-fail", + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), ).Return(errors.New("copy-from-failed")).Times(1), mockDocker.EXPECT().ContainerRemove(gomock.Any(), "cid-copyfrom-fail").Times(1), ) diff --git a/internal/stages/executor/run_tests.sh b/internal/stages/executor/run_tests.sh index 87bc4ef..897087f 100644 --- a/internal/stages/executor/run_tests.sh +++ b/internal/stages/executor/run_tests.sh @@ -29,6 +29,50 @@ if [[ -z "${USER_ERROR_FILES:-}" ]]; then exit 1 fi +# Handle compilation if needed +if [[ "${REQUIRES_COMPILATION:-}" == "true" ]]; then + if [[ -z "${COMPILE_CMD:-}" ]]; then + echo "ERROR: COMPILE_CMD must be set when REQUIRES_COMPILATION is true" >&2 + exit 1 + fi + if [[ -z "${SOURCE_FILE:-}" ]]; then + echo "ERROR: SOURCE_FILE must be set when REQUIRES_COMPILATION is true" >&2 + exit 1 + fi + if [[ -z "${EXEC_FILE:-}" ]]; then + echo "ERROR: EXEC_FILE must be set when REQUIRES_COMPILATION is true" >&2 + exit 1 + fi + if [[ -z "${COMPILE_ERR_FILE:-}" ]]; then + echo "ERROR: COMPILE_ERR_FILE must be set when REQUIRES_COMPILATION is true" >&2 + exit 1 + fi + + read -r -a compile_cmd <<< "$COMPILE_CMD" + + echo "Compiling: ${compile_cmd[@]}" + + # Run compilation and capture stderr + if ! "${compile_cmd[@]}" 2> "${COMPILE_ERR_FILE}"; then + echo "Compilation failed. Error details saved to ${COMPILE_ERR_FILE}" + + # Intenially exit with 0 as this is not treated as a container failure, + # compilation errors are handled separately in the pipeline by inspecting the compile error file. + exit 0 + fi + + # Check if executable was created + if [[ ! -f "${EXEC_FILE}" ]]; then + echo "Compilation failed: executable not created" > "${COMPILE_ERR_FILE}" + + # Same as above with + exit 0 + fi + + # Make executable if needed + chmod +x "${EXEC_FILE}" 2>/dev/null || true +fi + read -r -a times <<< "$TIME_LIMITS_MS" read -r -a mems <<< "$MEM_LIMITS_KB" read -r -a inputs <<< "$INPUT_FILES" diff --git a/tests/mocks/mocks_generated.go b/tests/mocks/mocks_generated.go index 575638c..81dc287 100644 --- a/tests/mocks/mocks_generated.go +++ b/tests/mocks/mocks_generated.go @@ -281,54 +281,6 @@ func (mr *MockSchedulerMockRecorder) ProcessTask(responseQueueName, messageID, t return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessTask", reflect.TypeOf((*MockScheduler)(nil).ProcessTask), responseQueueName, messageID, task) } -// Code generated by MockGen. DO NOT EDIT. -// Source: /Users/mateuszosik/repos/Testerka/worker/internal/stages/compiler (interfaces: Compiler) -// -// Generated by this command: -// -// mockgen /Users/mateuszosik/repos/Testerka/worker/internal/stages/compiler Compiler -// - -// Package mock_compiler is a generated GoMock package. - -// MockCompiler is a mock of Compiler interface. -type MockCompiler struct { - ctrl *gomock.Controller - recorder *MockCompilerMockRecorder - isgomock struct{} -} - -// MockCompilerMockRecorder is the mock recorder for MockCompiler. -type MockCompilerMockRecorder struct { - mock *MockCompiler -} - -// NewMockCompiler creates a new mock instance. -func NewMockCompiler(ctrl *gomock.Controller) *MockCompiler { - mock := &MockCompiler{ctrl: ctrl} - mock.recorder = &MockCompilerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockCompiler) EXPECT() *MockCompilerMockRecorder { - return m.recorder -} - -// CompileSolutionIfNeeded mocks base method. -func (m *MockCompiler) CompileSolutionIfNeeded(langType languages.LanguageType, langVersion, sourceFilePath, outFilePath, compErrFilePath, messageID string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CompileSolutionIfNeeded", langType, langVersion, sourceFilePath, outFilePath, compErrFilePath, messageID) - ret0, _ := ret[0].(error) - return ret0 -} - -// CompileSolutionIfNeeded indicates an expected call of CompileSolutionIfNeeded. -func (mr *MockCompilerMockRecorder) CompileSolutionIfNeeded(langType, langVersion, sourceFilePath, outFilePath, compErrFilePath, messageID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompileSolutionIfNeeded", reflect.TypeOf((*MockCompiler)(nil).CompileSolutionIfNeeded), langType, langVersion, sourceFilePath, outFilePath, compErrFilePath, messageID) -} - // Code generated by MockGen. DO NOT EDIT. // Source: /Users/mateuszosik/repos/Testerka/worker/internal/stages/executor (interfaces: Executor) // @@ -710,17 +662,17 @@ func (mr *MockDockerClientMockRecorder) ContainerRemove(ctx, containerID any) *g } // CopyFromContainerFiltered mocks base method. -func (m *MockDockerClient) CopyFromContainerFiltered(ctx context.Context, containerID, srcPath, dstPath string, allowedDirs []string, maxFileSize int64, maxFilesInDir int) error { +func (m *MockDockerClient) CopyFromContainerFiltered(ctx context.Context, containerID, srcPath, dstPath string, allowedDirs, alwaysCopyFiles []string, maxFileSize int64, maxFilesInDir int) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CopyFromContainerFiltered", ctx, containerID, srcPath, dstPath, allowedDirs, maxFileSize, maxFilesInDir) + ret := m.ctrl.Call(m, "CopyFromContainerFiltered", ctx, containerID, srcPath, dstPath, allowedDirs, alwaysCopyFiles, maxFileSize, maxFilesInDir) ret0, _ := ret[0].(error) return ret0 } // CopyFromContainerFiltered indicates an expected call of CopyFromContainerFiltered. -func (mr *MockDockerClientMockRecorder) CopyFromContainerFiltered(ctx, containerID, srcPath, dstPath, allowedDirs, maxFileSize, maxFilesInDir any) *gomock.Call { +func (mr *MockDockerClientMockRecorder) CopyFromContainerFiltered(ctx, containerID, srcPath, dstPath, allowedDirs, alwaysCopyFiles, maxFileSize, maxFilesInDir any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyFromContainerFiltered", reflect.TypeOf((*MockDockerClient)(nil).CopyFromContainerFiltered), ctx, containerID, srcPath, dstPath, allowedDirs, maxFileSize, maxFilesInDir) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyFromContainerFiltered", reflect.TypeOf((*MockDockerClient)(nil).CopyFromContainerFiltered), ctx, containerID, srcPath, dstPath, allowedDirs, alwaysCopyFiles, maxFileSize, maxFilesInDir) } // CopyToContainerFiltered mocks base method. diff --git a/utils/utils.go b/utils/utils.go index 4d83334..f6bc3a7 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -347,28 +347,79 @@ func validateAndTrackFileCount(header *tar.Header, dirFileCount map[string]int, return nil } +type tarExtractionContext struct { + absDstPath string + allowedSet map[string]struct{} + alwaysCopySet map[string]struct{} + dirFileCount map[string]int + maxFileSize int64 + maxFilesInDir int +} + +func (ctx *tarExtractionContext) shouldExtractEntry(header *tar.Header) bool { + _, alwaysCopy := ctx.alwaysCopySet[filepath.Base(header.Name)] + return alwaysCopy || isAllowedDirectory(header, ctx.allowedSet) +} + +func (ctx *tarExtractionContext) validateEntry(header *tar.Header) error { + if err := validateTarEntrySize(header, ctx.maxFileSize); err != nil { + return err + } + + if err := validatePathDepth(header.Name); err != nil { + return err + } + + _, alwaysCopy := ctx.alwaysCopySet[filepath.Base(header.Name)] + if !alwaysCopy { + if err := validateAndTrackFileCount(header, ctx.dirFileCount, ctx.maxFilesInDir); err != nil { + return err + } + } + + return nil +} + +func (ctx *tarExtractionContext) processEntry(tarReader *tar.Reader, header *tar.Header) error { + if !ctx.shouldExtractEntry(header) { + return nil + } + + if err := ctx.validateEntry(header); err != nil { + return err + } + + target, err := safeArchiveTarget(ctx.absDstPath, header.Name) + if err != nil { + return err + } + + return materializeTarEntry(tarReader, header, target) +} + func ExtractTarArchiveFiltered( reader io.Reader, dstPath string, allowedDirs []string, + alwaysCopyFiles []string, maxFileSize int64, maxFilesInDir int, ) error { - tarReader := tar.NewReader(reader) - absDstPath, err := filepath.Abs(dstPath) if err != nil { return err } - allowedSet := make(map[string]struct{}, len(allowedDirs)) - for _, dir := range allowedDirs { - allowedSet[dir] = struct{}{} + ctx := &tarExtractionContext{ + absDstPath: absDstPath, + allowedSet: makeStringSet(allowedDirs), + alwaysCopySet: makeStringSet(alwaysCopyFiles), + dirFileCount: make(map[string]int, len(allowedDirs)), + maxFileSize: maxFileSize, + maxFilesInDir: maxFilesInDir, } - // Track files per allowed directory to enforce per-directory limits - dirFileCount := make(map[string]int, len(allowedDirs)) - + tarReader := tar.NewReader(reader) for { header, err := tarReader.Next() if errors.Is(err, io.EOF) { @@ -378,29 +429,16 @@ func ExtractTarArchiveFiltered( return err } - if !isAllowedDirectory(header, allowedSet) { - continue - } - - if err := validateTarEntrySize(header, maxFileSize); err != nil { - return err - } - - if err := validatePathDepth(header.Name); err != nil { - return err - } - - if err := validateAndTrackFileCount(header, dirFileCount, maxFilesInDir); err != nil { - return err - } - - target, err := safeArchiveTarget(absDstPath, header.Name) - if err != nil { + if err := ctx.processEntry(tarReader, header); err != nil { return err } + } +} - if err := materializeTarEntry(tarReader, header, target); err != nil { - return err - } +func makeStringSet(items []string) map[string]struct{} { + set := make(map[string]struct{}, len(items)) + for _, item := range items { + set[item] = struct{}{} } + return set }