save test option flags ; ignore stderr or stdout when it should be

Author: csabahruska

external-stg-interpreter/app/RunStgiTestsuite.hs

@@ -18,6 +18,7 @@ import System.Environment
 import Data.Maybe
 import Data.Set (Set)
 import qualified Data.Set as Set
+import qualified Data.Map as Map
 import Data.ByteString (ByteString)
 import qualified Data.ByteString.Char8 as BS8
 import Text.PrettyPrint.ANSI.Leijen hiding ((</>), (<$>))
@@ -110,11 +111,17 @@ runTestProcess path cmd args input = do
   errData <- BS8.readFile stderrPath
   pure (exitCode, outData, errData)
 
+readTestOpts :: FilePath -> IO (Bool, Bool)
+readTestOpts optsPath = do
+  opts <- Map.fromList . read <$> readFile optsPath
+  pure . fromJust $ (,) <$> Map.lookup "ignore_stdout" opts <*> Map.lookup "ignore_stderr" opts
+
 runTest :: Set FilePath -> FilePath -> IO TestResult
 runTest skipSet stderrPath = do
   let stdoutPath    = stderrPath -<.> ".stdout"
       stdinPath     = stderrPath -<.> ".stdin"
       argsPath      = stderrPath -<.> ".args"
+      optsPath      = stderrPath -<.> ".opts"
       exitcodePath  = stderrPath -<.> ".exitcode"
       testName      = takeFileName . dropExtension $ dropExtension stderrPath
       testDir       = takeDirectory stderrPath
@@ -145,6 +152,7 @@ runTest skipSet stderrPath = do
             False -> pure ""
             True  -> BS8.readFile stdinPath
 
+          (ignoreStdout, ignoreStderr) <- readTestOpts optsPath
           expectedStderr <- BS8.readFile stderrPath
           expectedStdout <- BS8.readFile stdoutPath
           expectedExitCode <- (read <$> readFile exitcodePath) >>= \case
@@ -161,7 +169,7 @@ runTest skipSet stderrPath = do
           case mResult of
             Nothing -> report $ Timeout ghcstgappPath
             Just (exitCode, out, err) -> do
-              if expectedExitCode == exitCode && expectedStdout == out && expectedStderr == err
+              if expectedExitCode == exitCode && (expectedStdout == out || ignoreStdout) && (expectedStderr == err || ignoreStderr)
                 then report $ OK ghcstgappPath
                 else report $ Fail ghcstgappPath (expectedExitCode, exitCode) (expectedStdout, out) (expectedStderr, err)
 

ghc-wpc-testsuite-ci/ghc-9.2.7-testsuite.patch

@@ -7,7 +7,7 @@ index 81885ac6fb..17cd446607 100644
          self.rootdirs = []
 
 +        # Run only the runnable tests
-+        self.runnable_only = True
++        self.runnable_only = False
 +
          # Run these tests only (run all tests if empty)
          self.run_only_some_tests = False
@@ -40,10 +40,13 @@ index 8b58084921..188a31971b 100644
      stdout_arg = in_testdir(name, 'run.stdout')
      if opts.combined_output:
          stderr_arg = subprocess.STDOUT # type: Union[int, Path]
-@@ -1739,6 +1749,10 @@ def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: str) ->
+@@ -1739,6 +1749,13 @@ def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: str) ->
      # run the command
      exit_code = runCmd(cmd, stdin_arg, stdout_arg, stderr_arg, opts.run_timeout_multiplier)
 
++    # save test opts
++    in_testdir(name, 'run.opts').write_text(f'[("ignore_stdout",{opts.ignore_stdout}),("ignore_stderr",{opts.ignore_stderr})]')
++
 +    # save exit code
 +    in_testdir(name, 'run.exitcode').write_text(str(exit_code))
 +    in_testdir(name, 'expected.exitcode').write_text(str(opts.exit_code))