[Backtracing] Fix color=tty to check the actual output.

The `color=tty` setting was checking `stdout` to see if it was a TTY, but
we may be using `stderr`, in which case it's easy to end up with coloured
output in a file we're redirecting to.  Fix it to check the selected
output instead.

rdar://164624364

Author: carlpeto

stdlib/public/runtime/Backtrace.cpp

@@ -221,6 +221,16 @@ char swiftBacktraceOutputPath[SWIFT_BACKTRACE_OUTPUT_PATH_SIZE] __attribute__((s
 
 void _swift_backtraceSetupEnvironment();
 
+bool isStderrATty()
+{
+#ifndef _WIN32
+  return isatty(STDERR_FILENO);
+#else
+  DWORD dwMode;
+  return GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &dwMode);
+#endif
+}
+
 bool isStdoutATty()
 {
 #ifndef _WIN32
@@ -412,10 +422,6 @@ BacktraceInitializer::BacktraceInitializer() {
       (isStdoutATty() && isStdinATty()) ? OnOffTty::On : OnOffTty::Off;
   }
 
-  if (_swift_backtraceSettings.color == OnOffTty::TTY)
-    _swift_backtraceSettings.color =
-      isStdoutATty() ? OnOffTty::On : OnOffTty::Off;
-
   if (_swift_backtraceSettings.preset == Preset::Auto) {
     if (_swift_backtraceSettings.interactive == OnOffTty::On)
       _swift_backtraceSettings.preset = Preset::Friendly;
@@ -454,6 +460,16 @@ BacktraceInitializer::BacktraceInitializer() {
       _swift_backtraceSettings.outputTo = OutputTo::Stderr;
   }
 
+  if (_swift_backtraceSettings.color == OnOffTty::TTY) {
+    bool outputToIsTty;
+    if (_swift_backtraceSettings.outputTo == OutputTo::Stderr)
+      outputToIsTty = isStderrATty();
+    else
+      outputToIsTty = isStdoutATty();
+    _swift_backtraceSettings.color =
+      outputToIsTty ? OnOffTty::On : OnOffTty::Off;
+  }
+
   if (_swift_backtraceSettings.enabled == OnOffTty::On) {
     // Copy the path to swift-backtrace into swiftBacktracePath, then write
     // protect it so that it can't be overwritten easily at runtime.  We do