Backtrace incorrectly picks up colour from stdout instead of stderr

stdlib/public/runtime/Backtrace.cpp

@@ -221,6 +221,16 @@ char swiftBacktraceOutputPath[SWIFT_BACKTRACE_OUTPUT_PATH_SIZE] __attribute__((s
 
 void _swift_backtraceSetupEnvironment();
 
+bool isStderrATty()
+{
+#ifndef _WIN32
+  return isatty(STDERR_FILENO);
+#else
+  DWORD dwMode;
+  return GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &dwMode);
+#endif
+}
+
 bool isStdoutATty()
 {
 #ifndef _WIN32
@@ -412,10 +422,6 @@ BacktraceInitializer::BacktraceInitializer() {
       (isStdoutATty() && isStdinATty()) ? OnOffTty::On : OnOffTty::Off;
   }
 
-  if (_swift_backtraceSettings.color == OnOffTty::TTY)
-    _swift_backtraceSettings.color =
-      isStdoutATty() ? OnOffTty::On : OnOffTty::Off;
-
   if (_swift_backtraceSettings.preset == Preset::Auto) {
     if (_swift_backtraceSettings.interactive == OnOffTty::On)
       _swift_backtraceSettings.preset = Preset::Friendly;
@@ -454,6 +460,16 @@ BacktraceInitializer::BacktraceInitializer() {
       _swift_backtraceSettings.outputTo = OutputTo::Stderr;
   }
 
+  if (_swift_backtraceSettings.color == OnOffTty::TTY) {
+    bool outputToIsTty;
+    if (_swift_backtraceSettings.outputTo == OutputTo::Stderr)
+      outputToIsTty = isStderrATty();
+    else
+      outputToIsTty = isStdoutATty();
+    _swift_backtraceSettings.color =
+      outputToIsTty ? OnOffTty::On : OnOffTty::Off;
+  }
+
   if (_swift_backtraceSettings.enabled == OnOffTty::On) {
     // Copy the path to swift-backtrace into swiftBacktracePath, then write
     // protect it so that it can't be overwritten easily at runtime.  We do

test/Backtracing/TTYDetection.swift

@@ -0,0 +1,73 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift %s -parse-as-library -Onone -g -o %t/TTYDetection
+// RUN: %target-codesign %t/TTYDetection
+
+// RUN: (env SWIFT_BACKTRACE=enable=yes,cache=no %target-run %t/TTYDetection 2> %t/default-output || true) && cat %t/default-output | %FileCheck %s
+// RUN: (env SWIFT_BACKTRACE=enable=yes,cache=no,output-to=stderr %target-run %t/TTYDetection 2> %t/stderr-output || true) && cat %t/stderr-output | %FileCheck %s --check-prefix STDERR
+// RUN: (env SWIFT_BACKTRACE=enable=yes,cache=no,output-to=stdout %target-run %t/TTYDetection > %t/stdout-output || true) && cat %t/stdout-output | %FileCheck %s --check-prefix STDOUT
+
+
+// UNSUPPORTED: use_os_stdlib
+// UNSUPPORTED: back_deployment_runtime
+// UNSUPPORTED: asan
+// REQUIRES: executable_test
+// REQUIRES: backtracing
+// REQUIRES: OS=macosx || OS=linux-gnu
+// COM: we should be able to add Windows to this test
+
+func level1() {
+  level2()
+}
+
+func level2() {
+  level3()
+}
+
+func level3() {
+  level4()
+}
+
+func level4() {
+  level5()
+}
+
+func level5() {
+  print("About to crash")
+  let ptr = UnsafeMutablePointer<Int>(bitPattern: 4)!
+  ptr.pointee = 42
+}
+
+@main
+struct TTYDetection {
+  static func main() {
+    level1()
+  }
+}
+
+// CHECK: *** Program crashed: Bad pointer dereference at 0x{{0+}}4 ***
+
+// CHECK: Thread 0 {{(".*" )?}}crashed:
+
+// CHECK: 0               0x{{[0-9a-f]+}} level5() + {{[0-9]+}} in TTYDetection
+// CHECK-NEXT: 1 [ra]          0x{{[0-9a-f]+}} level4() + {{[0-9]+}} in TTYDetection
+// CHECK-NEXT: 2 [ra]          0x{{[0-9a-f]+}} level3() + {{[0-9]+}} in TTYDetection
+// CHECK-NEXT: 3 [ra]          0x{{[0-9a-f]+}} level2() + {{[0-9]+}} in TTYDetection
+
+
+// STDERR: *** Program crashed: Bad pointer dereference at 0x{{0+}}4 ***
+
+// STDERR: Thread 0 {{(".*" )?}}crashed:
+
+// STDERR: 0               0x{{[0-9a-f]+}} level5() + {{[0-9]+}} in TTYDetection
+// STDERR-NEXT: 1 [ra]          0x{{[0-9a-f]+}} level4() + {{[0-9]+}} in TTYDetection
+// STDERR-NEXT: 2 [ra]          0x{{[0-9a-f]+}} level3() + {{[0-9]+}} in TTYDetection
+// STDERR-NEXT: 3 [ra]          0x{{[0-9a-f]+}} level2() + {{[0-9]+}} in TTYDetection
+
+// STDOUT: *** Program crashed: Bad pointer dereference at 0x{{0+}}4 ***
+
+// STDOUT: Thread 0 {{(".*" )?}}crashed:
+
+// STDOUT: 0               0x{{[0-9a-f]+}} level5() + {{[0-9]+}} in TTYDetection
+// STDOUT-NEXT: 1 [ra]          0x{{[0-9a-f]+}} level4() + {{[0-9]+}} in TTYDetection
+// STDOUT-NEXT: 2 [ra]          0x{{[0-9a-f]+}} level3() + {{[0-9]+}} in TTYDetection
+// STDOUT-NEXT: 3 [ra]          0x{{[0-9a-f]+}} level2() + {{[0-9]+}} in TTYDetection

test/Backtracing/TTYDetectionColourLinux.swift

@@ -0,0 +1,46 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift %s -parse-as-library -Onone -g -o %t/TTYDetection
+// RUN: %target-codesign %t/TTYDetection
+
+// RUN: (env SWIFT_BACKTRACE=enable=yes,cache=no,interactive=no %target-run script -ef -O %t/script-output %t/TTYDetection || true) && cat %t/script-output | %FileCheck %s
+
+
+// UNSUPPORTED: use_os_stdlib
+// UNSUPPORTED: back_deployment_runtime
+// UNSUPPORTED: asan
+// REQUIRES: executable_test
+// REQUIRES: backtracing
+// REQUIRES: OS=linux-gnu
+// COM: this won't work on Windows, but is not critical function
+
+func level1() {
+  level2()
+}
+
+func level2() {
+  level3()
+}
+
+func level3() {
+  level4()
+}
+
+func level4() {
+  level5()
+}
+
+func level5() {
+  print("About to crash")
+  let ptr = UnsafeMutablePointer<Int>(bitPattern: 4)!
+  ptr.pointee = 42
+}
+
+@main
+struct TTYDetection {
+  static func main() {
+    level1()
+  }
+}
+
+// CHECK: 💣{{.*}}Program crashed: Bad pointer dereference at 0x{{0+}}4
+// CHECK: Thread 0 {{(".*" )?}}crashed:

test/Backtracing/TTYDetectionColourMacOS.swift

@@ -0,0 +1,46 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift %s -parse-as-library -Onone -g -o %t/TTYDetection
+// RUN: %target-codesign %t/TTYDetection
+
+// RUN: (env SWIFT_BACKTRACE=enable=yes,cache=no,interactive=no %target-run script -reF %t/script-output %t/TTYDetection || true) && cat %t/script-output | %FileCheck %s
+
+
+// UNSUPPORTED: use_os_stdlib
+// UNSUPPORTED: back_deployment_runtime
+// UNSUPPORTED: asan
+// REQUIRES: executable_test
+// REQUIRES: backtracing
+// REQUIRES: OS=macosx
+// COM: this won't work on Windows, but is not critical function
+
+func level1() {
+  level2()
+}
+
+func level2() {
+  level3()
+}
+
+func level3() {
+  level4()
+}
+
+func level4() {
+  level5()
+}
+
+func level5() {
+  print("About to crash")
+  let ptr = UnsafeMutablePointer<Int>(bitPattern: 4)!
+  ptr.pointee = 42
+}
+
+@main
+struct TTYDetection {
+  static func main() {
+    level1()
+  }
+}
+
+// CHECK: 💣{{.*}}Program crashed: Bad pointer dereference at 0x{{0+}}4
+// CHECK: Thread 0 {{(".*" )?}}crashed: