diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index ce0028854..01ca81f0b 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -5,6 +5,11 @@ * Require a function definition named `main` directly in a test suite and provide a more direct error message than a failing compiler output. * Suppress skip reason messages in the compact and failures-only reporters. +* Add `vm-asan`, `vm-msan`, and `vm-tsan` runtimes to run tests on the standalone + Dart VM under Address Sanitizer, Memory Sanitizer or Thread Sanitizer. This is + useful for finding issues when using foreign libraries through dart:ffi, such + as use-after-free, use of initialized memory and data races, or for detecting + data races in Dart code using shared fields. ## 1.28.0 diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml index 1f8d8db60..c024ee2c7 100644 --- a/pkgs/test/pubspec.yaml +++ b/pkgs/test/pubspec.yaml @@ -35,7 +35,7 @@ dependencies: stream_channel: ^2.1.0 # Use an exact version until the test_api and test_core package are stable. - test_api: 0.7.8 + test_api: 0.7.9-wip test_core: 0.6.15-wip typed_data: ^1.3.0 diff --git a/pkgs/test/test/io.dart b/pkgs/test/test/io.dart index 8ad784e08..886843c40 100644 --- a/pkgs/test/test/io.dart +++ b/pkgs/test/test/io.dart @@ -21,6 +21,9 @@ final Future packageDir = Isolate.resolvePackageUri( return dir; }); +/// The root directory of the Dart SDK. +final String sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); + /// The platform-specific message emitted when a nonexistent file is loaded. final String noSuchFileMessage = Platform.isWindows diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index 312ed428f..592070139 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -23,7 +23,10 @@ void main() { for (var compiler in runtime.supportedCompilers) { // Ignore the platforms we can't run on this OS. if ((runtime == Runtime.edge && !Platform.isWindows) || - (runtime == Runtime.safari && !Platform.isMacOS)) { + (runtime == Runtime.safari && !Platform.isMacOS) || + (runtime == Runtime.vmAsan && !Platform.isLinux) || + (runtime == Runtime.vmMsan && !Platform.isLinux) || + (runtime == Runtime.vmTsan && !Platform.isLinux)) { continue; } String? skipReason; @@ -36,6 +39,15 @@ void main() { skipReason = 'https://github.com/dart-lang/test/issues/1942'; } else if (runtime == Runtime.firefox && Platform.isMacOS) { skipReason = 'https://github.com/dart-lang/test/pull/2276'; + } else if (runtime == Runtime.vmAsan && + !File('$sdkDir/bin/dartaotruntime_asan').existsSync()) { + skipReason = 'SDK too old'; + } else if (runtime == Runtime.vmMsan && + !File('$sdkDir/bin/dartaotruntime_msan').existsSync()) { + skipReason = 'SDK too old'; + } else if (runtime == Runtime.vmTsan && + !File('$sdkDir/bin/dartaotruntime_tsan').existsSync()) { + skipReason = 'SDK too old'; } group( '--runtime ${runtime.identifier} --compiler ${compiler.identifier}', diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index 2096f1a3c..c2e3a8896 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -129,12 +129,15 @@ Output: '''; final _runtimes = - '[vm (default), chrome, firefox' + '[vm (default), vm-asan, vm-msan, vm-tsan, chrome, firefox' '${Platform.isMacOS ? ', safari' : ''}' ', edge, node]'; final _runtimeCompilers = [ '[vm]: kernel (default), source, exe', + '[vm-asan]: exe (default)', + '[vm-msan]: exe (default)', + '[vm-tsan]: exe (default)', '[chrome]: dart2js (default), dart2wasm', '[firefox]: dart2js (default), dart2wasm', if (Platform.isMacOS) '[safari]: dart2js (default)', diff --git a/pkgs/test/test/runner/sanitizer_test.dart b/pkgs/test/test/runner/sanitizer_test.dart new file mode 100644 index 000000000..e4f228ff7 --- /dev/null +++ b/pkgs/test/test/runner/sanitizer_test.dart @@ -0,0 +1,239 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn('vm && linux') +library; + +import 'dart:io'; +import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; + +import '../io.dart'; + +void main() { + setUpAll(precompileTestExecutable); + + String? skipReason; + if (!File('$sdkDir/bin/dartaotruntime_asan').existsSync()) { + skipReason = 'SDK too old'; + } + + test('asan success', () async { + final testSource = ''' +@TestOn('vm-asan') +library asan_environment_test; + +import 'package:test/test.dart'; + +void main() { + test('const', () { + // I.e., correct during kernel compilation. + expect(const bool.fromEnvironment("dart.vm.asan"), equals(true)); + + expect(const bool.fromEnvironment("dart.vm.msan"), equals(false)); + expect(const bool.fromEnvironment("dart.vm.tsan"), equals(false)); + }); + + test('new', () { + // I.e., correct during VM lookup. + expect(new bool.fromEnvironment("dart.vm.asan"), equals(true)); + + expect(new bool.fromEnvironment("dart.vm.msan"), equals(false)); + expect(new bool.fromEnvironment("dart.vm.tsan"), equals(false)); + }); +} +'''; + + await d.file('test.dart', testSource).create(); + var test = await runTest(['test.dart', '-p', 'vm-asan']); + + expect(test.stdout, emitsThrough(contains('+2: All tests passed!'))); + await test.shouldExit(0); + }, skip: skipReason); + + test('asan failure', () async { + final testSource = ''' +@TestOn('vm-asan') +library asan_test; + +import 'package:test/test.dart'; +import 'dart:ffi'; + +@Native(symbol: 'malloc') +external Pointer malloc(int size); +@Native(symbol: 'free') +external void free(Pointer ptr); +@Native(symbol: 'memset') +external void memset(Pointer ptr, int char, int size); + +void main() { + test('use-after-free', () { + var p = malloc(sizeOf()).cast(); + free(p); + memset(p, 42, sizeOf()); // ASAN: heap-use-after-free + }); +} +'''; + + await d.file('test.dart', testSource).create(); + var test = await runTest(['test.dart', '-p', 'vm-asan']); + + expect( + test.stderr, + emitsThrough(contains('AddressSanitizer: heap-use-after-free')), + ); + await test.shouldExit(6); + }, skip: skipReason); + + test('msan success', () async { + final testSource = ''' +@TestOn('vm-msan') +library msan_environment_test; + +import 'package:test/test.dart'; + +void main() { + test('const', () { + // I.e., correct during kernel compilation. + expect(const bool.fromEnvironment("dart.vm.msan"), equals(true)); + + expect(const bool.fromEnvironment("dart.vm.asan"), equals(false)); + expect(const bool.fromEnvironment("dart.vm.tsan"), equals(false)); + }); + + test('new', () { + // I.e., correct during VM lookup. + expect(new bool.fromEnvironment("dart.vm.msan"), equals(true)); + + expect(new bool.fromEnvironment("dart.vm.asan"), equals(false)); + expect(new bool.fromEnvironment("dart.vm.tsan"), equals(false)); + }); +} +'''; + + await d.file('test.dart', testSource).create(); + var test = await runTest(['test.dart', '-p', 'vm-msan']); + + expect(test.stdout, emitsThrough(contains('+2: All tests passed!'))); + await test.shouldExit(0); + }, skip: skipReason); + + test('msan failure', () async { + final testSource = ''' +@TestOn('vm-msan') +library msan_test; + +import 'dart:ffi'; +import 'package:test/test.dart'; + +@Native(symbol: 'malloc') +external Pointer malloc(int size); +@Native(symbol: 'free') +external void free(Pointer ptr); +@Native(symbol: 'memcmp') +external void memcmp(Pointer a, Pointer b, int size); + +void main() { + test('uninitialized', () { + var a = malloc(8); + var b = malloc(8); + memcmp(a, b, 8); // MSAN: use-of-uninitialized-value + free(b); + free(a); + }); +} +'''; + + await d.file('test.dart', testSource).create(); + var test = await runTest(['test.dart', '-p', 'vm-msan']); + + expect( + test.stderr, + emitsThrough(contains('MemorySanitizer: use-of-uninitialized-value')), + ); + await test.shouldExit(6); + }, skip: skipReason); + + test('tsan success', () async { + final testSource = ''' +@TestOn('vm-tsan') +library tsan_environment_test; + +import 'package:test/test.dart'; + +void main() { + test('const', () { + // I.e., correct during kernel compilation. + expect(const bool.fromEnvironment("dart.vm.tsan"), equals(true)); + + expect(const bool.fromEnvironment("dart.vm.asan"), equals(false)); + expect(const bool.fromEnvironment("dart.vm.msan"), equals(false)); + }); + + test('new', () { + // I.e., correct during VM lookup. + expect(new bool.fromEnvironment("dart.vm.tsan"), equals(true)); + + expect(new bool.fromEnvironment("dart.vm.asan"), equals(false)); + expect(new bool.fromEnvironment("dart.vm.msan"), equals(false)); + }); +} +'''; + + await d.file('test.dart', testSource).create(); + var test = await runTest(['test.dart', '-p', 'vm-tsan']); + + expect(test.stdout, emitsThrough(contains('+2: All tests passed!'))); + await test.shouldExit(0); + }, skip: skipReason); + + test('tsan failure', () async { + final testSource = ''' +@TestOn('vm-tsan') +library tsan_test; + +import 'dart:ffi'; +import 'dart:isolate'; +import 'package:test/test.dart'; + +@Native(symbol: 'malloc') +external Pointer malloc(int size); +@Native(symbol: 'free') +external void free(Pointer ptr); +@Native(symbol: 'memset', isLeaf: true) +external void memset_leaf(Pointer ptr, int char, int size); +@Native(symbol: 'usleep', isLeaf: true) +external void usleep_leaf(int useconds); + +child(addr) { + var p = Pointer.fromAddress(addr); + for (var i = 0; i < 50000; i++) { + memset_leaf(p, 42, sizeOf()); // TSAN: data race + usleep_leaf(100); + } +} + +void main() { + test('data race', () async { + var p = malloc(sizeOf()).cast(); + var f = Isolate.run(() => child(p.address)); + + for (var i = 0; i < 50000; i++) { + p[0] = p[0] + 1; // TSAN: data race + usleep_leaf(100); + } + + await f; + free(p); + }); +} +'''; + + await d.file('test.dart', testSource).create(); + var test = await runTest(['test.dart', '-p', 'vm-tsan']); + + expect(test.stderr, emitsThrough(contains('ThreadSanitizer: data race'))); + await test.shouldExit(6); + }, skip: skipReason); +} diff --git a/pkgs/test_api/CHANGELOG.md b/pkgs/test_api/CHANGELOG.md index d4728e30b..60d980697 100644 --- a/pkgs/test_api/CHANGELOG.md +++ b/pkgs/test_api/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.7.9-wip + +* Add `vmAsan`, `vmMsan` and `vmTsan` runtimes. + ## 0.7.8 * Add a zone function available from the test suite `main` that allows creating diff --git a/pkgs/test_api/lib/src/backend/runtime.dart b/pkgs/test_api/lib/src/backend/runtime.dart index c4db8fa79..0ce9cd765 100644 --- a/pkgs/test_api/lib/src/backend/runtime.dart +++ b/pkgs/test_api/lib/src/backend/runtime.dart @@ -15,6 +15,27 @@ final class Runtime { Compiler.source, Compiler.exe, ], isDartVM: true); + static const Runtime vmAsan = Runtime( + 'VM with Address Sanitizer', + 'vm-asan', + Compiler.exe, + [Compiler.exe], + isDartVM: true, + ); + static const Runtime vmMsan = Runtime( + 'VM with Memory Sanitizer', + 'vm-msan', + Compiler.exe, + [Compiler.exe], + isDartVM: true, + ); + static const Runtime vmTsan = Runtime( + 'VM with Thread Sanitizer', + 'vm-tsan', + Compiler.exe, + [Compiler.exe], + isDartVM: true, + ); /// Google Chrome. static const Runtime chrome = Runtime( @@ -69,6 +90,9 @@ final class Runtime { /// The platforms that are supported by the test runner by default. static const List builtIn = [ Runtime.vm, + Runtime.vmAsan, + Runtime.vmMsan, + Runtime.vmTsan, Runtime.chrome, Runtime.firefox, Runtime.safari, diff --git a/pkgs/test_api/pubspec.yaml b/pkgs/test_api/pubspec.yaml index e88d2ed95..b179a06e3 100644 --- a/pkgs/test_api/pubspec.yaml +++ b/pkgs/test_api/pubspec.yaml @@ -1,5 +1,5 @@ name: test_api -version: 0.7.8 +version: 0.7.9-wip description: >- The user facing API for structuring Dart tests and checking expectations. repository: https://github.com/dart-lang/test/tree/master/pkgs/test_api diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 61d8b60c1..107a64ef5 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -11,6 +11,11 @@ * Fix default coverage filter when running in a workspace package. Default filter now includes all the workspace's package. * Add support for reading test package version within pub workspaces. +* Add `vm-asan`, `vm-msan`, and `vm-tsan` runtimes to run tests on the standalone + Dart VM under Address Sanitizer, Memory Sanitizer or Thread Sanitizer. This is + useful for finding issues when using foreign libraries through dart:ffi, such + as use-after-free, use of initialized memory and data races, or for detecting + data races in Dart code using shared fields. ## 0.6.14 diff --git a/pkgs/test_core/lib/src/runner/loader.dart b/pkgs/test_core/lib/src/runner/loader.dart index 525ea79d8..ac238c549 100644 --- a/pkgs/test_core/lib/src/runner/loader.dart +++ b/pkgs/test_core/lib/src/runner/loader.dart @@ -65,7 +65,12 @@ class Loader { /// Creates a new loader that loads tests on platforms defined in /// [Configuration.current]. Loader() { - _registerPlatformPlugin([Runtime.vm], VMPlatform.new); + _registerPlatformPlugin([ + Runtime.vm, + if (File('$sdkDir/bin/dartaotruntime_asan').existsSync()) Runtime.vmAsan, + if (File('$sdkDir/bin/dartaotruntime_msan').existsSync()) Runtime.vmMsan, + if (File('$sdkDir/bin/dartaotruntime_tsan').existsSync()) Runtime.vmTsan, + ], VMPlatform.new); platformCallbacks.forEach((runtime, plugin) { _registerPlatformPlugin([runtime], plugin); diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 53e2aaea4..87670e245 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -62,6 +62,7 @@ class VMPlatform extends PlatformPlugin { Process process; try { process = await _spawnExecutable( + platform, path, suiteConfig.metadata, serverSocket, @@ -70,8 +71,15 @@ class VMPlatform extends PlatformPlugin { unawaited(serverSocket.close()); rethrow; } - process.stdout.listen(stdout.add); - process.stderr.listen(stderr.add); + + // If the child crashes, at least don't hang the parent. + // https://github.com/dart-lang/test/issues/2577 + process.exitCode.then((exitCode) async { + if (exitCode == 6) { + exit(exitCode); + } + }); + var socket = await serverSocket.first; outerChannel = MultiChannel(jsonSocketStreamChannel(socket)); cleanupCallbacks @@ -186,11 +194,57 @@ class VMPlatform extends PlatformPlugin { () => Future.wait([_compiler.dispose(), _tempDir.deleteWithRetry()]), ); + String _aotRuntimeFor(SuitePlatform platform) { + var sanSuffix = ''; + switch (platform.runtime) { + case Runtime.vmAsan: + sanSuffix = '_asan'; + break; + case Runtime.vmMsan: + sanSuffix = '_msan'; + break; + case Runtime.vmTsan: + sanSuffix = '_tsan'; + break; + } + final exeSuffix = Platform.isWindows ? '.exe' : ''; + return p.join( + p.dirname(Platform.resolvedExecutable), + 'dartaotruntime$sanSuffix$exeSuffix', + ); + } + + Map _environmentFor(SuitePlatform platform) { + // The sanitizers have inconsistent default options, so provide some + // better defaults if our caller hasn't provided any. SIGABRT=6 + final parentEnv = Platform.environment; + final env = {}; + switch (platform.runtime) { + case Runtime.vmAsan: + if (parentEnv['ASAN_OPTIONS'] == null) { + env['ASAN_OPTIONS'] = 'halt_on_error=1:exitcode=6:symbolize=1'; + } + break; + case Runtime.vmMsan: + if (parentEnv['MSAN_OPTIONS'] == null) { + env['MSAN_OPTIONS'] = 'halt_on_error=1:exitcode=6:symbolize=1'; + } + break; + case Runtime.vmTsan: + if (parentEnv['TSAN_OPTIONS'] == null) { + env['TSAN_OPTIONS'] = 'halt_on_error=1:exitcode=6:symbolize=1'; + } + break; + } + return env; + } + /// Compiles [path] to a native executable and spawns it as a process. /// /// Sets up a communication channel as well by passing command line arguments /// for the host and port of [socket]. Future _spawnExecutable( + SuitePlatform platform, String path, Metadata suiteMetadata, ServerSocket socket, @@ -200,29 +254,44 @@ class VMPlatform extends PlatformPlugin { 'Precompiled native executable tests are not supported at this time', ); } - var executable = await _compileToNative(path, suiteMetadata); - return await Process.start(executable, [ - socket.address.host, - socket.port.toString(), - ]); + + var sharedLibrary = await _compileToNative(platform, path, suiteMetadata); + return await Process.start( + _aotRuntimeFor(platform), + [sharedLibrary, socket.address.host, socket.port.toString()], + environment: _environmentFor(platform), + mode: ProcessStartMode.inheritStdio, + ); } - /// Compiles [path] to a native executable using `dart compile exe`. - Future _compileToNative(String path, Metadata suiteMetadata) async { + /// Compiles [path] to a native shared library using + /// `dart compile aot-snapshot`. + /// + /// Preferable to `dart compile exe` because embedded snapshots are invisible + /// to native profilers and symbolizers. This should eventually be replaced + /// with something that supports build hooks. + Future _compileToNative( + SuitePlatform platform, + String path, + Metadata suiteMetadata, + ) async { var bootstrapPath = await _bootstrapNativeTestFile( path, suiteMetadata.languageVersionComment ?? await rootPackageLanguageVersionComment, ); - var output = File(p.setExtension(bootstrapPath, '.exe')); + var output = File(p.setExtension(bootstrapPath, '.so')); var processResult = await Process.run(Platform.resolvedExecutable, [ 'compile', - 'exe', + 'aot-snapshot', bootstrapPath, '--output', output.path, '--packages', (await packageConfigUri).toFilePath(), + if (platform.runtime == Runtime.vmAsan) '--target-sanitizer=asan', + if (platform.runtime == Runtime.vmMsan) '--target-sanitizer=msan', + if (platform.runtime == Runtime.vmTsan) '--target-sanitizer=tsan', ]); if (processResult.exitCode != 0 || !(await output.exists())) { throw LoadException(path, ''' diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml index c45a920d6..165c5d4cc 100644 --- a/pkgs/test_core/pubspec.yaml +++ b/pkgs/test_core/pubspec.yaml @@ -29,7 +29,7 @@ dependencies: stack_trace: ^1.10.0 stream_channel: ^2.1.0 # Use an exact version until the test_api package is stable. - test_api: 0.7.8 + test_api: 0.7.9-wip vm_service: '>=6.0.0 <16.0.0' yaml: ^3.0.0