Data assets (flutter#169273)

Refiling of flutter#164094, which itself is a rebase of flutter#159675

This PR adds bundling support for the experimental dart data asset
feature: Dart packages with hooks can now emit data assets which the
flutter tool will bundle.

It relies on flutter's existing asset bundling mechanism (e.g. entries
in AssetManifest.json, DevFS syncing in reload/restart, ...).

The support is added under an experimental flag (similar to the existing
native assets experimental flag).

Also, kNativeAssets is removed to also bundle data assets on flutter
build bundle.

The chrome sandbox is disabled as per flutter#165664.

## Pre-launch Checklist

- [x] I read the [Contributor Guide] and followed the process outlined
there for submitting PRs.
- [x] I read the [Tree Hygiene] wiki page, which explains my
responsibilities.
- [x] I read and followed the [Flutter Style Guide], including [Features
we expect every widget to implement].
- [x] I signed the [CLA].
- [x] I listed at least one issue that this PR fixes in the description
above.
- [x] I updated/added relevant documentation (doc comments with `///`).
- [x] I added new tests to check the change I am making, or this PR is
[test-exempt].
- [x] I followed the [breaking change policy] and added [Data Driven
Fixes] where supported.
- [x] All existing and new tests are passing.

If you need help, consider asking for advice on the #hackers-new channel
on [Discord].

<!-- Links -->
[Contributor Guide]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview
[Tree Hygiene]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md
[test-exempt]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests
[Flutter Style Guide]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md
[Features we expect every widget to implement]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement
[CLA]: https://cla.developers.google.com/
[flutter/tests]: https://github.com/flutter/tests
[breaking change policy]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes
[Discord]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md
[Data Driven Fixes]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md

Author: mosuem

packages/flutter_tools/lib/executable.dart

@@ -11,6 +11,7 @@ import 'src/base/template.dart';
 import 'src/base/terminal.dart';
 import 'src/base/user_messages.dart';
 import 'src/build_system/build_targets.dart';
+import 'src/build_system/targets/hook_runner_native.dart' show FlutterHookRunnerNative;
 import 'src/cache.dart';
 import 'src/commands/analyze.dart';
 import 'src/commands/assemble.dart';
@@ -48,6 +49,7 @@ import 'src/devtools_launcher.dart';
 import 'src/features.dart';
 import 'src/globals.dart' as globals;
 // Files in `isolated` are intentionally excluded from google3 tooling.
+import 'src/hook_runner.dart' show FlutterHookRunner;
 import 'src/isolated/build_targets.dart';
 import 'src/isolated/mustache_template.dart';
 import 'src/isolated/native_assets/test/native_assets.dart';
@@ -117,6 +119,7 @@ Future<void> main(List<String> args) async {
     muteCommandLogging: muteCommandLogging,
     verboseHelp: verboseHelp,
     overrides: <Type, Generator>{
+      FlutterHookRunner: () => FlutterHookRunnerNative(),
       // The web runner is not supported in google3 because it depends
       // on dwds.
       WebRunnerFactory: () => DwdsWebRunnerFactory(),

packages/flutter_tools/lib/src/asset.dart

@@ -25,6 +25,84 @@ import 'flutter_manifest.dart';
 import 'license_collector.dart';
 import 'project.dart';
 
+class FlutterHookResult {
+  const FlutterHookResult({
+    required this.buildStart,
+    required this.buildEnd,
+    required this.dataAssets,
+    required this.dependencies,
+  });
+
+  FlutterHookResult.empty()
+    : this(
+        buildStart: DateTime.fromMillisecondsSinceEpoch(0),
+        buildEnd: DateTime.fromMillisecondsSinceEpoch(0),
+        dataAssets: <HookAsset>[],
+        dependencies: <Uri>[],
+      );
+
+  final List<HookAsset> dataAssets;
+
+  /// The timestamp at which we start a build - so the timestamp of the inputs.
+  final DateTime buildStart;
+
+  /// The timestamp at which we finish a build - so the timestamp of the
+  /// outputs.
+  final DateTime buildEnd;
+
+  /// The dependencies of the build are used to check if the build needs to be
+  /// rerun.
+  final List<Uri> dependencies;
+
+  /// Whether caller may need to re-run the Dart build.
+  bool hasAnyModifiedFiles(FileSystem fileSystem) =>
+      _wasAnyFileModifiedSince(fileSystem, buildStart, dependencies);
+
+  /// Whether the files produced by the build are up-to-date.
+  ///
+  /// NOTICE: The build itself may be up-to-date but the output may not be (as
+  /// the output may be existing on disk and not be produced by the build
+  /// itself - in which case we may not need to re-build if the file changes,
+  /// but we may need to make a new asset bundle with the modified file).
+  bool isOutputDirty(FileSystem fileSystem) => _wasAnyFileModifiedSince(
+    fileSystem,
+    buildEnd,
+    dataAssets.map((HookAsset e) => e.file).toList(),
+  );
+
+  static bool _wasAnyFileModifiedSince(FileSystem fileSystem, DateTime since, List<Uri> uris) {
+    for (final Uri uri in uris) {
+      final DateTime modified = fileSystem.statSync(uri.toFilePath()).modified;
+      if (modified.isAfter(since)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @override
+  String toString() {
+    return dataAssets.toString();
+  }
+}
+
+/// A convenience class to wrap native assets
+///
+/// When translating from a `DartHooksResult` to a [FlutterHookResult], where we
+/// need to have different classes to not import `isolated/` stuff.
+class HookAsset {
+  const HookAsset({required this.file, required this.name, required this.package});
+
+  final Uri file;
+  final String name;
+  final String package;
+
+  @override
+  String toString() {
+    return 'HookAsset(file: $file, name: $name, package: $package)';
+  }
+}
+
 const String defaultManifestPath = 'pubspec.yaml';
 
 const String kFontManifestJson = 'FontManifest.json';
@@ -113,6 +191,7 @@ abstract class AssetBundle {
 
   /// Returns 0 for success; non-zero for failure.
   Future<int> build({
+    FlutterHookResult? flutterHookResult,
     String manifestPath = defaultManifestPath,
     required String packageConfigPath,
     bool deferredComponentsEnabled = false,
@@ -163,7 +242,8 @@ class ManifestAssetBundle implements AssetBundle {
        _platform = platform,
        _flutterRoot = flutterRoot,
        _splitDeferredAssets = splitDeferredAssets,
-       _licenseCollector = LicenseCollector(fileSystem: fileSystem);
+       _licenseCollector = LicenseCollector(fileSystem: fileSystem),
+       _lastHookResult = FlutterHookResult.empty();
 
   final Logger _logger;
   final FileSystem _fileSystem;
@@ -189,6 +269,8 @@ class ManifestAssetBundle implements AssetBundle {
 
   DateTime? _lastBuildTimestamp;
 
+  FlutterHookResult _lastHookResult;
+
   // We assume the main asset is designed for a device pixel ratio of 1.0.
   static const String _kAssetManifestJsonFilename = 'AssetManifest.json';
   static const String _kAssetManifestBinFilename = 'AssetManifest.bin';
@@ -208,13 +290,19 @@ class ManifestAssetBundle implements AssetBundle {
 
   @override
   bool needsBuild({String manifestPath = defaultManifestPath}) {
-    final DateTime? lastBuildTimestamp = _lastBuildTimestamp;
-    if (lastBuildTimestamp == null) {
+    if (!wasBuiltOnce() ||
+        // We need to re-run the Dart build.
+        _lastHookResult.hasAnyModifiedFiles(_fileSystem) ||
+        // We don't have to re-run the Dart build, but some files the Dart build
+        // wants us to bundle have changed contents.
+        _lastHookResult.isOutputDirty(_fileSystem)) {
       return true;
     }
+    final DateTime lastBuildTimestamp = _lastBuildTimestamp!;
 
     final FileStat manifestStat = _fileSystem.file(manifestPath).statSync();
-    if (manifestStat.type == FileSystemEntityType.notFound) {
+    if (manifestStat.type == FileSystemEntityType.notFound ||
+        manifestStat.modified.isAfter(lastBuildTimestamp)) {
       return true;
     }
 
@@ -223,18 +311,19 @@ class ManifestAssetBundle implements AssetBundle {
         return true; // directory was deleted.
       }
       for (final File file in directory.listSync().whereType<File>()) {
-        final DateTime dateTime = file.statSync().modified;
-        if (dateTime.isAfter(lastBuildTimestamp)) {
+        final DateTime lastModified = file.statSync().modified;
+        if (lastModified.isAfter(lastBuildTimestamp)) {
           return true;
         }
       }
     }
 
-    return manifestStat.modified.isAfter(lastBuildTimestamp);
+    return false;
   }
 
   @override
   Future<int> build({
+    FlutterHookResult? flutterHookResult,
     String manifestPath = defaultManifestPath,
     FlutterProject? flutterProject,
     required String packageConfigPath,
@@ -258,6 +347,7 @@ class ManifestAssetBundle implements AssetBundle {
     // hang on hot reload, as the incremental dill files will never be copied to the
     // device.
     _lastBuildTimestamp = DateTime.now();
+    _lastHookResult = flutterHookResult ?? FlutterHookResult.empty();
     if (flutterManifest.isEmpty) {
       entries[_kAssetManifestJsonFilename] = AssetBundleEntry(
         DevFSStringContent('{}'),
@@ -425,9 +515,38 @@ class ManifestAssetBundle implements AssetBundle {
         );
       }
     }
+    for (final HookAsset dataAsset in flutterHookResult?.dataAssets ?? <HookAsset>[]) {
+      final Package package = packageConfig[dataAsset.package]!;
+      final Uri fileUri = dataAsset.file;
+
+      final String baseDir;
+      final Uri relativeUri;
+      if (fileUri.isAbsolute) {
+        final String filePath = fileUri.toFilePath();
+        baseDir = _fileSystem.path.dirname(filePath);
+        relativeUri = Uri(path: _fileSystem.path.basename(filePath));
+      } else {
+        baseDir = package.root.toFilePath();
+        relativeUri = fileUri;
+      }
+
+      final _Asset asset = _Asset(
+        baseDir: baseDir,
+        relativeUri: relativeUri,
+        entryUri: Uri.parse(_fileSystem.path.join('packages', dataAsset.package, dataAsset.name)),
+        package: package,
+      );
+      if (assetVariants.containsKey(asset)) {
+        _logger.printError(
+          'Conflicting assets: The asset "$asset" was declared in the pubspec and the hook.',
+        );
+        return 1;
+      }
+      assetVariants[asset] = <_Asset>[asset];
+    }
 
     // Save the contents of each image, image variant, and font
-    // asset in entries.
+    // asset in [entries].
     for (final _Asset asset in assetVariants.keys) {
       final File assetFile = asset.lookupAssetFile(_fileSystem);
       final List<_Asset> variants = assetVariants[asset]!;

packages/flutter_tools/lib/src/build_info.dart

@@ -590,6 +590,33 @@ enum TargetPlatform {
     }
   }
 
+  String get osName {
+    switch (this) {
+      case TargetPlatform.linux_x64:
+      case TargetPlatform.linux_arm64:
+        return 'linux';
+      case TargetPlatform.darwin:
+        return 'macos';
+      case TargetPlatform.windows_x64:
+      case TargetPlatform.windows_arm64:
+        return 'windows';
+      case TargetPlatform.android:
+      case TargetPlatform.android_arm:
+      case TargetPlatform.android_arm64:
+      case TargetPlatform.android_x64:
+        return 'android';
+      case TargetPlatform.fuchsia_arm64:
+      case TargetPlatform.fuchsia_x64:
+        return 'fuchsia';
+      case TargetPlatform.ios:
+        return 'ios';
+      case TargetPlatform.tester:
+        return 'flutter-tester';
+      case TargetPlatform.web_javascript:
+        return 'web';
+    }
+  }
+
   String get simpleName {
     switch (this) {
       case TargetPlatform.linux_x64:
@@ -712,6 +739,15 @@ DarwinArch getDarwinArchForName(String arch) {
   };
 }
 
+List<DarwinArch> getDarwinArchsFromEnv(Map<String, String> defines) {
+  const List<DarwinArch> defaultDarwinArchitectures = <DarwinArch>[
+    DarwinArch.x86_64,
+    DarwinArch.arm64,
+  ];
+  return defines[kDarwinArchs]?.split(' ').map(getDarwinArchForName).toList() ??
+      defaultDarwinArchitectures;
+}
+
 String getNameForTargetPlatform(TargetPlatform platform, {DarwinArch? darwinArch}) {
   return switch (platform) {
     TargetPlatform.ios when darwinArch != null => 'ios-${darwinArch.name}',
@@ -948,20 +984,6 @@ const String kSdkRoot = 'SdkRoot';
 /// Whether to enable Dart obfuscation and where to save the symbol map.
 const String kDartObfuscation = 'DartObfuscation';
 
-/// Whether to enable Native Assets.
-///
-/// If true, native assets are built and the mapping for native assets lookup
-/// at runtime is embedded in the kernel file.
-///
-/// If false, native assets are not built, and an empty mapping is embedded in
-/// the kernel file. Used for targets that trigger kernel builds but
-/// are not OS/architecture specific.
-///
-/// Supported values are 'true' and 'false'.
-///
-/// Defaults to 'true'.
-const String kNativeAssets = 'NativeAssets';
-
 /// An output directory where one or more code-size measurements may be written.
 const String kCodeSizeDirectory = 'CodeSizeDirectory';
 

packages/flutter_tools/lib/src/build_system/targets/android.dart

@@ -9,6 +9,7 @@ import '../../base/file_system.dart';
 import '../../build_info.dart';
 import '../../devfs.dart';
 import '../../globals.dart' as globals show xcode;
+import '../../isolated/native_assets/dart_hook_result.dart';
 import '../../project.dart';
 import '../build_system.dart';
 import '../depfile.dart';
@@ -70,9 +71,11 @@ abstract class AndroidAssetBundle extends Target {
           .file(isolateSnapshotData)
           .copySync(outputDirectory.childFile('isolate_snapshot_data').path);
     }
+    final DartHookResult dartHookResult = await DartBuild.loadHookResult(environment);
     final Depfile assetDepfile = await copyAssets(
       environment,
       outputDirectory,
+      dartHookResult: dartHookResult,
       targetPlatform: TargetPlatform.android,
       buildMode: buildMode,
       flavor: environment.defines[kFlavor],
@@ -89,7 +92,11 @@ abstract class AndroidAssetBundle extends Target {
   }
 
   @override
-  List<Target> get dependencies => const <Target>[KernelSnapshot(), InstallCodeAssets()];
+  List<Target> get dependencies => const <Target>[
+    DartBuildForNative(),
+    KernelSnapshot(),
+    InstallCodeAssets(),
+  ];
 }
 
 /// An implementation of [AndroidAssetBundle] that includes dependencies on vm

packages/flutter_tools/lib/src/build_system/targets/assets.dart

@@ -12,6 +12,7 @@ import '../../build_info.dart';
 import '../../dart/package_map.dart';
 import '../../devfs.dart';
 import '../../flutter_manifest.dart';
+import '../../isolated/native_assets/dart_hook_result.dart';
 import '../build_system.dart';
 import '../depfile.dart';
 import '../exceptions.dart';
@@ -33,6 +34,7 @@ import 'native_assets.dart';
 Future<Depfile> copyAssets(
   Environment environment,
   Directory outputDirectory, {
+  required DartHookResult dartHookResult,
   Map<String, DevFSContent> additionalContent = const <String, DevFSContent>{},
   required TargetPlatform targetPlatform,
   required BuildMode buildMode,
@@ -49,6 +51,7 @@ Future<Depfile> copyAssets(
         splitDeferredAssets: buildMode != BuildMode.debug && buildMode != BuildMode.jitRelease,
       ).createBundle();
   final int resultCode = await assetBundle.build(
+    flutterHookResult: dartHookResult.asFlutterResult,
     manifestPath: pubspecFile.path,
     packageConfigPath: findPackageConfigFileOrDefault(environment.projectDir).path,
     deferredComponentsEnabled: environment.defines[kDeferredComponents] == 'true',
@@ -248,7 +251,11 @@ class CopyAssets extends Target {
   String get name => 'copy_assets';
 
   @override
-  List<Target> get dependencies => const <Target>[KernelSnapshot(), InstallCodeAssets()];
+  List<Target> get dependencies => const <Target>[
+    DartBuildForNative(),
+    KernelSnapshot(),
+    InstallCodeAssets(),
+  ];
 
   @override
   List<Source> get inputs => const <Source>[
@@ -274,9 +281,11 @@ class CopyAssets extends Target {
     final BuildMode buildMode = BuildMode.fromCliName(buildModeEnvironment);
     final Directory output = environment.buildDir.childDirectory('flutter_assets');
     output.createSync(recursive: true);
+    final DartHookResult dartHookResult = await DartBuild.loadHookResult(environment);
     final Depfile depfile = await copyAssets(
       environment,
       output,
+      dartHookResult: dartHookResult,
       targetPlatform: TargetPlatform.android,
       buildMode: buildMode,
       flavor: environment.defines[kFlavor],