add initial macro expansion protocol (#2021)

This adds a `MacroExecutor` interface which is intended for use by language frontends (cfe, analyzer) in order to load and execute macros, as well as create library augmention files.

This should help more clearly define the role of a frontend when it comes to macros (discovery and introspection), as well as provide a unified api for producing the final library augmentations.

It does not actually write files though, we still need to figure out how we want to coordinate that part, and make sure that the CFE and analyzer can consistently talk about these files.

This also allows for multiple different implementations in terms of how macros are actually loaded and executed, without the frontends having to know about them.

Followup PR here #2022 which starts an implementation using `IsolateMirror.loadUri`.

Author: jakemac53

.gitignore

@@ -22,6 +22,8 @@
 *.vcxproj.filters
 /*.vcxproj.user
 *.stamp
+.dart_tool
+.packages
 
 # Gyp generated files
 *.xcodeproj

working/macros/api/builders.dart

@@ -6,14 +6,7 @@ import 'macros.dart'; // For dart docs.
 
 /// The base interface used to add declarations to the program as well
 /// as augment existing ones.
-abstract class Builder {
-  /// Used to construct a [TypeAnnotation] to a runtime type available to the
-  /// the macro implementation code.
-  ///
-  /// This can be used to emit a reference to it in generated code, or do
-  /// subtype checks (depending on support in the current phase).
-  TypeAnnotation typeAnnotationOf<T>();
-}
+abstract class Builder {}
 
 /// The api used by [Macro]s to contribute new type declarations to the
 /// current library, and get [TypeAnnotation]s from runtime [Type] objects.
@@ -22,30 +15,30 @@ abstract class TypeBuilder implements Builder {
   void declareType(DeclarationCode typeDeclaration);
 }
 
-/// The api used by [Macro]s to contribute new (non-type)
-/// declarations to the current library.
+/// The interface to resolve a [TypeAnnotation] to a [StaticType].
 ///
-/// Can also be used to do subtype checks on types.
-abstract class DeclarationBuilder implements Builder {
-  /// Adds a new regular declaration to the surrounding library.
+/// The [StaticType]s can be compared against other [StaticType]s to see how
+/// they relate to each other.
+///
+/// This api is only available to the declaration and definition phases of
+/// macro expansion.
+abstract class TypeResolver {
+  /// Resolves [typeAnnotation] to a [StaticType].
   ///
-  /// Note that type declarations are not supported.
-  Declaration declareInLibrary(DeclarationCode declaration);
-
-  /// Returns true if [leftType] is a subtype of [rightType].
-  bool isSubtypeOf(TypeAnnotation leftType, TypeAnnotation rightType);
-
-  /// Retruns true if [leftType] is an identical type to [rightType].
-  bool isExactly(TypeAnnotation leftType, TypeAnnotation rightType);
-}
-
-/// The api used by [Macro]s to contribute new members to a class.
-abstract class ClassMemberDeclarationBuilder implements DeclarationBuilder {
-  /// Adds a new declaration to the surrounding class.
-  void declareInClass(DeclarationCode declaration);
+  /// Throws an error if the type annotation cannot be resolved. This should
+  /// only happen in the case of incomplete or invalid programs, but macros
+  /// may be asked to run in this state during the development cycle. It is
+  /// helpful for users if macros provide a best effort implementation in that
+  /// case or handle the error in a useful way.
+  Future<StaticType> resolve(TypeAnnotation typeAnnotation);
 }
 
 /// The api used to introspect on a [ClassDeclaration].
+///
+/// Available in the declaration and definition phases, but limited in the
+/// declaration phase to immediately annotated [ClassDeclaration]s. This is
+/// done by limiting the access to the [TypeDeclarationResolver] to the
+/// definition phase.
 abstract class ClassIntrospector {
   /// The fields available for [clazz].
   ///
@@ -75,23 +68,47 @@ abstract class ClassIntrospector {
   Future<List<ClassDeclaration>> interfacesOf(ClassDeclaration clazz);
 }
 
+/// The api used by [Macro]s to contribute new (non-type)
+/// declarations to the current library.
+///
+/// Can also be used to do subtype checks on types.
+abstract class DeclarationBuilder
+    implements Builder, TypeResolver, ClassIntrospector {
+  /// Adds a new regular declaration to the surrounding library.
+  ///
+  /// Note that type declarations are not supported.
+  void declareInLibrary(DeclarationCode declaration);
+}
+
+/// The api used by [Macro]s to contribute new members to a class.
+abstract class ClassMemberDeclarationBuilder implements DeclarationBuilder {
+  /// Adds a new declaration to the surrounding class.
+  void declareInClass(DeclarationCode declaration);
+}
+
 /// The api used by [Macro]s to reflect on the currently available
 /// members, superclass, and mixins for a given [ClassDeclaration]
 abstract class ClassDeclarationBuilder
     implements ClassMemberDeclarationBuilder, ClassIntrospector {}
 
-/// The api used by [Macro] to get a [TypeDeclaration] for any given
-/// [TypeAnnotation].
-abstract class TypeIntrospector {
-  /// Resolves a [NamedTypeAnnotation] to its declaration.
-  Future<TypeDeclaration> resolve(NamedTypeAnnotation annotation);
+/// The interface used by [Macro]s to resolve any [NamedStaticType] to its
+/// declaration.
+///
+/// Only available in the definition phase of macro expansion.
+abstract class TypeDeclarationResolver {
+  /// Resolves a [NamedStaticType] to its [TypeDeclaration].
+  Future<TypeDeclaration> declarationOf(NamedStaticType annotation);
 }
 
 /// The base class for builders in the definition phase. These can convert
 /// any [TypeAnnotation] into its corresponding [TypeDeclaration], and also
 /// reflect more deeply on those.
 abstract class DefinitionBuilder
-    implements Builder, ClassIntrospector, TypeIntrospector {}
+    implements
+        Builder,
+        TypeResolver,
+        ClassIntrospector,
+        TypeDeclarationResolver {}
 
 /// The apis used by [Macro]s that run on classes, to fill in the definitions
 /// of any external declarations within that class.

working/macros/api/expansion_protocol.dart

@@ -0,0 +1,109 @@
+import 'builders.dart';
+import 'code.dart';
+import 'introspection.dart';
+
+/// The interface used by Dart language implementations, in order to load
+/// and execute macros, as well as produce library augmentations from those
+/// macro applications.
+///
+/// This class more clearly defines the role of a Dart language implementation
+/// during macro discovery and expansion, and unifies how augmentation libraries
+/// are produced.
+abstract class MacroExecutor {
+  /// Invoked when an implementation discovers a new macro definition in a
+  /// [library] with [name], and prepares this executor to run the macro.
+  ///
+  /// May be invoked more than once for the same macro, which will cause the
+  /// macro to be re-loaded. Previous [MacroClassIdentifier]s and
+  /// [MacroInstanceIdentifier]s given for this macro will be invalid after
+  /// that point and should be discarded.
+  ///
+  /// Throws an exception if the macro fails to load.
+  Future<MacroClassIdentifier> loadMacro(Uri library, String name);
+
+  /// Creates an instance of [macroClass] in the executor, and returns an
+  /// identifier for that instance.
+  ///
+  /// Throws an exception if an instance is not created.
+  Future<MacroInstanceIdentifier> instantiateMacro(
+      MacroClassIdentifier macroClass, String constructor, Arguments arguments);
+
+  /// Runs the type phase for [macro] on a given [declaration].
+  ///
+  /// Throws an exception if there is an error executing the macro.
+  Future<MacroExecutionResult> executeTypesPhase(
+      MacroInstanceIdentifier macro, Declaration declaration);
+
+  /// Runs the declarations phase for [macro] on a given [declaration].
+  ///
+  /// Throws an exception if there is an error executing the macro.
+  Future<MacroExecutionResult> executeDeclarationsPhase(
+      MacroInstanceIdentifier macro,
+      Declaration declaration,
+      TypeResolver typeResolver,
+      ClassIntrospector classIntrospector);
+
+  /// Runs the definitions phase for [macro] on a given [declaration].
+  ///
+  /// Throws an exception if there is an error executing the macro.
+  Future<MacroExecutionResult> executeDefinitionsPhase(
+      MacroInstanceIdentifier macro,
+      Declaration declaration,
+      TypeResolver typeResolver,
+      ClassIntrospector classIntrospector,
+      TypeDeclarationResolver typeDeclarationResolver);
+
+  /// Combines multiple [MacroExecutionResult]s into a single library
+  /// augmentation file, and returns a [String] representing that file.
+  Future<String> buildAugmentationLibrary(
+      Iterable<MacroExecutionResult> macroResults);
+}
+
+/// The arguments passed to a macro constructor.
+///
+/// All argument instances must be of type [Code] or a built-in value type that
+/// is serializable (num, bool, String, null, etc).
+class Arguments {
+  final List<Object?> positional;
+
+  final Map<String, Object?> named;
+
+  Arguments(this.positional, this.named);
+}
+
+/// An opaque identifier for a macro class, retrieved by
+/// [MacroExecutor.loadMacro].
+///
+/// Used to execute or reload this macro in the future.
+abstract class MacroClassIdentifier {}
+
+/// An opaque identifier for an instance of a macro class, retrieved by
+/// [MacroExecutor.instantiateMacro].
+///
+/// Used to execute or reload this macro in the future.
+abstract class MacroInstanceIdentifier {}
+
+/// A summary of the results of running a macro in a given phase.
+///
+/// All modifications are expressed in terms of library augmentation
+/// declarations.
+abstract class MacroExecutionResult {
+  /// Any library imports that should be added to support the code used in
+  /// the augmentations.
+  Iterable<DeclarationCode> get imports;
+
+  /// Any augmentations that should be applied as a result of executing a macro.
+  Iterable<DeclarationCode> get augmentations;
+}
+
+/// Each of the different macro execution phases.
+enum Phase {
+  /// Only new types are added in this phase.
+  types,
+
+  /// New non-type declarations are added in this phase.
+  declarations,
+
+  /// This phase allows augmenting existing declarations.
+  defintions,
+}

working/macros/api/introspection.dart

@@ -39,6 +39,24 @@ abstract class NamedTypeAnnotation implements TypeAnnotation {
   Iterable<TypeAnnotation> get typeArguments;
 }
 
+/// The interface representing a resolved type.
+///
+/// Resolved types understand exactly what type they represent, and can be
+/// compared to other static types.
+abstract class StaticType {
+  /// Returns true if this is a subtype of [other].
+  Future<bool> isSubtypeOf(StaticType other);
+
+  /// Returns true if this is an identical type to [other].
+  Future<bool> isExactly(StaticType other);
+}
+
+/// A subtype of [StaticType] representing types that can be resolved by name
+/// to a concrete declaration.
+abstract class NamedStaticType implements StaticType {
+  String get name;
+}
+
 /// The base class for all declarations.
 abstract class Declaration {
   /// The name of this declaration.
@@ -50,8 +68,16 @@ abstract class TypeDeclaration implements Declaration {
   /// The type parameters defined for this type declaration.
   Iterable<TypeParameterDeclaration> get typeParameters;
 
-  /// Create a type annotation representing this type with [typeArguments].
-  TypeAnnotation instantiate({List<TypeAnnotation> typeArguments});
+  /// Create a static type representing this type with [typeArguments].
+  ///
+  /// If [isNullable] is `true`, then this type will behave as if it has a
+  /// trailing `?`.
+  ///
+  /// Throws an exception if the type could not be instantiated, typically due
+  /// to one of the type arguments not matching the bounds of the corresponding
+  /// type parameter.
+  Future<StaticType> instantiate(
+      {required List<StaticType> typeArguments, required bool isNullable});
 }
 
 /// Class (and enum) introspection information.

working/macros/pubspec.yaml

@@ -0,0 +1,4 @@
+name: macro_proposal
+publish_to: none
+environment:
+  sdk: ">=2.12.0 <3.0.0"