[CAS] Full support for make-style dependencies file caching hit

Fully support make-style `.d` dependencies file output by making
following improvements:
* All correct dependency file render when cache hit for a different
  output file location. The dependency file should list the correct
  output path, not the stale output path for the initial compilation
* When enable a path prefix mapper to canonicalize the path, the
  dependency file should render the input file correctly as the input
  file path on disk.

rdar://132250067

Author: cachemeifyoucan

include/swift/Frontend/CASOutputBackends.h

@@ -38,6 +38,9 @@ class SwiftCASOutputBackend : public llvm::vfs::OutputBackend {
 private:
   file_types::ID getOutputFileType(llvm::StringRef Path) const;
 
+  /// Return true if the file type is stored into CAS Backend directly.
+  static bool isStoredDirectly(file_types::ID Kind);
+
 public:
   SwiftCASOutputBackend(llvm::cas::ObjectStore &CAS,
                         llvm::cas::ActionCache &Cache,
@@ -49,6 +52,9 @@ class SwiftCASOutputBackend : public llvm::vfs::OutputBackend {
   llvm::Error storeCachedDiagnostics(unsigned InputIndex,
                                      llvm::StringRef Bytes);
 
+  llvm::Error storeMakeDependenciesFile(StringRef OutputFilename,
+                                        llvm::StringRef Bytes);
+
   /// Store the MCCAS CASID \p ID as the object file output for the input
   /// that corresponds to the \p OutputFilename
   llvm::Error storeMCCASObjectID(StringRef OutputFilename, llvm::cas::CASID ID);

include/swift/Frontend/CachingUtils.h

@@ -40,11 +40,13 @@ createSwiftCachingOutputBackend(
 
 /// Replay the output of the compilation from cache.
 /// Return true if outputs are replayed, false otherwise.
-bool replayCachedCompilerOutputs(
-    llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
-    llvm::cas::ObjectRef BaseKey, DiagnosticEngine &Diag,
-    const FrontendInputsAndOutputs &InputsAndOutputs,
-    CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend);
+bool replayCachedCompilerOutputs(llvm::cas::ObjectStore &CAS,
+                                 llvm::cas::ActionCache &Cache,
+                                 llvm::cas::ObjectRef BaseKey,
+                                 DiagnosticEngine &Diag,
+                                 const FrontendOptions &Opts,
+                                 CachingDiagnosticsProcessor &CDP,
+                                 bool CacheRemarks, bool UseCASBackend);
 
 /// Load the cached compile result from cache.
 std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(

include/swift/Frontend/MakeStyleDependencies.h

@@ -0,0 +1,46 @@
+//===--- MakeStyleDependencies.h -- header for emitting dependency files --===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines interface for emitting Make Style Dependency Files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_FRONTEND_MAKESTYLEDEPENDENCIES_H
+#define SWIFT_FRONTEND_MAKESTYLEDEPENDENCIES_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class raw_ostream;
+} // namespace llvm
+
+namespace swift {
+
+class CompilerInstance;
+class DiagnosticEngine;
+class FrontendOptions;
+class InputFile;
+
+/// Emit make style dependency file from compiler instance if needed.
+bool emitMakeDependenciesIfNeeded(CompilerInstance &instance,
+                                  const InputFile &input);
+
+/// Emit make style dependency file from a serialized buffer.
+bool emitMakeDependenciesFromSerializedBuffer(llvm::StringRef buffer,
+                                              llvm::raw_ostream &os,
+                                              const FrontendOptions &opts,
+                                              const InputFile &input,
+                                              DiagnosticEngine &diags);
+
+} // end namespace swift
+
+#endif

lib/Frontend/CASOutputBackends.cpp

@@ -73,7 +73,7 @@ class SwiftCASOutputBackend::Implementation {
     auto OutputType = ProducingInput->second.second;
 
     // Uncached output kind.
-    if (file_types::isProducedFromDiagnostics(OutputType))
+    if (!isStoredDirectly(OutputType))
       return std::make_unique<llvm::vfs::NullOutputFileImpl>();
 
     return std::make_unique<SwiftCASOutputFile>(
@@ -115,6 +115,11 @@ SwiftCASOutputBackend::SwiftCASOutputBackend(
 
 SwiftCASOutputBackend::~SwiftCASOutputBackend() { delete &Impl; }
 
+bool SwiftCASOutputBackend::isStoredDirectly(file_types::ID Kind) {
+  return !file_types::isProducedFromDiagnostics(Kind) &&
+         Kind != file_types::TY_Dependencies;
+}
+
 IntrusiveRefCntPtr<OutputBackend> SwiftCASOutputBackend::cloneImpl() const {
   return makeIntrusiveRefCnt<SwiftCASOutputBackend>(
       Impl.CAS, Impl.Cache, Impl.BaseKey, Impl.InputsAndOutputs, Impl.Action);
@@ -142,6 +147,18 @@ Error SwiftCASOutputBackend::storeCachedDiagnostics(unsigned InputIndex,
                    file_types::ID::TY_CachedDiagnostics);
 }
 
+Error SwiftCASOutputBackend::storeMakeDependenciesFile(StringRef OutputFilename,
+                                                       llvm::StringRef Bytes) {
+  auto Input = Impl.OutputToInputMap.find(OutputFilename);
+  if (Input == Impl.OutputToInputMap.end())
+    return llvm::createStringError("InputIndex for output file not found!");
+  auto InputIndex = Input->second.first;
+  assert(Input->second.second == file_types::TY_Dependencies &&
+         "wrong output type");
+  return storeImpl(OutputFilename, Bytes, InputIndex,
+                   file_types::TY_Dependencies);
+}
+
 Error SwiftCASOutputBackend::storeMCCASObjectID(StringRef OutputFilename,
                                                 llvm::cas::CASID ID) {
   auto Input = Impl.OutputToInputMap.find(OutputFilename);

lib/Frontend/CMakeLists.txt

@@ -15,6 +15,7 @@ add_swift_host_library(swiftFrontend STATIC
   Frontend.cpp
   FrontendInputsAndOutputs.cpp
   FrontendOptions.cpp
+  MakeStyleDependencies.cpp
   ModuleInterfaceBuilder.cpp
   ModuleInterfaceLoader.cpp
   ModuleInterfaceSupport.cpp

lib/Frontend/CachingUtils.cpp

@@ -20,6 +20,7 @@
 #include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Frontend/CompileJobCacheResult.h"
 #include "swift/Frontend/FrontendOptions.h"
+#include "swift/Frontend/MakeStyleDependencies.h"
 #include "swift/Option/Options.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/CAS/IncludeTree.h"
@@ -135,19 +136,21 @@ lookupCacheKey(ObjectStore &CAS, ActionCache &Cache, ObjectRef CacheKey) {
 
 bool replayCachedCompilerOutputs(
     ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
-    DiagnosticEngine &Diag, const FrontendInputsAndOutputs &InputsAndOutputs,
+    DiagnosticEngine &Diag, const FrontendOptions &Opts,
     CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend) {
   bool CanReplayAllOutput = true;
   struct OutputEntry {
     std::string Path;
     CASID Key;
+    file_types::ID Kind;
+    const InputFile &Input;
     ObjectProxy Proxy;
   };
   SmallVector<OutputEntry> OutputProxies;
   std::optional<OutputEntry> DiagnosticsOutput;
-  std::string ObjFile;
 
-  auto replayOutputsForInputFile = [&](const std::string &InputPath,
+  auto replayOutputsForInputFile = [&](const InputFile &Input,
+                                       const std::string &InputPath,
                                        unsigned InputIndex,
                                        const DenseMap<file_types::ID,
                                                       std::string> &Outputs) {
@@ -190,15 +193,13 @@ bool replayCachedCompilerOutputs(
           if (!Proxy)
             return Proxy.takeError();
 
-          if (Kind == file_types::ID::TY_Object && UseCASBackend)
-            ObjFile = OutputPath->second;
-
           if (Kind == file_types::ID::TY_CachedDiagnostics) {
             assert(!DiagnosticsOutput && "more than 1 diagnotics found");
-            DiagnosticsOutput = OutputEntry{OutputPath->second, OutID, *Proxy};
+            DiagnosticsOutput.emplace(
+                OutputEntry{OutputPath->second, OutID, Kind, Input, *Proxy});
           } else
             OutputProxies.emplace_back(
-                OutputEntry{OutputPath->second, OutID, *Proxy});
+                OutputEntry{OutputPath->second, OutID, Kind, Input, *Proxy});
           return Error::success();
         })) {
       Diag.diagnose(SourceLoc(), diag::error_cas, toString(std::move(Err)));
@@ -211,7 +212,7 @@ bool replayCachedCompilerOutputs(
     auto InputPath = Input.getFileName();
     DenseMap<file_types::ID, std::string> Outputs;
     if (!Input.outputFilename().empty())
-      Outputs.try_emplace(InputsAndOutputs.getPrincipalOutputType(),
+      Outputs.try_emplace(Opts.InputsAndOutputs.getPrincipalOutputType(),
                           Input.outputFilename());
 
     Input.getPrimarySpecificPaths()
@@ -233,15 +234,15 @@ bool replayCachedCompilerOutputs(
     Outputs.try_emplace(file_types::ID::TY_CachedDiagnostics,
                         "<cached-diagnostics>");
 
-    return replayOutputsForInputFile(InputPath, InputIndex, Outputs);
+    return replayOutputsForInputFile(Input, InputPath, InputIndex, Outputs);
   };
 
-  auto AllInputs = InputsAndOutputs.getAllInputs();
+  auto AllInputs = Opts.InputsAndOutputs.getAllInputs();
   // If there are primary inputs, look up only the primary input files.
   // Otherwise, prepare to do cache lookup for all inputs.
   for (unsigned Index = 0; Index < AllInputs.size(); ++Index) {
     const auto &Input = AllInputs[Index];
-    if (InputsAndOutputs.hasPrimaryInputs() && !Input.isPrimary())
+    if (Opts.InputsAndOutputs.hasPrimaryInputs() && !Input.isPrimary())
       continue;
 
     replayOutputFromInput(Input, Index);
@@ -255,8 +256,9 @@ bool replayCachedCompilerOutputs(
   // diagnostics from first file.
   if (!DiagnosticsOutput)
     replayOutputsForInputFile(
+        Opts.InputsAndOutputs.getFirstOutputProducingInput(),
         "<cached-diagnostics>",
-        InputsAndOutputs.getIndexOfFirstOutputProducingInput(),
+        Opts.InputsAndOutputs.getIndexOfFirstOutputProducingInput(),
         {{file_types::ID::TY_CachedDiagnostics, "<cached-diagnostics>"}});
 
   // Check again to make sure diagnostics is fetched successfully.
@@ -287,10 +289,16 @@ bool replayCachedCompilerOutputs(
       continue;
     }
 
-    if (UseCASBackend && Output.Path == ObjFile) {
+    if (UseCASBackend && Output.Kind == file_types::ID::TY_Object) {
       auto Schema = std::make_unique<llvm::mccasformats::v1::MCSchema>(CAS);
-      if (auto E = Schema->serializeObjectFile(Output.Proxy, *File))
+      if (auto E = Schema->serializeObjectFile(Output.Proxy, *File)) {
         Diag.diagnose(SourceLoc(), diag::error_mccas, toString(std::move(E)));
+        return false;
+      }
+    } else if (Output.Kind == file_types::ID::TY_Dependencies) {
+      if (emitMakeDependenciesFromSerializedBuffer(
+              Output.Proxy.getData(), *File, Opts, Output.Input, Diag))
+        return false;
     } else
       *File << Output.Proxy.getData();