Experimental markdown output (#1303)

* Add experimental markdown output flag and pass it through to the convert feature flags

* Initial export of Markdown from Article

* Initial processing of a type-level symbol

* Adds symbol declarations and article reference links

* Output tutorials to markdown

* Be smarter about removing indentation from within block directives

* Baseline for adding new tests for markdown output

* Basic test infrastructure for markdown output

* Adds symbol link tests to markdown output

* Tutoorial code rendering markdown tests

* Adding metadata to markdown output

* Include package source for markdown output test catalog

* Output metadata updates

* Adds default availability for modules to markdown export

* Move availability out of symbol and in to general metadata for markdown output

* Refactor markdown output so the final node type is standalone

* Add generated markdown flag to render node metadata

* Only include unavailable in markdown header if it is true

* Initial setup of manifest output, no references

* Output of manifest / relationships

* Manifest output format updates

* Remove member symbol relationship

* More compact availability, deal with metadata availability for symbols

* Update tests so all inputs are locally defined

* Remove print statements from unused visitors

* Added snippet handling

* Remove or _spi-hide new public API

* Remove or _spi-hide new public API (part 2)

* Prevent recursion when the abstract of a link contains a link and we are rendering links-with-abstracts

* Test and temporary fix for colspan issue swiftlang/swift-markdown#238

* Remove separate SwiftDocCMarkdownOutput target

* Bump swift-markdown, remove temporary bug fix

* Updates to handle removed bundle parameter

* Extract common writer code into a helper function

* Clarify use of article.md in test catalog

* Deal with internal links with anchor tags

* Deal with unresolved symbol links

* Updates to improve public API / clarity of MarkdownOutputNode

* Improve public API of manifest, sort contents before writing to prevent unneccessary diffs on re-run

* Don't use unstructured `Data` for WritableMarkdownOutputNode

* Reduce public SPI surface around markdown writing

* Make relationship subtype a specific type rather than a string

* Make it clear that path component is a fallback title, harden linked list rendering

* Add todos and missing tests

* Move markdown output types from public SPI to package

* don't include scheme or host in documentation links

* Reformat multi-condition if statements

* remove whitespace-only change

* Remove unused declaration

* Remove redundant return types

* Remove MarkdownOutputNodeTranslator

* Add TODO for alternate declarations

* Add TODO for structural review

* Add stacks todo, set writable type back to package

* uri -> identifier

* Use RelationshipsGroup.Kind instead of creating a mirror type

* Lower access level of manifest, remove children method

* Restrict package level declarations to those actually used at package level

* Merging main

---------

Co-authored-by: Richard Turton <r_turton@apple.com>

Author: jrturton

Package.resolved

@@ -11,7 +11,7 @@
 import Foundation
 import SwiftDocC
 
-struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {
+struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer, ConvertOutputMarkdownConsumer {
     var targetFolder: URL
     var bundleRootFolder: URL?
     var fileManager: any FileManagerProtocol
@@ -68,6 +68,21 @@ struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {
         indexer?.index(renderNode)
     }
 
+    func consume(markdownNode: WritableMarkdownOutputNode) throws {
+        try renderNodeWriter.write(markdownNode)
+    }
+    
+    func consume(markdownManifest: MarkdownOutputManifest) throws {
+        let url = targetFolder.appendingPathComponent("\(markdownManifest.title)-markdown-manifest.json", isDirectory: false)
+        let encoder = JSONEncoder()
+        encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes]
+        #if DEBUG
+        encoder.outputFormatting.insert(.prettyPrinted)
+        #endif
+        let data = try encoder.encode(markdownManifest)
+        try fileManager.createFile(at: url, contents: data)
+    }
+    
     func consume(externalRenderNode: ExternalRenderNode) throws {
         // Index the external node, if indexing is enabled.
         indexer?.index(externalRenderNode)

Sources/DocCCommandLine/Action/Actions/Convert/ConvertFileWritingConsumer.swift

@@ -89,6 +89,25 @@ class JSONEncodingRenderNodeWriter {
         }
     }
 
+    /// Writes a markdown node to a file at a location based on the node's relative URL.
+    ///
+    /// If the target path to the markdown file includes intermediate folders that don't exist, the writer object will ask the file manager, with which it was created, to
+    /// create those intermediate folders before writing the markdown file.
+    ///
+    /// - Parameters:
+    ///   - markdownNode: The node which the writer object writes
+    func write(_ markdownNode: WritableMarkdownOutputNode) throws {
+        let fileSafePath = NodeURLGenerator.fileSafeReferencePath(
+            markdownNode.identifier,
+            lowercased: true
+        )
+        
+        try write(
+            markdownNode.node.generateDataRepresentation(),
+            toFileSafePath: "data/\(fileSafePath).md"
+        )
+    }
+    
     func write(_ data: Data, toFileSafePath fileSafePath: String) throws {
         // The path on disk to write the data at.
         let fileURL = targetFolder.appendingPathComponent(fileSafePath, isDirectory: false)

Sources/DocCCommandLine/Action/Actions/Convert/JSONEncodingRenderNodeWriter.swift

@@ -25,6 +25,8 @@ extension ConvertAction {
         FeatureFlags.current.isExperimentalOverloadedSymbolPresentationEnabled = convert.enableExperimentalOverloadedSymbolPresentation
         FeatureFlags.current.isMentionedInEnabled = convert.enableMentionedIn
         FeatureFlags.current.isParametersAndReturnsValidationEnabled = convert.enableParametersAndReturnsValidation
+        FeatureFlags.current.isExperimentalMarkdownOutputEnabled = convert.enableExperimentalMarkdownOutput
+        FeatureFlags.current.isExperimentalMarkdownOutputManifestEnabled = convert.enableExperimentalMarkdownOutputManifest
 
         // If the user-provided a URL for an external link resolver, attempt to
         // initialize an `OutOfProcessReferenceResolver` with the provided URL.

Sources/DocCCommandLine/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift

@@ -543,6 +543,12 @@ extension Docc {
             @available(*, deprecated, message: "This flag is unused and only exist for backwards compatibility")
             var _unusedExperimentalMentionedInFlagForBackwardsCompatibility = false
 
+            @Flag(help: "Experimental: Create markdown versions of documents")
+            var enableExperimentalMarkdownOutput = false
+            
+            @Flag(help: "Experimental: Create manifest file of markdown outputs. Ignored if --enable-experimental-markdown-output is not set.")
+            var enableExperimentalMarkdownOutputManifest = false
+            
             @Flag(
                 name: .customLong("parameters-and-returns-validation"),
                 inversion: .prefixedEnableDisable,
@@ -637,6 +643,18 @@ extension Docc {
             get { featureFlags.enableExperimentalOverloadedSymbolPresentation }
             set { featureFlags.enableExperimentalOverloadedSymbolPresentation = newValue }
         }
+        
+        /// A user-provided value that is true if the user enables experimental markdown output
+        public var enableExperimentalMarkdownOutput: Bool {
+            get { featureFlags.enableExperimentalMarkdownOutput }
+            set { featureFlags.enableExperimentalMarkdownOutput = newValue }
+        }
+        
+        /// A user-provided value that is true if the user enables experimental markdown output
+        public var enableExperimentalMarkdownOutputManifest: Bool {
+            get { featureFlags.enableExperimentalMarkdownOutputManifest }
+            set { featureFlags.enableExperimentalMarkdownOutputManifest = newValue }
+        }
 
         /// A user-provided value that is true if the user enables experimental automatically generated "mentioned in"
         /// links on symbols.

Sources/DocCCommandLine/ArgumentParsing/Subcommands/Convert.swift

@@ -113,4 +113,21 @@ public class DocumentationContextConverter {
         )
         return translator.visit(node.semantic) as? RenderNode
     }
+    
+    /// Converts a documentation node to a markdown node.
+    /// - Parameters:
+    ///   - node: The documentation node to convert.
+    /// - Returns: The markdown node representation of the documentation node.
+    internal func markdownOutput(for node: DocumentationNode) -> CollectedMarkdownOutput? {
+        guard !node.isVirtual else {
+            return nil
+        }
+        
+        var visitor = MarkdownOutputSemanticVisitor(context: context, node: node)
+        
+        if let node = visitor.createOutput() {
+            return CollectedMarkdownOutput(identifier: visitor.identifier, node: node, manifest: visitor.manifest)
+        }
+        return nil
+    }
 }

Sources/SwiftDocC/Converter/DocumentationContextConverter.swift

@@ -80,6 +80,7 @@ package enum ConvertActionConverter {
         var assets = [RenderReferenceType : [any RenderReference]]()
         var coverageInfo = [CoverageDataEntry]()
         let coverageFilterClosure = documentationCoverageOptions.generateFilterClosure()
+        var markdownManifest = MarkdownOutputManifest(title: context.inputs.displayName, documents: [])
 
         // An inner function to gather problems for errors encountered during the conversion.
         //
@@ -131,6 +132,21 @@ package enum ConvertActionConverter {
                         return
                     }
 
+                    if FeatureFlags.current.isExperimentalMarkdownOutputEnabled,
+                       let markdownConsumer = outputConsumer as? (any ConvertOutputMarkdownConsumer),
+                       let markdownNode = converter.markdownOutput(for: entity)
+                    {
+                        try markdownConsumer.consume(markdownNode: markdownNode.writable)
+                        if FeatureFlags.current.isExperimentalMarkdownOutputManifestEnabled,
+                           let manifest = markdownNode.manifest
+                        {
+                            resultsGroup.async(queue: resultsSyncQueue) {
+                                markdownManifest.documents.formUnion(manifest.documents)
+                                markdownManifest.relationships.formUnion(manifest.relationships)
+                            }
+                        }
+                    }
+                    
                     try outputConsumer.consume(renderNode: renderNode)
 
                     switch documentationCoverageOptions.level {
@@ -235,6 +251,12 @@ package enum ConvertActionConverter {
                 }
             }
         }
+        
+        if FeatureFlags.current.isExperimentalMarkdownOutputManifestEnabled,
+           let markdownConsumer = outputConsumer as? (any ConvertOutputMarkdownConsumer)
+        {
+            try markdownConsumer.consume(markdownManifest: markdownManifest)
+        }
 
         switch documentationCoverageOptions.level {
         case .detailed, .brief:

Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift

@@ -48,6 +48,15 @@ public protocol ConvertOutputConsumer {
 
     /// Consumes a file representation of the local link resolution information.
     func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws
+    
+}
+
+package protocol ConvertOutputMarkdownConsumer {
+    /// Consumes a markdown output node
+    func consume(markdownNode: WritableMarkdownOutputNode) throws
+    
+    /// Consumes a markdown output manifest
+    func consume(markdownManifest: MarkdownOutputManifest) throws
 }
 
 // Default implementations that discard the documentation conversion products, for consumers that don't need these