Fetch full documentation in code completion (#2207)

When resolving documentation for code completion items, we fetch full
documentation through the newly added
`swiftide_completion_item_get_doc_full_copy` SourceKitD function, if not
found we fallback to brief documentation as before using
`swiftide_completion_item_get_doc_brief`.

> [!NOTE]
> Unlike brief documentation, SourceKitD doesn't cache full
documentation for completion results to avoid bloating memory with a lot
of large strings.
>
> As of now, SourceKit-LSP doesn't cache completion item documentation
either, should we introduce a new full documentation cache (e.g. using
`LRUCache`)?

Author: a7medev

Sources/Csourcekitd/include/CodeCompletionSwiftInterop.h

@@ -324,6 +324,16 @@ typedef struct {
     void (^_Null_unspecified handler)(const char *_Null_unspecified)
   );
 
+  void (*_Nullable completion_item_get_doc_full_as_xml)(
+      _Nonnull swiftide_api_completion_response_t,
+      _Nonnull swiftide_api_completion_item_t,
+      void (^_Nonnull handler)(const char *_Nullable));
+
+  void (*_Nullable completion_item_get_doc_raw)(
+      _Nonnull swiftide_api_completion_response_t,
+      _Nonnull swiftide_api_completion_item_t,
+      void (^_Nonnull handler)(const char *_Nullable));
+
   void (*_Nonnull completion_item_get_associated_usrs)(
     _Null_unspecified swiftide_api_completion_response_t,
     _Null_unspecified swiftide_api_completion_item_t,

Sources/SKTestSupport/SkipUnless.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Csourcekitd
 import Foundation
 import InProcessClient
 import LanguageServerProtocol
@@ -250,13 +251,8 @@ package actor SkipUnless {
     line: UInt = #line
   ) async throws {
     return try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(6, 2), file: file, line: line) {
-      guard let sourcekitdPath = await ToolchainRegistry.forTesting.default?.sourcekitd else {
-        throw GenericError("Could not find SourceKitD")
-      }
-      let sourcekitd = try await SourceKitD.getOrCreate(
-        dylibPath: sourcekitdPath,
-        pluginPaths: try sourceKitPluginPaths
-      )
+      let sourcekitd = try await getSourceKitD()
+
       do {
         let response = try await sourcekitd.send(
           \.codeCompleteSetPopularAPI,
@@ -274,6 +270,17 @@ package actor SkipUnless {
     }
   }
 
+  package static func sourcekitdSupportsFullDocumentationInCompletion(
+    file: StaticString = #filePath,
+    line: UInt = #line
+  ) async throws {
+    return try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(6, 2), file: file, line: line) {
+      let sourcekitd = try await getSourceKitD()
+
+      return sourcekitd.ideApi.completion_item_get_doc_raw != nil
+    }
+  }
+
   package static func canLoadPluginsBuiltByToolchain(
     file: StaticString = #filePath,
     line: UInt = #line
@@ -424,6 +431,18 @@ package actor SkipUnless {
     }
     #endif
   }
+
+  private static func getSourceKitD() async throws -> SourceKitD {
+    guard let sourcekitdPath = await ToolchainRegistry.forTesting.default?.sourcekitd else {
+      throw GenericError("Could not find SourceKitD")
+    }
+    let sourcekitd = try await SourceKitD.getOrCreate(
+      dylibPath: sourcekitdPath,
+      pluginPaths: try sourceKitPluginPaths
+    )
+
+    return sourcekitd
+  }
 }
 
 // MARK: - Parsing Swift compiler version

Sources/SourceKitD/sourcekitd_functions.swift

@@ -146,6 +146,8 @@ extension sourcekitd_ide_api_functions_t {
       completion_item_get_source_text: try loadRequired("swiftide_completion_item_get_source_text"),
       completion_item_get_type_name: try loadRequired("swiftide_completion_item_get_type_name"),
       completion_item_get_doc_brief: try loadRequired("swiftide_completion_item_get_doc_brief"),
+      completion_item_get_doc_full_as_xml: loadOptional("swiftide_completion_item_get_doc_full_as_xml"),
+      completion_item_get_doc_raw: loadOptional("swiftide_completion_item_get_doc_raw"),
       completion_item_get_associated_usrs: try loadRequired("swiftide_completion_item_get_associated_usrs"),
       completion_item_get_kind: try loadRequired("swiftide_completion_item_get_kind"),
       completion_item_get_associated_kind: try loadRequired("swiftide_completion_item_get_associated_kind"),

Sources/SwiftLanguageService/CodeCompletionSession.swift

@@ -575,13 +575,24 @@ class CodeCompletionSession {
           fileContents: nil
         )
       }
-      if let docString: String = documentationResponse?[sourcekitd.keys.docBrief] {
+
+      if let response = documentationResponse,
+        let docString = documentationString(from: response, sourcekitd: sourcekitd)
+      {
         item.documentation = .markupContent(MarkupContent(kind: .markdown, value: docString))
       }
     }
     return item
   }
 
+  private static func documentationString(from response: SKDResponseDictionary, sourcekitd: SourceKitD) -> String? {
+    if let docComment: String = response[sourcekitd.keys.docComment] {
+      return docComment
+    }
+
+    return response[sourcekitd.keys.docBrief]
+  }
+
   private func computeCompletionTextEdit(
     completionPos: Position,
     requestPosition: Position,

Sources/SwiftSourceKitPlugin/ASTCompletion/CompletionSession.swift

@@ -241,6 +241,26 @@ struct ExtendedCompletionInfo {
     return result
   }
 
+  var fullDocumentationAsXML: String? {
+    var result: String? = nil
+    session.sourcekitd.ideApi.completion_item_get_doc_full_as_xml?(session.response, rawItem) {
+      if let cstr = $0 {
+        result = String(cString: cstr)
+      }
+    }
+    return result
+  }
+
+  var rawDocumentation: String? {
+    var result: String? = nil
+    session.sourcekitd.ideApi.completion_item_get_doc_raw?(session.response, rawItem) {
+      if let cstr = $0 {
+        result = String(cString: cstr)
+      }
+    }
+    return result
+  }
+
   var associatedUSRs: [String] {
     var result: [String] = []
     session.sourcekitd.ideApi.completion_item_get_associated_usrs(session.response, rawItem) { ptr, len in

Sources/SwiftSourceKitPlugin/CompletionProvider.swift

@@ -266,6 +266,8 @@ actor CompletionProvider {
 
     return request.sourcekitd.responseDictionary([
       request.sourcekitd.keys.docBrief: info.briefDocumentation,
+      request.sourcekitd.keys.docFullAsXML: info.fullDocumentationAsXML,
+      request.sourcekitd.keys.docComment: info.rawDocumentation,
       request.sourcekitd.keys.associatedUSRs: info.associatedUSRs as [SKDResponseValue]?,
     ])
   }

Tests/SourceKitLSPTests/SwiftCompletionTests.swift

@@ -31,6 +31,7 @@ final class SwiftCompletionTests: XCTestCase {
 
   func testCompletionBasic() async throws {
     try await SkipUnless.sourcekitdSupportsPlugin()
+    try await SkipUnless.sourcekitdSupportsFullDocumentationInCompletion()
 
     let testClient = try await TestSourceKitLSPClient()
     let uri = DocumentURI(for: .swift)
@@ -39,6 +40,13 @@ final class SwiftCompletionTests: XCTestCase {
       """
       struct S {
         /// Documentation for `abc`.
+        ///
+        /// _More_ documentation for `abc`.
+        ///
+        /// Usage:
+        /// ```swift
+        /// S().abc
+        /// ```
         var abc: Int
 
         func test(a: Int) {
@@ -67,7 +75,19 @@ final class SwiftCompletionTests: XCTestCase {
     if let abc = abc {
       XCTAssertEqual(abc.kind, .property)
       XCTAssertEqual(abc.detail, "Int")
-      XCTAssertEqual(abc.documentation, .markupContent(MarkupContent(kind: .markdown, value: "Documentation for abc.")))
+      assertMarkdown(
+        documentation: abc.documentation,
+        expected: """
+          Documentation for `abc`.
+
+          _More_ documentation for `abc`.
+
+          Usage:
+          ```swift
+          S().abc
+          ```
+          """
+      )
       XCTAssertEqual(abc.filterText, "abc")
       XCTAssertEqual(abc.textEdit, .textEdit(TextEdit(range: Range(positions["1️⃣"]), newText: "abc")))
       XCTAssertEqual(abc.insertText, "abc")
@@ -87,7 +107,19 @@ final class SwiftCompletionTests: XCTestCase {
       // If we switch to server-side filtering this will change.
       XCTAssertEqual(abc.kind, .property)
       XCTAssertEqual(abc.detail, "Int")
-      XCTAssertEqual(abc.documentation, .markupContent(MarkupContent(kind: .markdown, value: "Documentation for abc.")))
+      assertMarkdown(
+        documentation: abc.documentation,
+        expected: """
+          Documentation for `abc`.
+
+          _More_ documentation for `abc`.
+
+          Usage:
+          ```swift
+          S().abc
+          ```
+          """
+      )
       XCTAssertEqual(abc.filterText, "abc")
       XCTAssertEqual(abc.textEdit, .textEdit(TextEdit(range: positions["1️⃣"]..<offsetPosition, newText: "abc")))
       XCTAssertEqual(abc.insertText, "abc")
@@ -1154,6 +1186,7 @@ final class SwiftCompletionTests: XCTestCase {
 
   func testCompletionItemResolve() async throws {
     try await SkipUnless.sourcekitdSupportsPlugin()
+    try await SkipUnless.sourcekitdSupportsFullDocumentationInCompletion()
 
     let capabilities = ClientCapabilities(
       textDocument: TextDocumentClientCapabilities(
@@ -1187,9 +1220,37 @@ final class SwiftCompletionTests: XCTestCase {
     let item = try XCTUnwrap(completions.items.only)
     XCTAssertNil(item.documentation)
     let resolvedItem = try await testClient.send(CompletionItemResolveRequest(item: item))
-    XCTAssertEqual(
-      resolvedItem.documentation,
-      .markupContent(MarkupContent(kind: .markdown, value: "Creates a true value"))
+    assertMarkdown(
+      documentation: resolvedItem.documentation,
+      expected: "Creates a true value"
+    )
+  }
+
+  func testCompletionBriefDocumentationFallback() async throws {
+    try await SkipUnless.sourcekitdSupportsPlugin()
+    try await SkipUnless.sourcekitdSupportsFullDocumentationInCompletion()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+
+    // We test completion for result builder build functions since they don't have full documentation
+    // but still have brief documentation.
+    let positions = testClient.openDocument(
+      """
+      @resultBuilder
+      struct AnyBuilder {
+        static func 1️⃣
+      }
+      """,
+      uri: uri
+    )
+    let completions = try await testClient.send(
+      CompletionRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"])
+    )
+    let item = try XCTUnwrap(completions.items.filter { $0.label.contains("buildBlock") }.only)
+    assertMarkdown(
+      documentation: item.documentation,
+      expected: "Required by every result builder to build combined results from statement blocks"
     )
   }
 
@@ -1253,6 +1314,20 @@ private func countFs(_ response: CompletionList) -> Int {
   return response.items.filter { $0.label.hasPrefix("f") }.count
 }
 
+private func assertMarkdown(
+  documentation: StringOrMarkupContent?,
+  expected: String,
+  file: StaticString = #filePath,
+  line: UInt = #line
+) {
+  XCTAssertEqual(
+    documentation,
+    .markupContent(MarkupContent(kind: .markdown, value: expected)),
+    file: file,
+    line: line
+  )
+}
+
 fileprivate extension Position {
   func adding(columns: Int) -> Position {
     return Position(line: line, utf16index: utf16index + columns)

Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift

@@ -46,6 +46,8 @@ final class SwiftSourceKitPluginTests: XCTestCase {
 
   func testBasicCompletion() async throws {
     try await SkipUnless.sourcekitdSupportsPlugin()
+    try await SkipUnless.sourcekitdSupportsFullDocumentationInCompletion()
+
     let sourcekitd = try await getSourceKitD()
     let path = scratchFilePath()
     let positions = try await sourcekitd.openDocument(
@@ -203,7 +205,9 @@ final class SwiftSourceKitPluginTests: XCTestCase {
     XCTAssertEqual(result2.items.count, 1)
     XCTAssertEqual(result2.items[0].name, "")
     let doc = try await sourcekitd.completeDocumentation(id: result2.items[0].id)
-    XCTAssertEqual(doc.docBrief, nil)
+    XCTAssertNil(doc.docComment)
+    XCTAssertNil(doc.docFullAsXML)
+    XCTAssertNil(doc.docBrief)
   }
 
   func testMultipleFiles() async throws {
@@ -403,6 +407,8 @@ final class SwiftSourceKitPluginTests: XCTestCase {
 
   func testDocumentation() async throws {
     try await SkipUnless.sourcekitdSupportsPlugin()
+    try await SkipUnless.sourcekitdSupportsFullDocumentationInCompletion()
+
     let sourcekitd = try await getSourceKitD()
     let path = scratchFilePath()
     let positions = try await sourcekitd.openDocument(
@@ -436,14 +442,47 @@ final class SwiftSourceKitPluginTests: XCTestCase {
     let sym3 = try unwrap(result.items.first(where: { $0.name == "foo3()" }), "did not find foo3; got \(result.items)")
 
     let sym1Doc = try await sourcekitd.completeDocumentation(id: sym1.id)
+    XCTAssertEqual(sym1Doc.docComment, "Protocol P foo1")
+    XCTAssertEqual(
+      sym1Doc.docFullAsXML,
+      """
+      <Function file="\(path)" line="3" column="8">\
+      <Name>foo1()</Name>\
+      <USR>s:1a1PP4foo1yyF</USR>\
+      <Declaration>func foo1()</Declaration>\
+      <CommentParts>\
+      <Abstract><Para>Protocol P foo1</Para></Abstract>\
+      <Discussion><Note>\
+      <Para>This documentation comment was inherited from <codeVoice>P</codeVoice>.</Para>\
+      </Note></Discussion>\
+      </CommentParts>\
+      </Function>
+      """
+    )
     XCTAssertEqual(sym1Doc.docBrief, "Protocol P foo1")
     XCTAssertEqual(sym1Doc.associatedUSRs, ["s:1a1SV4foo1yyF", "s:1a1PP4foo1yyF"])
 
     let sym2Doc = try await sourcekitd.completeDocumentation(id: sym2.id)
+    XCTAssertEqual(sym2Doc.docComment, "Struct S foo2")
+    XCTAssertEqual(
+      sym2Doc.docFullAsXML,
+      """
+      <Function file="\(path)" line="8" column="8">\
+      <Name>foo2()</Name>\
+      <USR>s:1a1SV4foo2yyF</USR>\
+      <Declaration>func foo2()</Declaration>\
+      <CommentParts>\
+      <Abstract><Para>Struct S foo2</Para></Abstract>\
+      </CommentParts>\
+      </Function>
+      """
+    )
     XCTAssertEqual(sym2Doc.docBrief, "Struct S foo2")
     XCTAssertEqual(sym2Doc.associatedUSRs, ["s:1a1SV4foo2yyF"])
 
     let sym3Doc = try await sourcekitd.completeDocumentation(id: sym3.id)
+    XCTAssertNil(sym3Doc.docComment)
+    XCTAssertNil(sym3Doc.docFullAsXML)
     XCTAssertNil(sym3Doc.docBrief)
     XCTAssertEqual(sym3Doc.associatedUSRs, ["s:1a1SV4foo3yyF"])
   }
@@ -1766,11 +1805,15 @@ private struct CompletionResult: Equatable, Sendable {
 }
 
 private struct CompletionDocumentation {
+  var docComment: String? = nil
+  var docFullAsXML: String? = nil
   var docBrief: String? = nil
   var associatedUSRs: [String] = []
 
   init(_ dict: SKDResponseDictionary) {
     let keys = dict.sourcekitd.keys
+    self.docComment = dict[keys.docComment]
+    self.docFullAsXML = dict[keys.docFullAsXML]
     self.docBrief = dict[keys.docBrief]
     self.associatedUSRs = dict[keys.associatedUSRs]?.asStringArray ?? []
   }