Merge pull request #136 from hamishknight/so-extra

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/CustomCharClass.swift

@@ -42,6 +42,9 @@ extension AST {
       /// the contents should be interpreted literally.
       case quote(Quote)
 
+      /// Trivia such as non-semantic whitespace.
+      case trivia(Trivia)
+
       /// A binary operator applied to sets of members `abc&&def`
       case setOperation([Member], Located<SetOp>, [Member])
     }
@@ -81,11 +84,27 @@ extension CustomCC.Member {
     case .range(let r): return r
     case .atom(let a): return a
     case .quote(let q): return q
+    case .trivia(let t): return t
     case .setOperation(let lhs, let op, let rhs): return (lhs, op, rhs)
     }
   }
 
   func `as`<T>(_ t: T.Type = T.self) -> T? {
     _associatedValue as? T
   }
+
+  public var isTrivia: Bool {
+    if case .trivia = self { return true }
+    return false
+  }
+}
+
+extension AST.CustomCharacterClass {
+  /// Strip trivia from the character class members. This does not recurse into
+  /// nested custom character classes.
+  public var strippingTriviaShallow: Self {
+    var copy = self
+    copy.members = copy.members.filter { !$0.isTrivia }
+    return copy
+  }
 }

Sources/_MatchingEngine/Regex/AST/MatchingOptions.swift

@@ -50,6 +50,16 @@ extension AST {
       self.location = location
     }
 
+    /// If this is either the regular or extra extended syntax option.
+    public var isAnyExtended: Bool {
+      switch kind {
+      case .extended, .extraExtended:
+        return true
+      default:
+        return false
+      }
+    }
+
     public var isTextSegmentMode: Bool {
       switch kind {
       case .textSegmentGraphemeMode, .textSegmentWordMode:
@@ -93,6 +103,10 @@ extension AST {
       self.minusLoc = minusLoc
       self.removing = removing
     }
+
+    /// Whether this set of matching options first resets the options before
+    /// adding onto them.
+    public var resetsCurrentOptions: Bool { caretLoc != nil }
   }
 }
 
@@ -102,7 +116,10 @@ extension AST.MatchingOption: _ASTPrintable {
 
 extension AST.MatchingOptionSequence: _ASTPrintable {
   public var _dumpBase: String {
-    "adding: \(adding), removing: \(removing), hasCaret: \(caretLoc != nil)"
+    """
+    adding: \(adding), removing: \(removing), \
+    resetsCurrentOptions: \(resetsCurrentOptions)
+    """
   }
 }
 

Sources/_MatchingEngine/Regex/AST/Quantification.swift

@@ -17,16 +17,23 @@ extension AST {
     public let child: AST.Node
     public let location: SourceLocation
 
+    /// Any trivia intermixed between the operand and the quantifier, as well
+    /// as between the quantifier characters themselves. This can occur in
+    /// extended syntax mode where PCRE permits e.g `x * +`.
+    public let trivia: [AST.Trivia]
+
     public init(
       _ amount: Located<Amount>,
       _ kind: Located<Kind>,
       _ child: AST.Node,
-      _ r: SourceLocation
+      _ r: SourceLocation,
+      trivia: [AST.Trivia]
     ) {
       self.amount = amount
       self.kind = kind
       self.child = child
       self.location = r
+      self.trivia = trivia
     }
 
     @frozen

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -332,29 +332,42 @@ extension Source {
   ///     Quantifier -> ('*' | '+' | '?' | '{' Range '}') QuantKind?
   ///     QuantKind  -> '?' | '+'
   ///
-  mutating func lexQuantifier() throws -> (
-    Located<Quant.Amount>, Located<Quant.Kind>
-  )? {
+  mutating func lexQuantifier(
+    context: ParsingContext
+  ) throws -> (Located<Quant.Amount>, Located<Quant.Kind>, [AST.Trivia])? {
+    var trivia: [AST.Trivia] = []
+
+    if let t = try lexNonSemanticWhitespace(context: context) {
+      trivia.append(t)
+    }
+
     let amt: Located<Quant.Amount>? = try recordLoc { src in
       if src.tryEat("*") { return .zeroOrMore }
       if src.tryEat("+") { return .oneOrMore }
       if src.tryEat("?") { return .zeroOrOne }
 
       return try src.tryEating { src in
-        guard src.tryEat("{"), let range = try src.lexRange(), src.tryEat("}")
+        guard src.tryEat("{"),
+              let range = try src.lexRange(context: context),
+              src.tryEat("}")
         else { return nil }
         return range.value
       }
     }
     guard let amt = amt else { return nil }
 
+    // PCRE allows non-semantic whitespace here in extended syntax mode.
+    if let t = try lexNonSemanticWhitespace(context: context) {
+      trivia.append(t)
+    }
+
     let kind: Located<Quant.Kind> = recordLoc { src in
       if src.tryEat("?") { return .reluctant  }
       if src.tryEat("+") { return .possessive }
       return .eager
     }
 
-    return (amt, kind)
+    return (amt, kind, trivia)
   }
 
   /// Try to consume a range, returning `nil` if unsuccessful.
@@ -363,7 +376,7 @@ extension Source {
   ///                  | ExpRange
   ///     ExpRange    -> '..<' <Int> | '...' <Int>
   ///                  | <Int> '..<' <Int> | <Int> '...' <Int>?
-  mutating func lexRange() throws -> Located<Quant.Amount>? {
+  mutating func lexRange(context: ParsingContext) throws -> Located<Quant.Amount>? {
     try recordLoc { src in
       try src.tryEating { src in
         let lowerOpt = try src.lexNumber()
@@ -375,7 +388,7 @@ extension Source {
         let closedRange: Bool?
         if src.tryEat(",") {
           closedRange = true
-        } else if src.experimentalRanges && src.tryEat(".") {
+        } else if context.experimentalRanges && src.tryEat(".") {
           try src.expect(".")
           if src.tryEat(".") {
             closedRange = true
@@ -477,12 +490,12 @@ extension Source {
   ///
   /// TODO: Need to support some escapes
   ///
-  mutating func lexQuote() throws -> AST.Quote? {
+  mutating func lexQuote(context: ParsingContext) throws -> AST.Quote? {
     let str = try recordLoc { src -> String? in
       if src.tryEat(sequence: #"\Q"#) {
         return try src.expectQuoted(endingWith: #"\E"#).value
       }
-      if src.experimentalQuotes, src.tryEat("\"") {
+      if context.experimentalQuotes, src.tryEat("\"") {
         return try src.expectQuoted(endingWith: "\"", ignoreEscaped: true).value
       }
       return nil
@@ -499,16 +512,27 @@ extension Source {
   ///
   ///     ExpComment -> '/*' (!'*/' .)* '*/'
   ///
+  /// With `SyntaxOptions.endOfLineComments`
+  ///
+  ///     EndOfLineComment -> '#' .*
+  ///
   /// TODO: Swift-style nested comments, line-ending comments, etc
   ///
-  mutating func lexComment() throws -> AST.Trivia? {
+  mutating func lexComment(context: ParsingContext) throws -> AST.Trivia? {
     let trivia: Located<String>? = try recordLoc { src in
       if src.tryEat(sequence: "(?#") {
         return try src.expectQuoted(endingWith: ")").value
       }
-      if src.experimentalComments, src.tryEat(sequence: "/*") {
+      if context.experimentalComments, src.tryEat(sequence: "/*") {
         return try src.expectQuoted(endingWith: "*/").value
       }
+      if context.endOfLineComments, src.tryEat("#") {
+        // TODO: If we ever support multi-line regex literals, this will need
+        // to be updated to stop at a newline. Note though that PCRE specifies
+        // that the newline it matches against can be controlled by the global
+        // matching options e.g `(*CR)`, `(*ANY)`, ...
+        return src.lexUntil(\.isEmpty).value
+      }
       return nil
     }
     guard let trivia = trivia else { return nil }
@@ -517,16 +541,55 @@ extension Source {
 
   /// Try to consume non-semantic whitespace as trivia
   ///
+  ///     Whitespace -> WhitespaceChar+
+  ///
   /// Does nothing unless `SyntaxOptions.nonSemanticWhitespace` is set
-  mutating func lexNonSemanticWhitespace() throws -> AST.Trivia? {
-    guard syntax.ignoreWhitespace else { return nil }
+  mutating func lexNonSemanticWhitespace(
+    context: ParsingContext
+  ) throws -> AST.Trivia? {
+    guard context.ignoreWhitespace else { return nil }
+
+    func isWhitespace(_ c: Character) -> Bool {
+      // This is a list of characters that PCRE treats as whitespace when
+      // compiled with Unicode support. It is a subset of the characters with
+      // the `.isWhitespace` property. ICU appears to also follow this list.
+      // Oniguruma and .NET follow a subset of this list.
+      //
+      // FIXME: PCRE only treats space and tab characters as whitespace when
+      // inside a custom character class (and only treats whitespace as
+      // non-semantic there for the extra-extended `(?xx)` mode). If we get a
+      // strict-PCRE mode, we'll need to add a case for that.
+      switch c {
+      case " ", "\u{9}"..."\u{D}", // space, \t, \n, vertical tab, \f, \r
+           "\u{85}", "\u{200E}",   // next line, left-to-right mark
+           "\u{200F}", "\u{2028}", // right-to-left-mark, line separator
+           "\u{2029}":             // paragraph separator
+        return true
+      default:
+        return false
+      }
+    }
     let trivia: Located<String>? = recordLoc { src in
-      src.tryEatPrefix { $0 == " " }?.string
+      src.tryEatPrefix(isWhitespace)?.string
     }
     guard let trivia = trivia else { return nil }
     return AST.Trivia(trivia)
   }
 
+  /// Try to consume trivia.
+  ///
+  ///     Trivia -> Comment | Whitespace
+  ///
+  mutating func lexTrivia(context: ParsingContext) throws -> AST.Trivia? {
+    if let comment = try lexComment(context: context) {
+      return comment
+    }
+    if let whitespace = try lexNonSemanticWhitespace(context: context) {
+      return whitespace
+    }
+    return nil
+  }
+
   /// Try to lex a matching option.
   ///
   ///     MatchingOption -> 'i' | 'J' | 'm' | 'n' | 's' | 'U' | 'x' | 'xx' | 'w'
@@ -761,6 +824,7 @@ extension Source {
   /// comments, like quotes, cannot be quantified.
   ///
   mutating func lexGroupStart(
+    context: ParsingContext
   ) throws -> Located<AST.Group.Kind>? {
     try recordLoc { src in
       try src.tryEating { src in
@@ -825,7 +889,7 @@ extension Source {
         }
 
         // (_:)
-        if src.experimentalCaptures && src.tryEat(sequence: "_:") {
+        if context.experimentalCaptures && src.tryEat(sequence: "_:") {
           return .nonCapture
         }
         // TODO: (name:)
@@ -960,9 +1024,12 @@ extension Source {
   ///
   ///     GroupConditionalStart -> '(?' GroupStart
   ///
-  mutating func lexGroupConditionalStart() throws -> Located<AST.Group.Kind>? {
+  mutating func lexGroupConditionalStart(
+    context: ParsingContext
+  ) throws -> Located<AST.Group.Kind>? {
     try tryEating { src in
-      guard src.tryEat(sequence: "(?"), let group = try src.lexGroupStart()
+      guard src.tryEat(sequence: "(?"),
+            let group = try src.lexGroupStart(context: context)
       else { return nil }
 
       // Implicitly scoped groups are not supported here.
@@ -1607,7 +1674,7 @@ extension Source {
       var name: Located<String>?
       if src.tryEat(":") {
         // TODO: PCRE allows escaped delimiters or '\Q...\E' sequences in the
-        // name under PCRE2_ALT_VERBNAMES.
+        // name under PCRE2_ALT_VERBNAMES. It also allows whitespace under (?x).
         name = try src.expectQuoted(endingWith: ")", eatEnding: false)
       }
       try src.expect(")")