Skip to content

Commit ed2ed90

Browse files
rxweirxwei
committed
Add DSL support for backreferences.
Allow `capture` and `tryCapture` to assign the captured value to a `Reference`, which can be used as a regex later in the scope. `Reference` initialization creates a unique identifier. The compiler converts the identifier to an absolute backreference offset. ----- Example: ```swift let regex = Regex { let a = Reference() let b = Reference() capture("abc", as: a) capture("def", as: b) a capture(b) } ```
1 parent 5e2b77c commit ed2ed90

File tree

10 files changed

+505
-166
lines changed

10 files changed

+505
-166
lines changed

‎Sources/VariadicsGenerator/VariadicsGenerator.swift‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -538,52 +538,65 @@ struct VariadicsGenerator: ParsableCommand {
538538
output("""
539539
// MARK: - Non-builder capture arity \(arity)
540540
541-
public func capture<\(genericParams)>(_ component: R) -> \(regexTypeName)<\(newMatchType(newCaptureType: "W"))> \(whereClause) {
542-
.init(node: .group(.capture, component.regex.root))
541+
public func capture<\(genericParams)>(
542+
_ component: R, as reference: Reference? = nil
543+
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "W"))> \(whereClause) {
544+
.init(node: .group(.capture, component.regex.root, reference?.id))
543545
}
544546
545547
public func capture<\(genericParams), NewCapture>(
546-
_ component: R, transform: @escaping (Substring) -> NewCapture
548+
_ component: R,
549+
as reference: Reference? = nil,
550+
transform: @escaping (Substring) -> NewCapture
547551
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "NewCapture"))> \(whereClause) {
548552
.init(node: .groupTransform(
549553
.capture,
550554
component.regex.root,
551555
CaptureTransform(resultType: NewCapture.self) {
552556
transform($0) as Any
553-
}))
557+
},
558+
reference?.id))
554559
}
555560
556561
public func tryCapture<\(genericParams), NewCapture>(
557-
_ component: R, transform: @escaping (Substring) throws -> NewCapture
562+
_ component: R,
563+
as reference: Reference? = nil,
564+
transform: @escaping (Substring) throws -> NewCapture
558565
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "NewCapture"))> \(whereClause) {
559566
.init(node: .groupTransform(
560567
.capture,
561568
component.regex.root,
562569
CaptureTransform(resultType: NewCapture.self) {
563570
try transform($0) as Any
564-
}))
571+
},
572+
reference?.id))
565573
}
566574
567575
public func tryCapture<\(genericParams), NewCapture>(
568-
_ component: R, transform: @escaping (Substring) -> NewCapture?
576+
_ component: R,
577+
as reference: Reference? = nil,
578+
transform: @escaping (Substring) -> NewCapture?
569579
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "NewCapture"))> \(whereClause) {
570580
.init(node: .groupTransform(
571581
.capture,
572582
component.regex.root,
573583
CaptureTransform(resultType: NewCapture.self) {
574584
transform($0) as Any?
575-
}))
585+
},
586+
reference?.id))
576587
}
577588
578589
// MARK: - Builder capture arity \(arity)
579590
580591
public func capture<\(genericParams)>(
592+
as reference: Reference? = nil,
581593
@RegexBuilder _ component: () -> R
582594
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "W"))> \(whereClause) {
583-
.init(node: .group(.capture, component().regex.root))
595+
.init(node: .group(.capture, component().regex.root, reference?.id))
584596
}
585597
586598
public func capture<\(genericParams), NewCapture>(
599+
as reference: Reference? = nil,
587600
@RegexBuilder _ component: () -> R,
588601
transform: @escaping (Substring) -> NewCapture
589602
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "NewCapture"))> \(whereClause) {
@@ -592,10 +605,12 @@ struct VariadicsGenerator: ParsableCommand {
592605
component().regex.root,
593606
CaptureTransform(resultType: NewCapture.self) {
594607
transform($0) as Any
595-
}))
608+
},
609+
reference?.id))
596610
}
597611
598612
public func tryCapture<\(genericParams), NewCapture>(
613+
as reference: Reference? = nil,
599614
@RegexBuilder _ component: () -> R,
600615
transform: @escaping (Substring) throws -> NewCapture
601616
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "NewCapture"))> \(whereClause) {
@@ -604,10 +619,12 @@ struct VariadicsGenerator: ParsableCommand {
604619
component().regex.root,
605620
CaptureTransform(resultType: NewCapture.self) {
606621
try transform($0) as Any
607-
}))
622+
},
623+
reference?.id))
608624
}
609625
610626
public func tryCapture<\(genericParams), NewCapture>(
627+
as reference: Reference? = nil,
611628
@RegexBuilder _ component: () -> R,
612629
transform: @escaping (Substring) -> NewCapture?
613630
) -> \(regexTypeName)<\(newMatchType(newCaptureType: "NewCapture"))> \(whereClause) {
@@ -616,7 +633,8 @@ struct VariadicsGenerator: ParsableCommand {
616633
component().regex.root,
617634
CaptureTransform(resultType: NewCapture.self) {
618635
transform($0) as Any?
619-
}))
636+
},
637+
reference?.id))
620638
}
621639
622640
""")

‎Sources/_StringProcessing/ByteCodeGen.swift‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ extension Compiler.ByteCodeGen {
3131
case let .backreference(ref):
3232
try emitBackreference(ref)
3333

34+
case let .symbolicReference(id):
35+
builder.buildUnresolvedReference(id: id)
36+
3437
case let .unconverted(astAtom):
3538
if let consumer = try astAtom.generateConsumer(options) {
3639
builder.buildConsume(by: consumer)
@@ -293,8 +296,14 @@ extension Compiler.ByteCodeGen {
293296
}
294297

295298
mutating func emitGroup(
296-
_ kind: AST.Group.Kind, _ child: DSLTree.Node
299+
_ kind: AST.Group.Kind,
300+
_ child: DSLTree.Node,
301+
_ referenceID: Reference.ID?
297302
) throws -> CaptureRegister? {
303+
guard kind.isCapturing || referenceID == nil else {
304+
throw Unreachable("Reference ID shouldn't exist for non-capturing groups")
305+
}
306+
298307
options.beginScope()
299308
defer { options.endScope() }
300309

@@ -303,7 +312,7 @@ extension Compiler.ByteCodeGen {
303312
//
304313
// FIXME: Unify with .groupTransform
305314
if kind.isCapturing, case let .matcher(_, m) = child {
306-
let cap = builder.makeCapture()
315+
let cap = builder.makeCapture(id: referenceID)
307316
emitMatcher(m, into: cap)
308317
return cap
309318
}
@@ -319,7 +328,7 @@ extension Compiler.ByteCodeGen {
319328
throw Unreachable("TODO: reason")
320329

321330
case .capture, .namedCapture:
322-
let cap = builder.makeCapture()
331+
let cap = builder.makeCapture(id: referenceID)
323332
builder.buildBeginCapture(cap)
324333
try emitNode(child)
325334
builder.buildEndCapture(cap)
@@ -552,8 +561,8 @@ extension Compiler.ByteCodeGen {
552561
try emitConcatenationComponent(child)
553562
}
554563

555-
case let .group(kind, child):
556-
_ = try emitGroup(kind, child)
564+
case let .group(kind, child, referenceID):
565+
_ = try emitGroup(kind, child, referenceID)
557566

558567
case .conditional:
559568
throw Unsupported("Conditionals")
@@ -592,8 +601,8 @@ extension Compiler.ByteCodeGen {
592601
case let .convertedRegexLiteral(n, _):
593602
try emitNode(n)
594603

595-
case let .groupTransform(kind, child, t):
596-
guard let cap = try emitGroup(kind, child) else {
604+
case let .groupTransform(kind, child, t, referenceID):
605+
guard let cap = try emitGroup(kind, child, referenceID) else {
597606
assertionFailure("""
598607
What does it mean to not have a capture to transform?
599608
""")

‎Sources/_StringProcessing/Compiler.swift‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,15 @@ public func _compileRegex(
4343
return Executor(program: program)
4444
}
4545

46+
// An error produced when compiling a regular expression.
47+
public enum RegexCompilationError: Error, CustomStringConvertible {
48+
// TODO: Source location?
49+
case uncapturedReference
50+
51+
public var description: String {
52+
switch self {
53+
case .uncapturedReference:
54+
return "Found a reference used before it captured any match."
55+
}
56+
}
57+
}

‎Sources/_StringProcessing/ConsumerInterface.swift‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ extension DSLTree.Atom {
9393
// TODO: Should we handle?
9494
return nil
9595

96+
case .symbolicReference:
97+
// TODO: Should we handle?
98+
return nil
99+
96100
case let .unconverted(a):
97101
return try a.generateConsumer(opts)
98102
}

‎Sources/_StringProcessing/Engine/MEBuilder.swift‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ extension MEProgram where Input.Element: Hashable {
4242
// as we compile?
4343
var captureStructure: CaptureStructure = .empty
4444

45+
// Symbolic reference resolution
46+
var unresolvedReferences: [Reference.ID: [InstructionAddress]] = [:]
47+
var captureOffsets: [Reference.ID: Int] = [:]
48+
var captureCount: Int {
49+
// We currently deduce the capture count from the capture register number.
50+
nextCaptureRegister.rawValue
51+
}
52+
4553
public init() {}
4654
}
4755
}
@@ -67,6 +75,10 @@ extension MEProgram.Builder {
6775
staticElements.forEach { elements.store($0) }
6876
}
6977

78+
var lastInstructionAddress: InstructionAddress {
79+
.init(instructions.endIndex - 1)
80+
}
81+
7082
public mutating func buildNop(_ r: StringRegister? = nil) {
7183
instructions.append(.init(.nop, .init(optionalString: r)))
7284
}
@@ -262,9 +274,16 @@ extension MEProgram.Builder {
262274
.init(.backreference, .init(capture: cap)))
263275
}
264276

277+
public mutating func buildUnresolvedReference(id: Reference.ID) {
278+
buildBackreference(.init(0))
279+
unresolvedReferences[id, default: []].append(lastInstructionAddress)
280+
}
281+
265282
// TODO: Mutating because of fail address fixup, drop when
266283
// that's removed
267284
public mutating func assemble() throws -> MEProgram {
285+
try resolveReferences()
286+
268287
// TODO: This will add a fail instruction at the end every
269288
// time it's assembled. Better to do to the local instruction
270289
// list copy, but that complicates logic. It's possible we
@@ -401,10 +420,31 @@ extension MEProgram.Builder {
401420

402421
}
403422

423+
// Symbolic reference helpers
424+
fileprivate extension MEProgram.Builder {
425+
mutating func resolveReferences() throws {
426+
for (id, uses) in unresolvedReferences {
427+
guard let offset = captureOffsets[id] else {
428+
throw RegexCompilationError.uncapturedReference
429+
}
430+
for use in uses {
431+
instructions[use.rawValue] =
432+
Instruction(.backreference, .init(capture: .init(offset)))
433+
}
434+
}
435+
}
436+
}
437+
404438
// Register helpers
405439
extension MEProgram.Builder {
406-
public mutating func makeCapture() -> CaptureRegister {
440+
public mutating func makeCapture(id: Reference.ID?) -> CaptureRegister {
407441
defer { nextCaptureRegister.rawValue += 1 }
442+
// Register the capture for later lookup via symbolic references.
443+
if let id = id {
444+
let preexistingValue = captureOffsets.updateValue(
445+
captureCount, forKey: id)
446+
assert(preexistingValue == nil)
447+
}
408448
return nextCaptureRegister
409449
}
410450

‎Sources/_StringProcessing/PrintAsPattern.swift‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,10 @@ extension PrettyPrinter {
8787
}
8888
}
8989

90-
case let .group(kind, child):
90+
case let .group(kind, child, referenceID):
9191
let kind = kind._patternBase
92-
printBlock("Group(\(kind))") { printer in
92+
let refIDString = referenceID.map { ", referenceID: \($0)" } ?? ""
93+
printBlock("Group(\(kind)\(refIDString)") { printer in
9394
printer.printAsPattern(convertedFromAST: child)
9495
}
9596

@@ -123,7 +124,9 @@ extension PrettyPrinter {
123124
case .assertion:
124125
print("/* TODO: assertions */")
125126
case .backreference:
126-
print("/* TODO: backreferences */")
127+
print("/* TOOD: backreferences */")
128+
case .symbolicReference:
129+
print("/* TOOD: symbolic references */")
127130
}
128131

129132
case .trivia:

‎Sources/_StringProcessing/RegexDSL/DSL.swift‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,3 +184,25 @@ public func choiceOf<R: RegexProtocol>(
184184
) -> R {
185185
builder()
186186
}
187+
188+
// MARK: - Backreference
189+
190+
public struct Reference: RegexProtocol {
191+
// FIXME: Public for prototypes.
192+
public struct ID: Hashable, Equatable {
193+
private static var counter: Int = 0
194+
var base: Int
195+
init() {
196+
base = ID.counter
197+
ID.counter += 1
198+
}
199+
}
200+
201+
let id = ID()
202+
203+
public init() {}
204+
205+
public var regex: Regex<Substring> {
206+
.init(node: .atom(.symbolicReference(id)))
207+
}
208+
}

‎Sources/_StringProcessing/RegexDSL/DSLTree.swift‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ extension DSLTree {
3030
case concatenation([Node])
3131

3232
/// (...)
33-
case group(AST.Group.Kind, Node)
33+
case group(AST.Group.Kind, Node, Reference.ID? = nil)
3434

3535
/// (?(cond) true-branch | false-branch)
3636
///
@@ -81,7 +81,8 @@ extension DSLTree {
8181
case groupTransform(
8282
AST.Group.Kind,
8383
Node,
84-
CaptureTransform)
84+
CaptureTransform,
85+
Reference.ID? = nil)
8586

8687
case consumer(_ConsumerInterface)
8788

@@ -119,6 +120,7 @@ extension DSLTree {
119120

120121
case assertion(AST.Atom.AssertionKind)
121122
case backreference(AST.Reference)
123+
case symbolicReference(Reference.ID)
122124

123125
case unconverted(AST.Atom)
124126
}
@@ -162,8 +164,8 @@ extension DSLTree.Node {
162164
// Treat this transparently
163165
return n.children
164166

165-
case let .group(_, n): return [n]
166-
case let .groupTransform(_, n, _): return [n]
167+
case let .group(_, n, _): return [n]
168+
case let .groupTransform(_, n, _, _): return [n]
167169
case let .quantification(_, _, n): return [n]
168170

169171
case let .conditional(_, t, f): return [t,f]
@@ -224,8 +226,8 @@ extension DSLTree {
224226
extension DSLTree.Node {
225227
var hasCapture: Bool {
226228
switch self {
227-
case let .group(k, _) where k.isCapturing,
228-
let .groupTransform(k, _, _) where k.isCapturing:
229+
case let .group(k, _, _) where k.isCapturing,
230+
let .groupTransform(k, _, _, _) where k.isCapturing:
229231
return true
230232
case let .convertedRegexLiteral(n, re):
231233
assert(n.hasCapture == re.hasCapture)
@@ -257,14 +259,14 @@ extension DSLTree.Node {
257259
case let .concatenation(children):
258260
return constructor.concatenating(children)
259261

260-
case let .group(kind, child):
262+
case let .group(kind, child, _):
261263
if let type = child.matcherCaptureType {
262264
return constructor.grouping(
263265
child, as: kind, withType: type)
264266
}
265267
return constructor.grouping(child, as: kind)
266268

267-
case let .groupTransform(kind, child, transform):
269+
case let .groupTransform(kind, child, transform, _):
268270
return constructor.grouping(
269271
child, as: kind, withTransform: transform)
270272

0 commit comments

Comments
 (0)