Pulls token scanning out of tokeniser and into separate scanning class and adds performance log class

Sources/SwiftyMarkdown/CharacterRule.swift

@@ -0,0 +1,59 @@
+//
+//  CharacterRule.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+
+public enum SpaceAllowed {
+	case no
+	case bothSides
+	case oneSide
+	case leadingSide
+	case trailingSide
+}
+
+public enum Cancel {
+	case none
+	case allRemaining
+	case currentSet
+}
+
+public struct CharacterRule : CustomStringConvertible {
+	public let openTag : String
+	public let intermediateTag : String?
+	public let closingTag : String?
+	public let escapeCharacter : Character?
+	public let styles : [Int : [CharacterStyling]]
+	public var minTags : Int = 1
+	public var maxTags : Int = 1
+	public var spacesAllowed : SpaceAllowed = .oneSide
+	public var cancels : Cancel = .none
+	public var metadataLookup : Bool = false
+	
+	public var tagVarieties : [Int : String]
+	
+	public var description: String {
+		return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
+	}
+	
+	public init(openTag: String, intermediateTag: String? = nil, closingTag: String? = nil, escapeCharacter: Character? = nil, styles: [Int : [CharacterStyling]] = [:], minTags : Int = 1, maxTags : Int = 1, cancels : Cancel = .none, metadataLookup : Bool = false) {
+		self.openTag = openTag
+		self.intermediateTag = intermediateTag
+		self.closingTag = closingTag
+		self.escapeCharacter = escapeCharacter
+		self.styles = styles
+		self.minTags = minTags
+		self.maxTags = maxTags
+		self.cancels = cancels
+		self.metadataLookup = metadataLookup
+		
+		self.tagVarieties = [:]
+		for i in minTags...maxTags {
+			self.tagVarieties[i] = openTag.repeating(i)
+		}
+	}
+}
+

Sources/SwiftyMarkdown/PerfomanceLog.swift

@@ -0,0 +1,44 @@
+//
+//  PerfomanceLog.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+import os.log
+
+class PerformanceLog {
+	var timer : TimeInterval = 0
+	let enablePerfomanceLog : Bool
+	let log : OSLog
+	let identifier : String
+	
+	init( with environmentVariableName : String, identifier : String, log : OSLog  ) {
+		self.log = log
+		self.enablePerfomanceLog = (ProcessInfo.processInfo.environment[environmentVariableName] != nil)
+		self.identifier = identifier
+	}
+	
+	func start() {
+		guard enablePerfomanceLog else { return }
+		self.timer = Date().timeIntervalSinceReferenceDate
+		os_log("--- TIMER %{public}@ began", log: self.log, type: .info, self.identifier)
+	}
+	
+	func tag( with string : String) {
+		guard enablePerfomanceLog else { return }
+		if timer == 0 {
+			self.start()
+		}
+		os_log("TIMER %{public}@: %f %@", log: self.log, type: .info, self.identifier, Date().timeIntervalSinceReferenceDate - self.timer, string)
+	}
+	
+	func end() {
+		guard enablePerfomanceLog else { return }
+		self.timer = Date().timeIntervalSinceReferenceDate
+		os_log("--- TIMER %{public}@ finished. Total time: %f", log: self.log, type: .info, self.identifier, Date().timeIntervalSinceReferenceDate - self.timer)
+		self.timer = 0
+
+	}
+}

Sources/SwiftyMarkdown/SwiftyLineProcessor.swift

@@ -80,9 +80,8 @@ public class SwiftyLineProcessor {
     let lineRules : [LineRule]
 	let frontMatterRules : [FrontMatterRule]
 	
-	var timer : TimeInterval = 0
-	var enablePerformanceLog = (ProcessInfo.processInfo.environment["SwiftyLineProcessorPerformanceLogging"] != nil)
-    
+	let perfomanceLog = PerformanceLog(with: "SwiftyLineProcessorPerformanceLogging", identifier: "Line Processor", log: OSLog.swiftyLineProcessorPerformance)
+	    
 	public init( rules : [LineRule], defaultRule: LineStyling, frontMatterRules : [FrontMatterRule] = []) {
         self.lineRules = rules
         self.defaultType = defaultRule
@@ -213,17 +212,12 @@ public class SwiftyLineProcessor {
         var foundAttributes : [SwiftyLine] = []
 		
 		
-		if self.enablePerformanceLog {
-			self.timer = Date().timeIntervalSinceReferenceDate
-			os_log("TIMER (began)                 : 0", log: .swiftyMarkdownPerformance, type: .info)
-		}
+		self.perfomanceLog.start()
 		
 		var lines = string.components(separatedBy: CharacterSet.newlines)
 		lines = self.processFrontMatter(lines)
 		
-		if self.enablePerformanceLog {
-			os_log("TIMER (front matter completed): %f", log: .swiftyMarkdownPerformance, type: .info, Date().timeIntervalSinceReferenceDate - self.timer)
-		}
+		self.perfomanceLog.tag(with: "(Front matter completed)")
 		
 
         for  heading in lines {
@@ -245,9 +239,7 @@ public class SwiftyLineProcessor {
             }
             foundAttributes.append(input)
 			
-			if self.enablePerformanceLog {
-				os_log("TIMER (line complete)         : %f (%@)", log: .swiftyMarkdownPerformance, type: .info, Date().timeIntervalSinceReferenceDate - self.timer, heading)
-			}
+			self.perfomanceLog.tag(with: "(line completed: \(heading)")
         }
         return foundAttributes
     }

Sources/SwiftyMarkdown/SwiftyMarkdown+CharacterRule.swift

@@ -0,0 +1,26 @@
+//
+//  SwiftyMarkdown+CharacterRule.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+
+public extension CharacterRule {
+	init( openTag : String, intermediateTag : String, closingTag : String, escapeCharacter: Character? = nil, styles: [Int : [CharacterStyling]] = [:], metadataLookup : Bool ) {
+		self.openTag = openTag
+		self.intermediateTag = intermediateTag
+		self.closingTag = closingTag
+		self.escapeCharacter = escapeCharacter
+		self.styles = styles
+		self.minTags = 1
+		self.maxTags = 1
+		self.cancels = .none
+		self.metadataLookup = metadataLookup
+		self.tagVarieties = [:]
+		for i in minTags...maxTags {
+			self.tagVarieties[i] = openTag.repeating(i)
+		}
+	}
+}

Sources/SwiftyMarkdown/SwiftyMarkdown.swift

@@ -24,6 +24,8 @@ enum CharacterStyle : CharacterStyling {
 	case code
 	case link
 	case image
+	case referencedLink
+	case referencedImage
 	case strikethrough
 	
 	func isEqualTo(_ other: CharacterStyling) -> Bool {
@@ -166,7 +168,8 @@ If that is not set, then the system default will be used.
 	]
 	
 	static public var characterRules = [
-		CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]], maxTags: 1),
+		CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], metadataLookup: true),
+		CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]], metadataLookup: false),
 		CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1),
 		CharacterRule(openTag: "`", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.code]], maxTags: 1, cancels: .allRemaining),
 		CharacterRule(openTag: "~", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [2 : [CharacterStyle.strikethrough]], minTags: 2, maxTags: 2),
@@ -175,7 +178,7 @@ If that is not set, then the system default will be used.
 	]
 	
 	let lineProcessor = SwiftyLineProcessor(rules: SwiftyMarkdown.lineRules, defaultRule: MarkdownLineStyle.body, frontMatterRules: SwiftyMarkdown.frontMatterRules)
-	let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
+	let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules, scanner: SwiftyScanner())
 	
 	/// The styles to apply to any H1 headers found in the Markdown
 	open var h1 = LineStyles()
@@ -236,9 +239,8 @@ If that is not set, then the system default will be used.
 	var orderedListIndentFirstOrderCount = 0
 	var orderedListIndentSecondOrderCount = 0
 	
-	var timer : TimeInterval = 0
-	var enablePerformanceLog = (ProcessInfo.processInfo.environment["SwiftyMarkdownPerformanceLogging"] != nil)
-	
+	let perfomanceLog = PerformanceLog(with: "SwiftyMarkdownPerformanceLogging", identifier: "Swifty Markdown", log: .swiftyMarkdownPerformance)
+		
 	/**
 	
 	- parameter string: A string containing [Markdown](https://daringfireball.net/projects/markdown/) syntax to be converted to an NSAttributedString
@@ -360,36 +362,99 @@ If that is not set, then the system default will be used.
 	*/
 	open func attributedString(from markdownString : String? = nil) -> NSAttributedString {
 		
-		if enablePerformanceLog {
-			self.timer = Date().timeIntervalSinceReferenceDate
-			os_log("--- TIMER (began): 0", log: .swiftyMarkdownPerformance, type: .info)
-		}
+		self.perfomanceLog.start()
+		
 		if let existentMarkdownString = markdownString {
 			self.string = existentMarkdownString
 		}
 		let attributedString = NSMutableAttributedString(string: "")
 		self.lineProcessor.processEmptyStrings = MarkdownLineStyle.body
 		let foundAttributes : [SwiftyLine] = lineProcessor.process(self.string)
-
-		if enablePerformanceLog {
-			os_log("----- TIMER (line processing complete       ): %f", log: .swiftyMarkdownPerformance, type: .info, Date().timeIntervalSinceReferenceDate - self.timer)
+		
+		let references : [SwiftyLine] = foundAttributes.filter({ $0.line.starts(with: "[") && $0.line.contains("]:") })
+		let referencesRemoved : [SwiftyLine] = foundAttributes.filter({ !($0.line.starts(with: "[") && $0.line.contains("]:") ) })
+		var keyValuePairs : [String : String] = [:]
+		for line in references {
+			let strings = line.line.components(separatedBy: "]:")
+			guard strings.count >= 2 else {
+				continue
+			}
+			var key : String = strings[0]
+			if !key.isEmpty {
+				let newstart = key.index(key.startIndex, offsetBy: 1)
+				let range : Range<String.Index> = newstart..<key.endIndex
+				key = String(key[range]).trimmingCharacters(in: .whitespacesAndNewlines)
+			}
+			keyValuePairs[key] = strings[1].trimmingCharacters(in: .whitespacesAndNewlines)
 		}
 		
-		for (idx, line) in foundAttributes.enumerated() {
+		self.perfomanceLog.tag(with: "(line processing complete)")
+		
+		self.tokeniser.scanner.metadataLookup = keyValuePairs
+		
+		for (idx, line) in referencesRemoved.enumerated() {
 			if idx > 0 {
 				attributedString.append(NSAttributedString(string: "\n"))
 			}
-			let finalTokens = self.tokeniser.process(line.line)
-			if enablePerformanceLog {
-				os_log("TIMER (tokenising complete for line %i): %f", log: .swiftyMarkdownPerformance, type: .info, idx, Date().timeIntervalSinceReferenceDate - self.timer)
+			let originalTokens = self.tokeniser.process(line.line)
+			var finalTokens : [Token] = []
+			if line.line.contains("][") {
+				var tokenBag : [Token] = []
+				var addToBag = false
+				for token in originalTokens {
+					print(token)
+					if token.type == .closeTag  {
+						if token.inputString != "]" {
+							finalTokens.append(contentsOf: tokenBag)
+							tokenBag.removeAll()
+							addToBag = false
+						} else {
+							tokenBag.append(token)
+							guard let metadata = tokenBag.filter({ $0.isMetadata }).first else {
+								finalTokens.append(contentsOf: tokenBag)
+								tokenBag.removeAll()
+								addToBag = false
+								continue
+							}
+							if let hasLookup = keyValuePairs[metadata.inputString] {
+								for var token in tokenBag {
+									if token.type == .string {
+										token.metadataString = hasLookup
+									}
+									
+									finalTokens.append(token)
+								}
+							} else {
+								for token in tokenBag {
+									let stringToken = Token(type: .string, inputString: token.outputString)
+									finalTokens.append(stringToken)
+								}
+							}
+							tokenBag.removeAll()
+							addToBag = false
+							continue
+						}
+					}
+					
+					
+					if token.type == .openTag && token.inputString == "[" {
+						addToBag = true
+					}
+					if addToBag {
+						tokenBag.append(token)
+					} else {
+						finalTokens.append(token)
+					}
+				}
 			}
+			
+			self.perfomanceLog.tag(with: "(tokenising complete for line \(idx)")
+			
 			attributedString.append(attributedStringFor(tokens: finalTokens, in: line))
 			
 		}
 		
-		if enablePerformanceLog {
-			os_log("----- TIMER (processing complete            ): %f", log: .swiftyMarkdownPerformance, type: .info, Date().timeIntervalSinceReferenceDate - self.timer)
-		}
+		self.perfomanceLog.end()
 		
 		return attributedString
 	}

Sources/SwiftyMarkdown/SwiftyScanner.swift

@@ -0,0 +1,555 @@
+//
+//  SwiftyScanner.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+import os.log
+
+extension OSLog {
+	private static var subsystem = "SwiftyScanner"
+	static let swiftyScannerTokenising = OSLog(subsystem: subsystem, category: "Swifty Scanner Tokenising")
+	static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Peformance")
+}
+
+/// Swifty Scanning Protocol
+public protocol SwiftyScanning {
+	var metadataLookup : [String : String] { get set }
+	func scan( _ string : String, with rule : CharacterRule) -> [Token]
+}
+
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+class SwiftyScanner : SwiftyScanning {
+	var state : TagState = .none
+	var preOpenString = ""
+	var openTagString : [String] = []
+	var intermediateString = ""
+	var intermediateTagString = ""
+	var metadataString = ""
+	var closedTagString : [String] = []
+	var postClosedString = ""
+	
+	var rule : CharacterRule! = nil
+	var tokenGroup = 0
+	
+	var metadataLookup : [String : String] = [:]
+	
+	let performanceLog = PerformanceLog(with: "SwiftyScannerPerformanceLogging", identifier: "Swifty Scanner", log: .swiftyScannerPerformance)
+		
+	init() { }
+	
+	func append( _ string : String? ) {
+		guard let existentString = string else {
+			return
+		}
+		switch self.state {
+		case .none:
+			self.preOpenString += existentString
+		case .open:
+			self.intermediateString += existentString
+		case .intermediate:
+			self.metadataString += existentString
+		case .closed:
+			self.postClosedString += existentString
+		}
+	}
+	
+	func handleRepeatingTags( _ tokenGroup : [TokenGroup] ) {
+		var availableCount = self.rule.maxTags
+		var sameOpenGroup = false
+		for token in tokenGroup {
+			
+			switch token.state {
+			case .none:
+				self.append(token.string)
+				if self.state == .closed {
+					self.state = .none
+				}
+			case .open:
+				switch self.state {
+				case .none:
+					self.openTagString.append(token.string)
+					self.state = .open
+					availableCount = self.rule.maxTags - token.string.count
+					sameOpenGroup = true
+				case .open:
+					if availableCount > 0 {
+						if sameOpenGroup {
+							self.openTagString.append(token.string)
+							availableCount = self.rule.maxTags - token.string.count
+						} else {
+							self.closedTagString.append(token.string)
+							self.state = .closed
+						}
+					} else {
+						self.append(token.string)
+					}
+
+				case .intermediate:
+					self.preOpenString += self.openTagString.joined() + token.string
+				case .closed:
+					self.append(token.string)
+				}
+			case .intermediate:
+				switch self.state {
+				case .none:
+					self.preOpenString += token.string
+				case .open:
+					self.intermediateTagString += token.string
+					self.state = .intermediate
+				case .intermediate:
+					self.metadataString += token.string
+				case .closed:
+					self.postClosedString += token.string
+				}
+				
+			case .closed:
+				switch self.state {
+				case .intermediate:
+					self.closedTagString.append(token.string)
+					self.state = .closed
+				case .closed:
+					self.postClosedString += token.string
+				case .open:
+					if self.rule.intermediateTag == nil {
+						self.closedTagString.append(token.string)
+						self.state = .closed
+					} else {
+						self.preOpenString += self.openTagString.joined()
+						self.preOpenString += self.intermediateString
+						self.preOpenString += token.string
+						self.intermediateString = ""
+						self.openTagString.removeAll()
+					}
+				case .none:
+					self.preOpenString += token.string
+				}
+			}
+		}
+		if !self.openTagString.isEmpty && self.rule.closingTag == nil && self.state != .closed {
+			self.state = .open
+		}
+	}
+	
+	func handleRegularTags( _ tokenGroup : [TokenGroup] ) {
+		for token in tokenGroup {
+			
+			switch token.state {
+			case .none:
+				self.append(token.string)
+				if self.state == .closed {
+					self.state = .none
+				}
+			case .open:
+				switch self.state {
+				case .none:
+					self.openTagString.append(token.string)
+					self.state = .open
+				case .open:
+					if self.rule.maxTags == 1, self.openTagString.first == rule.openTag {
+						self.preOpenString = self.preOpenString + self.openTagString.joined() + self.intermediateString
+						self.intermediateString = ""
+						self.openTagString.removeAll()
+						self.openTagString.append(token.string)
+					} else {
+						self.openTagString.append(token.string)
+					}
+				case .intermediate:
+					self.preOpenString += self.openTagString.joined() + token.string
+				case .closed:
+					self.openTagString.append(token.string)
+				}
+			case .intermediate:
+				switch self.state {
+				case .none:
+					self.preOpenString += token.string
+				case .open:
+					self.intermediateTagString += token.string
+					self.state = .intermediate
+				case .intermediate:
+					self.metadataString += token.string
+				case .closed:
+					self.postClosedString += token.string
+				}
+				
+			case .closed:
+				switch self.state {
+				case .intermediate:
+					self.closedTagString.append(token.string)
+					self.state = .closed
+				case .closed:
+					self.postClosedString += token.string
+				case .open:
+					if self.rule.intermediateTag == nil {
+						self.closedTagString.append(token.string)
+						self.state = .closed
+					} else {
+						self.preOpenString += self.openTagString.joined()
+						self.preOpenString += self.intermediateString
+						self.preOpenString += token.string
+						self.intermediateString = ""
+						self.openTagString.removeAll()
+					}
+				case .none:
+					self.preOpenString += token.string
+				}
+			}
+		}
+		
+	}
+	
+	func append( contentsOf tokenGroup: [TokenGroup] ) {
+		if self.rule.closingTag == nil {
+			self.handleRepeatingTags(tokenGroup)
+		} else {
+			self.handleRegularTags(tokenGroup)
+		}
+	}
+	
+	func configureToken(ofType type : TokenType = .string, with string : String ) -> Token {
+		var token = Token(type: type, inputString: string)
+		token.group = self.tokenGroup
+		return token
+	}
+	
+	func reset() {
+		self.preOpenString = ""
+		self.openTagString.removeAll()
+		self.intermediateString = ""
+		self.intermediateTagString = ""
+		self.metadataString = ""
+		self.closedTagString.removeAll()
+		self.postClosedString = ""
+		
+		self.state = .none
+	}
+	
+	func consolidate(with string : String, into tokens : inout [Token]) -> [Token] {
+		self.reset()
+		guard !string.isEmpty else {
+			return tokens
+		}
+		tokens.append(self.configureToken(with: string))
+		return tokens
+	}
+	
+	func tokens(beginningGroupNumberAt group : Int = 0) -> [Token] {
+		self.tokenGroup = group
+		var tokens : [Token] = []
+		
+		if self.intermediateString.isEmpty && self.intermediateTagString.isEmpty && self.metadataString.isEmpty {
+			let actualString = self.preOpenString + self.openTagString.joined() + self.closedTagString.joined() + self.postClosedString
+			return self.consolidate(with: actualString, into: &tokens)
+		}
+		if self.state == .open && !self.openTagString.isEmpty {
+			let actualString = self.preOpenString + self.openTagString.joined() + self.intermediateString
+			return self.consolidate(with: actualString, into: &tokens)
+		}
+		
+		if !self.preOpenString.isEmpty {
+			tokens.append(self.configureToken(with: self.preOpenString))
+		}
+		
+		for tag in self.openTagString {
+			if self.rule.closingTag == nil {
+				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
+			} else {
+				tokens.append(self.configureToken(ofType: .openTag, with: tag))
+			}
+		}
+		self.tokenGroup += 1
+		if !self.intermediateString.isEmpty {
+			var token = self.configureToken(with: self.intermediateString)
+			token.metadataString = (self.metadataString.isEmpty) ? nil : self.metadataString
+			tokens.append(token)
+		}
+		if !self.intermediateTagString.isEmpty {
+			tokens.append(self.configureToken(ofType: .intermediateTag, with: self.intermediateTagString))
+		}
+		
+		self.tokenGroup += 1
+		
+		if !self.metadataString.isEmpty {
+			var token = Token(type: .string, inputString: self.metadataString)
+			token.group = self.tokenGroup
+			tokens.append(token)
+		}
+		var remainingTags = ( self.rule.closingTag == nil ) ? self.openTagString.joined() : ""
+		for tag in self.closedTagString {
+			if self.rule.closingTag == nil {
+				remainingTags = remainingTags.replacingOccurrences(of: tag, with: "")
+				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
+			} else {
+				tokens.append(self.configureToken(ofType: .closeTag, with: tag))
+			}
+		}
+		if !self.postClosedString.isEmpty {
+			tokens.append(self.configureToken(with: self.postClosedString))
+		}
+		
+		self.reset()
+		
+		if !remainingTags.isEmpty {
+			self.state = .open
+		}
+		
+		return tokens
+	}
+	
+	func scanSpacing( _ scanner : Scanner, usingCharactersIn set : CharacterSet ) -> (preTag : String?, foundChars : String?, postTag : String?) {
+		self.performanceLog.tag(with: "(scan space)")
+		let lastChar : String?
+		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+			lastChar = ( scanner.currentIndex > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
+		} else {
+			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
+				lastChar = ( scanLocation > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanLocation)..<scanLocation]) : nil
+			} else {
+				lastChar = nil
+			}
+			
+		}
+		let maybeFoundChars : String?
+		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+			maybeFoundChars = scanner.scanCharacters(from: set )
+		} else {
+			var string : NSString?
+			scanner.scanCharacters(from: set, into: &string)
+			maybeFoundChars = string as String?
+		}
+		
+		let nextChar : String?
+		if #available(iOS 13.0, OSX 10.15,  watchOS 6.0,tvOS 13.0, *) {
+			nextChar = (scanner.currentIndex != scanner.string.endIndex) ? String(scanner.string[scanner.currentIndex]) : nil
+		} else {
+			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
+				nextChar = (scanLocation != scanner.string.endIndex) ? String(scanner.string[scanLocation]) : nil
+			} else {
+				nextChar = nil
+			}
+		}
+		self.performanceLog.tag(with: "(end space)")
+		
+		return (lastChar, maybeFoundChars, nextChar)
+	}
+	
+	func getTokenGroups( for string : inout String, with rule : CharacterRule, shouldEmpty : Bool = false ) -> [TokenGroup] {
+		if string.isEmpty {
+			return []
+		}
+		var groups : [TokenGroup] 	= []
+		
+		if let closingTag = rule.closingTag, closingTag.contains(string) {
+			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			token.state = .closed
+			groups.append(token)
+			string.removeAll()
+		} else if let intermediateString = rule.intermediateTag, string.contains(intermediateString)  {
+			
+			if let range = string.range(of: intermediateString) {
+				let prior = string[string.startIndex..<range.lowerBound]
+				let tag = string[range]
+				let following = string[range.upperBound..<string.endIndex]
+				if !prior.isEmpty {
+					groups.append(TokenGroup(string: String(prior), isEscaped: false, type: .string))
+				}
+				var token = TokenGroup(string: String(tag), isEscaped: false, type: .tag)
+				token.state = .intermediate
+				groups.append(token)
+				if !following.isEmpty {
+					groups.append(TokenGroup(string: String(following), isEscaped: false, type: .string))
+				}
+				string.removeAll()
+			}
+		} else if string.contains(rule.openTag) {
+			if shouldEmpty || string == rule.tagVarieties[rule.maxTags]{
+				var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+				token.state = .open
+				groups.append(token)
+				string.removeAll()
+			}
+			
+		}
+		
+		if shouldEmpty && !string.isEmpty {
+			let token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			groups.append(token)
+			string.removeAll()
+		}
+		return groups
+	}
+	
+	func scan( _ string : String, with rule : CharacterRule) -> [Token] {
+		
+		self.rule = rule
+		
+		let scanner = Scanner(string: string)
+		scanner.charactersToBeSkipped = nil
+		var tokens : [Token] = []
+		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
+		if let existentEscape = rule.escapeCharacter {
+			set.insert(charactersIn: String(existentEscape))
+		}
+
+		var tokenGroup = 0
+		
+		self.performanceLog.tag(with: "(start scan \(rule.openTag)")
+		
+		if !string.contains( rule.openTag ) {
+			return [Token(type: .string, inputString: string)]
+		}
+		
+		while !scanner.isAtEnd {
+			self.performanceLog.tag(with: "(loop start \(rule.openTag))")
+			tokenGroup += 1
+			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+				if let start = scanner.scanUpToCharacters(from: set) {
+					self.performanceLog.tag(with: "(first chars \(rule.openTag))")
+					self.append(start)
+				}
+			} else {
+				var string : NSString?
+				scanner.scanUpToCharacters(from: set, into: &string)
+				if let existentString = string as String? {
+					self.append(existentString)
+				}
+			}
+			
+			// The end of the string
+			let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
+			guard let foundTag = spacing.foundChars else {
+				continue
+			}
+			
+			if foundTag == rule.openTag && foundTag.count < rule.minTags {
+				self.append(foundTag)
+				continue
+			}
+			
+			if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
+				let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
+				var escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
+				if let hasIntermediateTag = rule.intermediateTag {
+					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
+				}
+				if let existentClosingTag = rule.closingTag {
+					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
+				}
+				self.append(escaped)
+				continue
+			}
+			
+			
+			self.performanceLog.tag(with: "(found tag \(rule.openTag))")
+			
+			if !foundTag.contains(rule.openTag) && !foundTag.contains(rule.intermediateTag ?? "") && !foundTag.contains(rule.closingTag ?? "") {
+				self.append(foundTag)
+				continue
+			}
+			
+			
+			var tokenGroups : [TokenGroup] = []
+			var escapeCharacter : Character? = nil
+			var cumulatedString = ""
+			for char in foundTag {
+				if let existentEscapeCharacter = escapeCharacter {
+					
+					// If any of the tags feature the current character
+					let escape = String(existentEscapeCharacter)
+					let nextTagCharacter = String(char)
+					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
+						escapeCharacter = nil
+					} else if nextTagCharacter == escape {
+						// Doesn't apply to this rule
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
+					}
+					
+					continue
+				}
+				if let existentEscape = rule.escapeCharacter {
+					if char == existentEscape {
+						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+						escapeCharacter = char
+						continue
+					}
+				}
+				cumulatedString.append(char)
+				
+			}
+			if let remainingEscape = escapeCharacter {
+				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
+			}
+			
+			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+			self.append(contentsOf: tokenGroups)
+			
+			if self.state == .closed {
+				tokens.append(contentsOf: self.tokens(beginningGroupNumberAt : tokenGroup))
+			}
+		}
+		
+		tokens.append(contentsOf: self.tokens(beginningGroupNumberAt : tokenGroup))
+		self.performanceLog.end()
+
+		return tokens
+	}
+	
+	func validateSpacing( nextCharacter : String?, previousCharacter : String?, with rule : CharacterRule ) -> Bool {
+		switch rule.spacesAllowed {
+		case .leadingSide:
+			guard nextCharacter != nil else {
+				return true
+			}
+			if nextCharacter == " "  {
+				return false
+			}
+		case .trailingSide:
+			guard previousCharacter != nil else {
+				return true
+			}
+			if previousCharacter == " " {
+				return false
+			}
+		case .no:
+			switch (previousCharacter, nextCharacter) {
+			case (nil, nil), ( " ", _ ), (  _, " " ):
+				return false
+			default:
+				return true
+			}
+			
+		case .oneSide:
+			switch (previousCharacter, nextCharacter) {
+			case  (nil, " " ), (" ", nil), (" ", " " ):
+				return false
+			default:
+				return true
+			}
+		default:
+			break
+		}
+		return true
+	}
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+	
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
+}

Sources/SwiftyMarkdown/SwiftyTokeniser.swift

@@ -15,445 +15,25 @@ extension OSLog {
 	static let performance = OSLog(subsystem: subsystem, category: "Peformance")
 }
 
-// Tag definition
-public protocol CharacterStyling {
-	func isEqualTo( _ other : CharacterStyling ) -> Bool
-}
-
-public enum SpaceAllowed {
-	case no
-	case bothSides
-	case oneSide
-	case leadingSide
-	case trailingSide
-}
-
-public enum Cancel {
-	case none
-	case allRemaining
-	case currentSet
-}
-
-public struct CharacterRule : CustomStringConvertible {
-	public let openTag : String
-	public let intermediateTag : String?
-	public let closingTag : String?
-	public let escapeCharacter : Character?
-	public let styles : [Int : [CharacterStyling]]
-	public var minTags : Int = 1
-	public var maxTags : Int = 1
-	public var spacesAllowed : SpaceAllowed = .oneSide
-	public var cancels : Cancel = .none
-	
-	public var tagVarieties : [Int : String]
-	
-	public var description: String {
-		return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
-	}
-	
-	public init(openTag: String, intermediateTag: String? = nil, closingTag: String? = nil, escapeCharacter: Character? = nil, styles: [Int : [CharacterStyling]] = [:], minTags : Int = 1, maxTags : Int = 1, cancels : Cancel = .none) {
-		self.openTag = openTag
-		self.intermediateTag = intermediateTag
-		self.closingTag = closingTag
-		self.escapeCharacter = escapeCharacter
-		self.styles = styles
-		self.minTags = minTags
-		self.maxTags = maxTags
-		self.cancels = cancels
-		
-		self.tagVarieties = [:]
-		for i in minTags...maxTags {
-			self.tagVarieties[i] = openTag.repeating(i)
-		}
-	}
-}
-
-// Token definition
-public enum TokenType {
-	case repeatingTag
-	case openTag
-	case intermediateTag
-	case closeTag
-	case string
-	case escape
-	case replacement
-}
-
-
-
-public struct Token {
-	public let id = UUID().uuidString
-	public let type : TokenType
-	public let inputString : String
-	public fileprivate(set) var group : Int = 0
-	public fileprivate(set) var metadataString : String? = nil
-	public fileprivate(set) var characterStyles : [CharacterStyling] = []
-	public fileprivate(set) var count : Int = 0
-	public fileprivate(set) var shouldSkip : Bool = false
-	public fileprivate(set) var tokenIndex : Int = -1
-	public fileprivate(set) var isProcessed : Bool = false
-	public fileprivate(set) var isMetadata : Bool = false
-	
-	
-	public var outputString : String {
-		get {
-			switch self.type {
-			case .repeatingTag:
-				if count <= 0 {
-					return ""
-				} else {
-					let range = inputString.startIndex..<inputString.index(inputString.startIndex, offsetBy: self.count)
-					return String(inputString[range])
-				}
-			case .openTag, .closeTag, .intermediateTag:
-				return (self.isProcessed || self.isMetadata) ? "" : inputString
-			case .escape, .string:
-				return (self.isProcessed || self.isMetadata) ? "" : inputString
-			case .replacement:
-				return self.inputString
-			}
-		}
-	}
-	public init( type : TokenType, inputString : String, characterStyles : [CharacterStyling] = []) {
-		self.type = type
-		self.inputString = inputString
-		self.characterStyles = characterStyles
-		if type == .repeatingTag {
-			self.count = inputString.count
-		}
-	}
-	
-	func newToken( fromSubstring string: String,  isReplacement : Bool) -> Token {
-		var newToken = Token(type: (isReplacement) ? .replacement : .string , inputString: string, characterStyles: self.characterStyles)
-		newToken.metadataString = self.metadataString
-		newToken.isMetadata = self.isMetadata
-		newToken.isProcessed = self.isProcessed
-		return newToken
-	}
-}
-
-extension Sequence where Iterator.Element == Token {
-	var oslogDisplay: String {
-		return "[\"\(self.map( {  ($0.outputString.isEmpty) ? "\($0.type): \($0.inputString)" : $0.outputString }).joined(separator: "\", \""))\"]"
-	}
-}
-
-enum TagState {
-	case none
-	case open
-	case intermediate
-	case closed
-}
-
-struct TagString {
-	var state : TagState = .none
-	var preOpenString = ""
-	var openTagString : [String] = []
-	var intermediateString = ""
-	var intermediateTagString = ""
-	var metadataString = ""
-	var closedTagString : [String] = []
-	var postClosedString = ""
-	
-	let rule : CharacterRule
-	var tokenGroup = 0
-	
-	init( with rule : CharacterRule ) {
-		self.rule = rule
-	}
-	
-	mutating func append( _ string : String? ) {
-		guard let existentString = string else {
-			return
-		}
-		switch self.state {
-		case .none:
-			self.preOpenString += existentString
-		case .open:
-			self.intermediateString += existentString
-		case .intermediate:
-			self.metadataString += existentString
-		case .closed:
-			self.postClosedString += existentString
-		}
-	}
-	
-	mutating func handleRepeatingTags( _ tokenGroup : [TokenGroup] ) {
-		var availableCount = self.rule.maxTags
-		var sameOpenGroup = false
-		for token in tokenGroup {
-			
-			switch token.state {
-			case .none:
-				self.append(token.string)
-				if self.state == .closed {
-					self.state = .none
-				}
-			case .open:
-				switch self.state {
-				case .none:
-					self.openTagString.append(token.string)
-					self.state = .open
-					availableCount = self.rule.maxTags - token.string.count
-					sameOpenGroup = true
-				case .open:
-					if availableCount > 0 {
-						if sameOpenGroup {
-							self.openTagString.append(token.string)
-							availableCount = self.rule.maxTags - token.string.count
-						} else {
-							self.closedTagString.append(token.string)
-							self.state = .closed
-						}
-					} else {
-						self.append(token.string)
-					}
-
-				case .intermediate:
-					self.preOpenString += self.openTagString.joined() + token.string
-				case .closed:
-					self.append(token.string)
-				}
-			case .intermediate:
-				switch self.state {
-				case .none:
-					self.preOpenString += token.string
-				case .open:
-					self.intermediateTagString += token.string
-					self.state = .intermediate
-				case .intermediate:
-					self.metadataString += token.string
-				case .closed:
-					self.postClosedString += token.string
-				}
-				
-			case .closed:
-				switch self.state {
-				case .intermediate:
-					self.closedTagString.append(token.string)
-					self.state = .closed
-				case .closed:
-					self.postClosedString += token.string
-				case .open:
-					if self.rule.intermediateTag == nil {
-						self.closedTagString.append(token.string)
-						self.state = .closed
-					} else {
-						self.preOpenString += self.openTagString.joined()
-						self.preOpenString += self.intermediateString
-						self.preOpenString += token.string
-						self.intermediateString = ""
-						self.openTagString.removeAll()
-					}
-				case .none:
-					self.preOpenString += token.string
-				}
-			}
-		}
-		if !self.openTagString.isEmpty && self.rule.closingTag == nil && self.state != .closed {
-			self.state = .open
-		}
-	}
-	
-	mutating func handleRegularTags( _ tokenGroup : [TokenGroup] ) {
-		for token in tokenGroup {
-			
-			switch token.state {
-			case .none:
-				self.append(token.string)
-				if self.state == .closed {
-					self.state = .none
-				}
-			case .open:
-				switch self.state {
-				case .none:
-					self.openTagString.append(token.string)
-					self.state = .open
-				case .open:
-					if self.rule.maxTags == 1, self.openTagString.first == rule.openTag {
-						self.preOpenString = self.preOpenString + self.openTagString.joined() + self.intermediateString
-						self.intermediateString = ""
-						self.openTagString.removeAll()
-						self.openTagString.append(token.string)
-					} else {
-						self.openTagString.append(token.string)
-					}
-				case .intermediate:
-					self.preOpenString += self.openTagString.joined() + token.string
-				case .closed:
-					self.openTagString.append(token.string)
-				}
-			case .intermediate:
-				switch self.state {
-				case .none:
-					self.preOpenString += token.string
-				case .open:
-					self.intermediateTagString += token.string
-					self.state = .intermediate
-				case .intermediate:
-					self.metadataString += token.string
-				case .closed:
-					self.postClosedString += token.string
-				}
-				
-			case .closed:
-				switch self.state {
-				case .intermediate:
-					self.closedTagString.append(token.string)
-					self.state = .closed
-				case .closed:
-					self.postClosedString += token.string
-				case .open:
-					if self.rule.intermediateTag == nil {
-						self.closedTagString.append(token.string)
-						self.state = .closed
-					} else {
-						self.preOpenString += self.openTagString.joined()
-						self.preOpenString += self.intermediateString
-						self.preOpenString += token.string
-						self.intermediateString = ""
-						self.openTagString.removeAll()
-					}
-				case .none:
-					self.preOpenString += token.string
-				}
-			}
-		}
-		
-	}
-	
-	mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
-		if self.rule.closingTag == nil {
-			self.handleRepeatingTags(tokenGroup)
-		} else {
-			self.handleRegularTags(tokenGroup)
-		}
-	}
-	
-	func configureToken(ofType type : TokenType = .string, with string : String ) -> Token {
-		var token = Token(type: type, inputString: string)
-		token.group = self.tokenGroup
-		return token
-	}
-	
-	mutating func reset() {
-		self.preOpenString = ""
-		self.openTagString.removeAll()
-		self.intermediateString = ""
-		self.intermediateTagString = ""
-		self.metadataString = ""
-		self.closedTagString.removeAll()
-		self.postClosedString = ""
-		
-		self.state = .none
-	}
-	
-	mutating func consolidate(with string : String, into tokens : inout [Token]) -> [Token] {
-		self.reset()
-		guard !string.isEmpty else {
-			return tokens
-		}
-		tokens.append(self.configureToken(with: string))
-		return tokens
-	}
-	
-	mutating func tokens(beginningGroupNumberAt group : Int = 0) -> [Token] {
-		self.tokenGroup = group
-		var tokens : [Token] = []
-		
-		if self.intermediateString.isEmpty && self.intermediateTagString.isEmpty && self.metadataString.isEmpty {
-			let actualString = self.preOpenString + self.openTagString.joined() + self.closedTagString.joined() + self.postClosedString
-			return self.consolidate(with: actualString, into: &tokens)
-		}
-		if self.state == .open && !self.openTagString.isEmpty {
-			let actualString = self.preOpenString + self.openTagString.joined() + self.intermediateString
-			return self.consolidate(with: actualString, into: &tokens)
-		}
-		
-		if !self.preOpenString.isEmpty {
-			tokens.append(self.configureToken(with: self.preOpenString))
-		}
-		
-		for tag in self.openTagString {
-			if self.rule.closingTag == nil {
-				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
-			} else {
-				tokens.append(self.configureToken(ofType: .openTag, with: tag))
-			}
-		}
-		self.tokenGroup += 1
-		if !self.intermediateString.isEmpty {
-			var token = self.configureToken(with: self.intermediateString)
-			token.metadataString = (self.metadataString.isEmpty) ? nil : self.metadataString
-			tokens.append(token)
-		}
-		if !self.intermediateTagString.isEmpty {
-			tokens.append(self.configureToken(ofType: .intermediateTag, with: self.intermediateTagString))
-		}
-		
-		self.tokenGroup += 1
-		
-		if !self.metadataString.isEmpty {
-			tokens.append(self.configureToken(with: self.metadataString))
-		}
-		var remainingTags = ( self.rule.closingTag == nil ) ? self.openTagString.joined() : ""
-		for tag in self.closedTagString {
-			if self.rule.closingTag == nil {
-				remainingTags = remainingTags.replacingOccurrences(of: tag, with: "")
-				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
-			} else {
-				tokens.append(self.configureToken(ofType: .closeTag, with: tag))
-			}
-		}
-		if !self.postClosedString.isEmpty {
-			tokens.append(self.configureToken(with: self.postClosedString))
-		}
-		
-		self.reset()
-		
-		if !remainingTags.isEmpty {
-			self.state = .open
-		}
-		
-		return tokens
-	}
-}
-
-struct TokenGroup {
-	enum TokenGroupType {
-		case string
-		case tag
-		case escape
-	}
-	
-	let string : String
-	let isEscaped : Bool
-	let type : TokenGroupType
-	var state : TagState = .none
-}
-
 public class SwiftyTokeniser {
 	let rules : [CharacterRule]
 	var replacements : [String : [Token]] = [:]
 	
-	var currentRunTime : TimeInterval = 0
-	var totalTime : TimeInterval = 0
 	var enableLog = (ProcessInfo.processInfo.environment["SwiftyTokeniserLogging"] != nil)
-	var enablePerformanceLog = (ProcessInfo.processInfo.environment["SwiftyTokeniserPerformanceLogging"] != nil)
+	let totalPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Total Run Time", log: OSLog.performance)
+	let currentPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Current", log: OSLog.performance)
+		
+	public var scanner : SwiftyScanning
 	
-	public init( with rules : [CharacterRule] ) {
+	public init( with rules : [CharacterRule], scanner : SwiftyScanning ) {
 		self.rules = rules
-		if enablePerformanceLog {
-			self.totalTime = Date.timeIntervalSinceReferenceDate
-			os_log("--- TIMER: Tokeniser initialised", log: .performance, type: .info)
-		}
-
+		self.scanner = scanner
+		
+		self.totalPerfomanceLog.start()
 	}
 	
 	deinit {
-		if enablePerformanceLog {
-			os_log("--- TIMER (Tokeniser deinitialised): %f", log: .performance, type: .info, Date.timeIntervalSinceReferenceDate - self.totalTime)
-		}
+		self.totalPerfomanceLog.end()
 	}
 	
 	
@@ -476,12 +56,8 @@ public class SwiftyTokeniser {
 		var currentTokens : [Token] = []
 		var mutableRules = self.rules
 		
-		self.totalTime = Date().timeIntervalSinceReferenceDate
 		
-		if enablePerformanceLog {
-			self.currentRunTime = Date().timeIntervalSinceReferenceDate
-			os_log("TIMER (total run time): %f", log: .performance, type: .info, Date().timeIntervalSinceReferenceDate - self.totalTime)
-		}
+		self.currentPerfomanceLog.start()
 		
 		while !mutableRules.isEmpty {
 			let nextRule = mutableRules.removeFirst()
@@ -490,14 +66,12 @@ public class SwiftyTokeniser {
 				os_log("------------------------------", log: .tokenising, type: .info)
 				os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
 			}
-			if enablePerformanceLog {
-				os_log("TIMER (start rule %@): %f", log: .performance, type: .info, nextRule.openTag, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-			}
+			self.currentPerfomanceLog.tag(with: "(start rule %@)")
 
 			
 			if currentTokens.isEmpty {
 				// This means it's the first time through
-				currentTokens = self.applyStyles(to: self.scan(inputString, with: nextRule), usingRule: nextRule)
+				currentTokens = self.applyStyles(to: self.scanner.scan(inputString, with: nextRule), usingRule: nextRule)
 				continue
 			}
 			
@@ -555,9 +129,7 @@ public class SwiftyTokeniser {
 			// The one string token might then be exploded into multiple more tokens
 		}
 		
-		if enablePerformanceLog {
-			os_log("TIMER (finished all rules): %f", log: .performance, type: .info, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-		}
+		self.currentPerfomanceLog.tag(with: "(finished all rules)")
 		
 		if enableLog {
 			os_log("=====RULE PROCESSING COMPLETE=====", log: .tokenising, type: .info)
@@ -682,7 +254,7 @@ public class SwiftyTokeniser {
 		
 		let combinedString = tokens.map({ $0.outputString }).joined()
 		
-		let nextTokens = self.scan(combinedString, with: rule)
+		let nextTokens = self.scanner.scan(combinedString, with: rule)
 		var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
 		
 		/// It's necessary here to check to see if the first token (which will always represent the styles
@@ -929,256 +501,7 @@ public class SwiftyTokeniser {
 	}
 	
 	
-	func scanSpacing( _ scanner : Scanner, usingCharactersIn set : CharacterSet ) -> (preTag : String?, foundChars : String?, postTag : String?) {
-		if enablePerformanceLog {
-			os_log("TIMER (scan space)  : %f", log: .performance, type: .info, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-		}
-		let lastChar : String?
-		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-			lastChar = ( scanner.currentIndex > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
-		} else {
-			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
-				lastChar = ( scanLocation > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanLocation)..<scanLocation]) : nil
-			} else {
-				lastChar = nil
-			}
-			
-		}
-		let maybeFoundChars : String?
-		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-			maybeFoundChars = scanner.scanCharacters(from: set )
-		} else {
-			var string : NSString?
-			scanner.scanCharacters(from: set, into: &string)
-			maybeFoundChars = string as String?
-		}
-		
-		let nextChar : String?
-		if #available(iOS 13.0, OSX 10.15,  watchOS 6.0,tvOS 13.0, *) {
-			nextChar = (scanner.currentIndex != scanner.string.endIndex) ? String(scanner.string[scanner.currentIndex]) : nil
-		} else {
-			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
-				nextChar = (scanLocation != scanner.string.endIndex) ? String(scanner.string[scanLocation]) : nil
-			} else {
-				nextChar = nil
-			}
-		}
-		if enablePerformanceLog {
-			os_log("TIMER (end space)   : %f", log: .performance, type: .info, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-		}
-		
-		return (lastChar, maybeFoundChars, nextChar)
-	}
-	
-	func getTokenGroups( for string : inout String, with rule : CharacterRule, shouldEmpty : Bool = false ) -> [TokenGroup] {
-		if string.isEmpty {
-			return []
-		}
-		var groups : [TokenGroup] 	= []
-		
-		if string.contains(rule.openTag) {
-			if shouldEmpty || string == rule.tagVarieties[rule.maxTags]{
-				var token = TokenGroup(string: string, isEscaped: false, type: .tag)
-				token.state = .open
-				groups.append(token)
-				string.removeAll()
-			}
-			
-		} else if let intermediateString = rule.intermediateTag, string.contains(intermediateString)  {
-			
-			if let range = string.range(of: intermediateString) {
-				let prior = string[string.startIndex..<range.lowerBound]
-				let tag = string[range]
-				let following = string[range.upperBound..<string.endIndex]
-				if !prior.isEmpty {
-					groups.append(TokenGroup(string: String(prior), isEscaped: false, type: .string))
-				}
-				var token = TokenGroup(string: String(tag), isEscaped: false, type: .tag)
-				token.state = .intermediate
-				groups.append(token)
-				if !following.isEmpty {
-					groups.append(TokenGroup(string: String(following), isEscaped: false, type: .string))
-				}
-				string.removeAll()
-			}
-		} else if let closingTag = rule.closingTag, closingTag.contains(string) {
-			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
-			token.state = .closed
-			groups.append(token)
-			string.removeAll()
-		}
-		
-		if shouldEmpty && !string.isEmpty {
-			let token = TokenGroup(string: string, isEscaped: false, type: .tag)
-			groups.append(token)
-			string.removeAll()
-		}
-		return groups
-	}
-	
-	func scan( _ string : String, with rule : CharacterRule) -> [Token] {
-		
-		
-		let scanner = Scanner(string: string)
-		scanner.charactersToBeSkipped = nil
-		var tokens : [Token] = []
-		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
-		if let existentEscape = rule.escapeCharacter {
-			set.insert(charactersIn: String(existentEscape))
-		}
-
-		var tagString = TagString(with: rule)
-		var tokenGroup = 0
-		
-		if enablePerformanceLog {
-			os_log("TIMER (start scan %@): %f (string: %@)", log: .performance, type: .info, rule.openTag, Date().timeIntervalSinceReferenceDate - self.currentRunTime, string)
-		}
-		
-		if !string.contains( rule.openTag ) {
-			return [Token(type: .string, inputString: string)]
-		}
-		
-		while !scanner.isAtEnd {
-			if enablePerformanceLog {
-				os_log("TIMER (loop start %@): %f", log: .performance, type: .info, rule.openTag, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-			}
-			tokenGroup += 1
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				if let start = scanner.scanUpToCharacters(from: set) {
-					if enablePerformanceLog {
-						os_log("TIMER (first chars)  : %f", log: .performance, type: .info, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-					}
-					tagString.append(start)
-				}
-			} else {
-				var string : NSString?
-				scanner.scanUpToCharacters(from: set, into: &string)
-				if let existentString = string as String? {
-					tagString.append(existentString)
-				}
-			}
-			
-			// The end of the string
-			let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
-			guard let foundTag = spacing.foundChars else {
-				continue
-			}
-			
-			if foundTag == rule.openTag && foundTag.count < rule.minTags {
-				tagString.append(foundTag)
-				continue
-			}
-			
-			if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
-				let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
-				var escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
-				if let hasIntermediateTag = rule.intermediateTag {
-					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
-				}
-				if let existentClosingTag = rule.closingTag {
-					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
-				}
-				tagString.append(escaped)
-				continue
-			}
-			
-			
-			if enablePerformanceLog {
-				os_log("TIMER (found tag %@) : %f", log: .performance, type: .info, rule.openTag, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-			}
-			
-			if !foundTag.contains(rule.openTag) && !foundTag.contains(rule.intermediateTag ?? "") && !foundTag.contains(rule.closingTag ?? "") {
-				tagString.append(foundTag)
-				continue
-			}
-			
-			
-			var tokenGroups : [TokenGroup] = []
-			var escapeCharacter : Character? = nil
-			var cumulatedString = ""
-			for char in foundTag {
-				if let existentEscapeCharacter = escapeCharacter {
-					
-					// If any of the tags feature the current character
-					let escape = String(existentEscapeCharacter)
-					let nextTagCharacter = String(char)
-					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
-						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
-						escapeCharacter = nil
-					} else if nextTagCharacter == escape {
-						// Doesn't apply to this rule
-						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
-					}
-					
-					continue
-				}
-				if let existentEscape = rule.escapeCharacter {
-					if char == existentEscape {
-						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
-						escapeCharacter = char
-						continue
-					}
-				}
-				cumulatedString.append(char)
-				tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
-				
-			}
-			if let remainingEscape = escapeCharacter {
-				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
-			}
-			
-			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
-			tagString.append(contentsOf: tokenGroups)
-			
-			if tagString.state == .closed {
-				tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
-			}
-		}
-		
-		tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
-		if enablePerformanceLog {
-			os_log("TIMER (end scan %@)  : %f", log: .performance, type: .info, rule.openTag, Date().timeIntervalSinceReferenceDate - self.currentRunTime)
-		}
 
-		return tokens
-	}
-	
-	func validateSpacing( nextCharacter : String?, previousCharacter : String?, with rule : CharacterRule ) -> Bool {
-		switch rule.spacesAllowed {
-		case .leadingSide:
-			guard nextCharacter != nil else {
-				return true
-			}
-			if nextCharacter == " "  {
-				return false
-			}
-		case .trailingSide:
-			guard previousCharacter != nil else {
-				return true
-			}
-			if previousCharacter == " " {
-				return false
-			}
-		case .no:
-			switch (previousCharacter, nextCharacter) {
-			case (nil, nil), ( " ", _ ), (  _, " " ):
-				return false
-			default:
-				return true
-			}
-			
-		case .oneSide:
-			switch (previousCharacter, nextCharacter) {
-			case  (nil, " " ), (" ", nil), (" ", " " ):
-				return false
-			default:
-				return true
-			}
-		default:
-			break
-		}
-		return true
-	}
 	
 }
 

Sources/SwiftyMarkdown/Token.swift

@@ -0,0 +1,81 @@
+//
+//  Token.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+
+// Tag definition
+public protocol CharacterStyling {
+	func isEqualTo( _ other : CharacterStyling ) -> Bool
+}
+
+// Token definition
+public enum TokenType {
+	case repeatingTag
+	case openTag
+	case intermediateTag
+	case closeTag
+	case string
+	case escape
+	case replacement
+}
+
+public struct Token {
+	public let id = UUID().uuidString
+	public let type : TokenType
+	public let inputString : String
+	public var metadataString : String? = nil
+	public internal(set) var group : Int = 0
+	public internal(set) var characterStyles : [CharacterStyling] = []
+	public internal(set) var count : Int = 0
+	public internal(set) var shouldSkip : Bool = false
+	public internal(set) var tokenIndex : Int = -1
+	public internal(set) var isProcessed : Bool = false
+	public internal(set) var isMetadata : Bool = false
+	
+	
+	public var outputString : String {
+		get {
+			switch self.type {
+			case .repeatingTag:
+				if count <= 0 {
+					return ""
+				} else {
+					let range = inputString.startIndex..<inputString.index(inputString.startIndex, offsetBy: self.count)
+					return String(inputString[range])
+				}
+			case .openTag, .closeTag, .intermediateTag:
+				return (self.isProcessed || self.isMetadata) ? "" : inputString
+			case .escape, .string:
+				return (self.isProcessed || self.isMetadata) ? "" : inputString
+			case .replacement:
+				return self.inputString
+			}
+		}
+	}
+	public init( type : TokenType, inputString : String, characterStyles : [CharacterStyling] = []) {
+		self.type = type
+		self.inputString = inputString
+		self.characterStyles = characterStyles
+		if type == .repeatingTag {
+			self.count = inputString.count
+		}
+	}
+	
+	func newToken( fromSubstring string: String,  isReplacement : Bool) -> Token {
+		var newToken = Token(type: (isReplacement) ? .replacement : .string , inputString: string, characterStyles: self.characterStyles)
+		newToken.metadataString = self.metadataString
+		newToken.isMetadata = self.isMetadata
+		newToken.isProcessed = self.isProcessed
+		return newToken
+	}
+}
+
+extension Sequence where Iterator.Element == Token {
+	var oslogDisplay: String {
+		return "[\"\(self.map( {  ($0.outputString.isEmpty) ? "\($0.type): \($0.inputString)" : $0.outputString }).joined(separator: "\", \""))\"]"
+	}
+}

SwiftyMarkdown.xcodeproj/project.pbxproj

@@ -21,6 +21,11 @@
 /* End PBXAggregateTarget section */
 
 /* Begin PBXBuildFile section */
+		F4ACB6CB23E8A5C500EA665D /* SwiftyScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6CA23E8A5C500EA665D /* SwiftyScanner.swift */; };
+		F4ACB6CE23E8A88400EA665D /* Token.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6CD23E8A88400EA665D /* Token.swift */; };
+		F4ACB6D023E8A8A500EA665D /* CharacterRule.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6CF23E8A8A500EA665D /* CharacterRule.swift */; };
+		F4ACB6D223E8B08400EA665D /* PerfomanceLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6D123E8B08400EA665D /* PerfomanceLog.swift */; };
+		F4ACB6D423E8B5C500EA665D /* SwiftyMarkdown+CharacterRule.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6D323E8B5C500EA665D /* SwiftyMarkdown+CharacterRule.swift */; };
 		OBJ_40 /* String+SwiftyMarkdown.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_9 /* String+SwiftyMarkdown.swift */; };
 		OBJ_41 /* SwiftyLineProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_10 /* SwiftyLineProcessor.swift */; };
 		OBJ_42 /* SwiftyMarkdown+iOS.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_11 /* SwiftyMarkdown+iOS.swift */; };
@@ -38,6 +43,13 @@
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
+		F4ACB6CC23E8A5C600EA665D /* PBXContainerItemProxy */ = {
+			isa = PBXContainerItemProxy;
+			containerPortal = OBJ_1 /* Project object */;
+			proxyType = 1;
+			remoteGlobalIDString = "SwiftyMarkdown::SwiftyMarkdownTests";
+			remoteInfo = SwiftyMarkdownTests;
+		};
 		F4B37A0723E507C900833479 /* PBXContainerItemProxy */ = {
 			isa = PBXContainerItemProxy;
 			containerPortal = OBJ_1 /* Project object */;
@@ -48,6 +60,11 @@
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXFileReference section */
+		F4ACB6CA23E8A5C500EA665D /* SwiftyScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyScanner.swift; sourceTree = "<group>"; };
+		F4ACB6CD23E8A88400EA665D /* Token.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Token.swift; sourceTree = "<group>"; };
+		F4ACB6CF23E8A8A500EA665D /* CharacterRule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CharacterRule.swift; sourceTree = "<group>"; };
+		F4ACB6D123E8B08400EA665D /* PerfomanceLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerfomanceLog.swift; sourceTree = "<group>"; };
+		F4ACB6D323E8B5C500EA665D /* SwiftyMarkdown+CharacterRule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SwiftyMarkdown+CharacterRule.swift"; sourceTree = "<group>"; };
 		OBJ_10 /* SwiftyLineProcessor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyLineProcessor.swift; sourceTree = "<group>"; };
 		OBJ_11 /* SwiftyMarkdown+iOS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SwiftyMarkdown+iOS.swift"; sourceTree = "<group>"; };
 		OBJ_12 /* SwiftyMarkdown+macOS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SwiftyMarkdown+macOS.swift"; sourceTree = "<group>"; };
@@ -124,7 +141,7 @@
 			name = Products;
 			sourceTree = BUILT_PRODUCTS_DIR;
 		};
-		OBJ_5 /*  */ = {
+		OBJ_5 = {
 			isa = PBXGroup;
 			children = (
 				OBJ_6 /* Package.swift */,
@@ -141,7 +158,6 @@
 				OBJ_33 /* Gemfile.lock */,
 				OBJ_34 /* SwiftyMarkdown.podspec */,
 			);
-			name = "";
 			sourceTree = "<group>";
 		};
 		OBJ_7 /* Sources */ = {
@@ -160,7 +176,12 @@
 				OBJ_11 /* SwiftyMarkdown+iOS.swift */,
 				OBJ_12 /* SwiftyMarkdown+macOS.swift */,
 				OBJ_13 /* SwiftyMarkdown.swift */,
+				F4ACB6D323E8B5C500EA665D /* SwiftyMarkdown+CharacterRule.swift */,
 				OBJ_14 /* SwiftyTokeniser.swift */,
+				F4ACB6CD23E8A88400EA665D /* Token.swift */,
+				F4ACB6CA23E8A5C500EA665D /* SwiftyScanner.swift */,
+				F4ACB6CF23E8A8A500EA665D /* CharacterRule.swift */,
+				F4ACB6D123E8B08400EA665D /* PerfomanceLog.swift */,
 			);
 			name = SwiftyMarkdown;
 			path = Sources/SwiftyMarkdown;
@@ -232,7 +253,7 @@
 			knownRegions = (
 				en,
 			);
-			mainGroup = OBJ_5 /*  */;
+			mainGroup = OBJ_5;
 			productRefGroup = OBJ_23 /* Products */;
 			projectDirPath = "";
 			projectRoot = "";
@@ -250,12 +271,17 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 0;
 			files = (
+				F4ACB6D423E8B5C500EA665D /* SwiftyMarkdown+CharacterRule.swift in Sources */,
 				OBJ_40 /* String+SwiftyMarkdown.swift in Sources */,
 				OBJ_41 /* SwiftyLineProcessor.swift in Sources */,
+				F4ACB6D223E8B08400EA665D /* PerfomanceLog.swift in Sources */,
+				F4ACB6CB23E8A5C500EA665D /* SwiftyScanner.swift in Sources */,
 				OBJ_42 /* SwiftyMarkdown+iOS.swift in Sources */,
 				OBJ_43 /* SwiftyMarkdown+macOS.swift in Sources */,
 				OBJ_44 /* SwiftyMarkdown.swift in Sources */,
 				OBJ_45 /* SwiftyTokeniser.swift in Sources */,
+				F4ACB6D023E8A8A500EA665D /* CharacterRule.swift in Sources */,
+				F4ACB6CE23E8A88400EA665D /* Token.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -286,7 +312,7 @@
 		OBJ_57 /* PBXTargetDependency */ = {
 			isa = PBXTargetDependency;
 			target = "SwiftyMarkdown::SwiftyMarkdownTests" /* SwiftyMarkdownTests */;
-			targetProxy = "SwiftyMarkdown::SwiftyMarkdownTests" /* SwiftyMarkdownTests */;
+			targetProxy = F4ACB6CC23E8A5C600EA665D /* PBXContainerItemProxy */;
 		};
 		OBJ_71 /* PBXTargetDependency */ = {
 			isa = PBXTargetDependency;

SwiftyMarkdown.xcodeproj/xcshareddata/xcschemes/SwiftyMarkdown-Package.xcscheme

@@ -59,7 +59,12 @@
          <EnvironmentVariable
             key = "SwiftyLineProcessorPerformanceLogging"
             value = ""
-            isEnabled = "YES">
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyScannerPerformanceLogging"
+            value = ""
+            isEnabled = "NO">
          </EnvironmentVariable>
          <EnvironmentVariable
             key = "SwiftyTokeniserPerformanceLogging"

Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift

@@ -13,21 +13,29 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func testIsolatedCase() {
 
-		challenge = TokenTest(input: "A Bold [**Link**](http://voyagetravelapps.com/)", output: "A Bold Link", tokens: [
-			Token(type: .string, inputString: "A Bold ", characterStyles: []),
-			Token(type: .string, inputString: "Link", characterStyles: [CharacterStyle.bold, CharacterStyle.link])
+		challenge = TokenTest(input: "A [referenced link][link]\n[link]: https://www.neverendingvoyage.com/", output: "A referenced link", tokens: [
+			Token(type: .string, inputString: "A ", characterStyles: []),
+			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
 		])
-		results = self.attempt(challenge, rules:[.links, .asterisks])
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		results = self.attempt(challenge, rules: [.links, .images, .referencedLinks])
 		XCTAssertEqual(results.attributedString.string, challenge.output)
-		XCTAssertEqual(results.links.count, 1)
 		if results.links.count == 1 {
-			XCTAssertEqual(results.links[0].metadataString, "http://voyagetravelapps.com/")
+			
+			// It's the tests that are wrong!!!
+			XCTAssertEqual(results.links[0].metadataString, "https://www.neverendingvoyage.com/")
 		} else {
 			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
 		}
 		
+		return
+		
+		challenge = TokenTest(input: "A [referenced link][link]\n[notLink]: https://www.neverendingvoyage.com/", output: "A [referenced link][link]", tokens: [
+			Token(type: .string, inputString: "A [referenced link][link]", characterStyles: [])
+		])
+		results = self.attempt(challenge, rules: [.links, .images, .referencedLinks])
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 0)
+		
 	}
 	
 	func testThatBoldTraitsAreRecognised() {

Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift

@@ -222,7 +222,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		}
 	}
 	
-	func offtestForReferencedLinks() {
+	func testForReferencedLinks() {
 		challenge = TokenTest(input: "A [referenced link][link]\n[link]: https://www.neverendingvoyage.com/", output: "A referenced link", tokens: [
 			Token(type: .string, inputString: "A ", characterStyles: []),
 			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])

Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift

@@ -43,7 +43,7 @@ enum Rule {
 		case .underscores:
 			return CharacterRule(openTag: "_", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
 		case .referencedLinks:
-			return CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1)
+			return CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1)
 				
 		}
 	}
@@ -63,7 +63,7 @@ class SwiftyMarkdownCharacterTests : XCTestCase {
 		}
 		
 		let md = SwiftyMarkdown(string: challenge.input)
-		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
+		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules, scanner: SwiftyScanner())
 		let lines = challenge.input.components(separatedBy: .newlines)
 		var tokens : [Token] = []
 		for line in lines {