Browse Source

Merge branch 'feature/referencedLists' into develop

Simon Fairbairn 5 years ago
parent
commit
bcbf3bd38c
30 changed files with 3724 additions and 1733 deletions
  1. 2 0
      Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj
  2. 50 0
      Example/SwiftyMarkdownExample/example copy.md
  3. 1 50
      Example/SwiftyMarkdownExample/example.md
  4. 285 0
      Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Contents.swift
  5. 32 0
      Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Sources/Tokens.swift
  6. 1 0
      Playground/SwiftyMarkdown.playground/Sources/Swifty Tokeniser.swift
  7. 2 0
      Playground/SwiftyMarkdown.playground/contents.xcplayground
  8. 46 7
      README.md
  9. 65 0
      Resources/test.md
  10. 108 0
      Sources/SwiftyMarkdown/CharacterRule.swift
  11. 44 0
      Sources/SwiftyMarkdown/PerfomanceLog.swift
  12. 21 1
      Sources/SwiftyMarkdown/SwiftyLineProcessor.swift
  13. 2 1
      Sources/SwiftyMarkdown/SwiftyMarkdown+iOS.swift
  14. 2 0
      Sources/SwiftyMarkdown/SwiftyMarkdown+macOS.swift
  15. 89 28
      Sources/SwiftyMarkdown/SwiftyMarkdown.swift
  16. 59 0
      Sources/SwiftyMarkdown/SwiftyScanner.swift
  17. 565 0
      Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift
  18. 442 742
      Sources/SwiftyMarkdown/SwiftyTokeniser.swift
  19. 81 0
      Sources/SwiftyMarkdown/Token.swift
  20. 560 717
      SwiftyMarkdown.xcodeproj/project.pbxproj
  21. 42 0
      SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/88991ED5-B954-422F-B610-BDC9A4AEC008.plist
  22. 3 1
      SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/AD1DF83E-20BC-4E7E-8C14-683818ED0A26.plist
  23. 24 0
      SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/Info.plist
  24. 37 0
      SwiftyMarkdown.xcodeproj/xcshareddata/xcschemes/SwiftyMarkdown-Package.xcscheme
  25. 0 0
      Tests/SwiftyMarkdownTests/SwiftyMarkdownAttributedStringTests.swift
  26. 453 93
      Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift
  27. 1 6
      Tests/SwiftyMarkdownTests/SwiftyMarkdownLineTests.swift
  28. 632 77
      Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift
  29. 1 1
      Tests/SwiftyMarkdownTests/SwiftyMarkdownPerformanceTests.swift
  30. 74 9
      Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift

+ 2 - 0
Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj

@@ -63,6 +63,7 @@
 
 /* Begin PBXFileReference section */
 		F421DD951C8AF34F00B86D66 /* example.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = "<group>"; };
+		F4576C2E2437F67B0013E2B6 /* example copy.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = "example copy.md"; sourceTree = "<group>"; };
 		F4B4A44923E4E17400550249 /* SwiftyMarkdownExample macOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "SwiftyMarkdownExample macOS.app"; sourceTree = BUILT_PRODUCTS_DIR; };
 		F4B4A44B23E4E17400550249 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
 		F4B4A44D23E4E17400550249 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
@@ -172,6 +173,7 @@
 				F4CE98B21C8AEF7D00D735C1 /* Assets.xcassets */,
 				F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */,
 				F4CE98B71C8AEF7D00D735C1 /* Info.plist */,
+				F4576C2E2437F67B0013E2B6 /* example copy.md */,
 				F421DD951C8AF34F00B86D66 /* example.md */,
 			);
 			path = SwiftyMarkdownExample;

+ 50 - 0
Example/SwiftyMarkdownExample/example copy.md

@@ -0,0 +1,50 @@
+# Swifty Markdown
+
+SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts.
+
+Show Images From Your App Bundle!
+---
+![Image](bubble)
+
+Customise fonts and colors easily in a Swift-like way: 
+
+    md.code.fontName = "CourierNewPSMT"
+
+    md.h2.fontName = "AvenirNextCondensed-Medium"
+    md.h2.color = UIColor.redColor()
+    md.h2.alignment = .center
+
+It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings.
+
+It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/).
+
+> It also now supports blockquotes
+> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun!
+
+**Lists**
+
+- It Supports
+- Unordered
+- Lists
+	- Indented item with a longer string to make sure indentation is consistent
+		- Second level indent with a longer string to make sure indentation is consistent
+- List item with a longer string to make sure indentation is consistent
+
+1. And
+1. Ordered
+1. Lists
+	1. Indented item
+		1. Second level indent
+1. (Use `1.` as the list item identifier)
+1. List item
+1. List item
+	- Mix
+		- List styles
+1. List item with a longer string to make sure indentation is consistent
+1. List item
+1. List item
+1. List item
+1. List item
+
+
+

+ 1 - 50
Example/SwiftyMarkdownExample/example.md

@@ -1,50 +1 @@
-# Swifty Markdown
-
-SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts.
-
-Show Images From Your App Bundle!
----
-![Image](bubble)
-
-Customise fonts and colors easily in a Swift-like way: 
-
-    md.code.fontName = "CourierNewPSMT"
-
-    md.h2.fontName = "AvenirNextCondensed-Medium"
-    md.h2.color = UIColor.redColor()
-    md.h2.alignment = .center
-
-It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings.
-
-It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/).
-
-> It also now supports blockquotes
-> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun!
-
-**Lists**
-
-- It Supports
-- Unordered
-- Lists
-	- Indented item with a longer string to make sure indentation is consistent
-		- Second level indent with a longer string to make sure indentation is consistent
-- List item with a longer string to make sure indentation is consistent
-
-1. And
-1. Ordered
-1. Lists
-	1. Indented item
-		1. Second level indent
-1. (Use `1.` as the list item identifier)
-1. List item
-1. List item
-	- Mix
-		- List styles
-1. List item with a longer string to make sure indentation is consistent
-1. List item
-1. List item
-1. List item
-1. List item
-
-
-
+[a](b)

+ 285 - 0
Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Contents.swift

@@ -0,0 +1,285 @@
+//: [Previous](@previous)
+
+import Foundation
+
+extension String {
+	func repeating( _ max : Int ) -> String {
+		var output = self
+		for _ in 1..<max {
+			output += self
+		}
+		return output
+	}
+}
+
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+struct TagString {
+	var state : TagState = .none
+	var preOpenString = ""
+	var openTagString = ""
+	var intermediateString = ""
+	var intermediateTagString = ""
+	var metadataString = ""
+	var closedTagString = ""
+	var postClosedString = ""
+
+	let rule : Rule
+	
+	init( with rule : Rule ) {
+		self.rule = rule
+	}
+	
+	mutating func append( _ string : String? ) {
+		guard let existentString = string else {
+			return
+		}
+		switch self.state {
+		case .none:
+			self.preOpenString += existentString
+		case .open:
+			self.intermediateString += existentString
+		case .intermediate:
+			self.metadataString += existentString
+		case .closed:
+			self.postClosedString += existentString
+		}
+	}
+	
+	mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
+		print(tokenGroup)
+		for token in tokenGroup {
+			switch token.state {
+			case .none:
+				self.append(token.string)
+			case .open:
+				if self.state != .none {
+					self.preOpenString += token.string
+				} else {
+					self.openTagString += token.string
+				}
+			case .intermediate:
+				if self.state != .open {
+					self.intermediateString += token.string
+				} else {
+					self.intermediateTagString += token.string
+				}
+			case .closed:
+				if self.rule.intermediateTag != nil && self.state != .intermediate {
+					self.metadataString += token.string
+				} else {
+					self.closedTagString += token.string
+				}
+			}
+			self.state = token.state
+		}
+	}
+	
+	mutating func tokens() -> [Token] {
+		print(self)
+		var tokens : [Token] = []
+
+		if !self.preOpenString.isEmpty {
+			tokens.append(Token(type: .string, inputString: self.preOpenString))
+		}
+		if !self.openTagString.isEmpty {
+			tokens.append(Token(type: .openTag, inputString: self.openTagString))
+		}
+		if !self.intermediateString.isEmpty {
+			var token = Token(type: .string, inputString: self.intermediateString)
+			token.metadataString = self.metadataString
+			tokens.append(token)
+		}
+		if !self.intermediateTagString.isEmpty {
+			tokens.append(Token(type: .intermediateTag, inputString: self.intermediateTagString))
+		}
+		if !self.metadataString.isEmpty {
+			tokens.append(Token(type: .metadata, inputString: self.metadataString))
+		}
+		if !self.closedTagString.isEmpty {
+			tokens.append(Token(type: .closeTag, inputString: self.closedTagString))
+		}
+		
+		self.preOpenString = ""
+		self.openTagString = ""
+		self.intermediateString = ""
+		self.intermediateTagString = ""
+		self.metadataString = ""
+		self.closedTagString = ""
+		self.postClosedString = ""
+		
+		self.state = .none
+		
+		return tokens
+	}
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
+}
+
+
+
+func getTokenGroups( for string : inout String, with rule : Rule, shouldEmpty : Bool = false ) -> [TokenGroup] {
+	if string.isEmpty {
+		return []
+	}
+	let maxCount = rule.openTag.count * rule.maxTags
+	var groups : [TokenGroup] = []
+	
+	let maxTag = rule.openTag.repeating(rule.maxTags)
+	
+	if maxTag.contains(string) {
+		if string.count == maxCount || shouldEmpty {
+			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			token.state = .open
+			groups.append(token)
+			string.removeAll()
+		}
+	
+	} else if string == rule.intermediateTag {
+		var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		token.state = .intermediate
+		groups.append(token)
+		string.removeAll()
+	} else if string == rule.closingTag {
+		var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		token.state = .closed
+		groups.append(token)
+		string.removeAll()
+	}
+	
+	if shouldEmpty && !string.isEmpty {
+		let token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		groups.append(token)
+		string.removeAll()
+	}
+	return groups
+}
+
+func scan( _ string : String, with rule : Rule) -> [Token] {
+		let scanner = Scanner(string: string)
+		scanner.charactersToBeSkipped = nil
+		var tokens : [Token] = []
+		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
+		if let existentEscape = rule.escapeCharacter {
+			set.insert(charactersIn: String(existentEscape))
+		}
+		
+		var openTag = rule.openTag.repeating(rule.maxTags)
+		
+		var tagString = TagString(with: rule)
+		
+		var openTagFound : TagState = .none
+		var regularCharacters = ""
+		var tagGroupCount = 0
+		while !scanner.isAtEnd {
+			tagGroupCount += 1
+			
+			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+				if let start = scanner.scanUpToCharacters(from: set) {
+					tagString.append(start)
+				}
+			} else {
+				var string : NSString?
+				scanner.scanUpToCharacters(from: set, into: &string)
+				if let existentString = string as String? {
+					tagString.append(existentString)
+				}
+			}
+			
+			// The end of the string
+			let maybeFoundChars = scanner.scanCharacters(from: set )
+			guard let foundTag = maybeFoundChars else {
+				continue
+			}
+			
+			if foundTag == rule.openTag && foundTag.count < rule.minTags {
+				tagString.append(foundTag)
+				continue
+			}
+			
+			//:--
+			print(foundTag)
+			var tokenGroups : [TokenGroup] = []
+			var escapeCharacter : Character? = nil
+			var cumulatedString = ""
+			for char in foundTag {
+				if let existentEscapeCharacter = escapeCharacter {
+					
+					// If any of the tags feature the current character
+					let escape = String(existentEscapeCharacter)
+					let nextTagCharacter = String(char)
+					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
+						escapeCharacter = nil
+					} else if nextTagCharacter == escape {
+						// Doesn't apply to this rule
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
+					}
+					
+					continue
+				}
+				if let existentEscape = rule.escapeCharacter {
+					if char == existentEscape {
+						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+						escapeCharacter = char
+						continue
+					}
+				}
+				cumulatedString.append(char)
+				tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
+				
+			}
+			if let remainingEscape = escapeCharacter {
+				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
+			}
+
+			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+			tagString.append(contentsOf: tokenGroups)
+			
+			if tagString.state == .closed {
+				tokens.append(contentsOf: tagString.tokens())
+			}
+			
+			
+		}
+		
+		tokens.append(contentsOf: tagString.tokens())
+		
+		
+		return tokens
+	}
+
+//: [Next](@next)
+
+
+
+var string = "[]([[\\[Some Link]\\]](\\(\\(\\url) [Regular link](url)"
+//string = "Text before [Regular link](url) Text after"
+var output = "[]([[Some Link]] Regular link"
+
+var tokens = scan(string, with: LinkRule())
+print( tokens.filter( { $0.type == .string }).map({ $0.outputString }).joined())
+//print( tokens )
+
+//string = "**\\*\\Bold\\*\\***"
+//output = "*\\Bold**"
+
+//tokens = scan(string, with: AsteriskRule())
+//print( tokens )
+

+ 32 - 0
Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Sources/Tokens.swift

@@ -0,0 +1,32 @@
+import Foundation
+
+
+
+public protocol Rule {
+	var escapeCharacter : Character? { get }
+	var openTag : String { get }
+	var intermediateTag : String? { get }
+	var closingTag : String? { get }
+	var maxTags : Int { get }
+	var minTags : Int { get }
+}
+
+public struct LinkRule : Rule {
+	public let escapeCharacter : Character? = "\\"
+	public let openTag : String = "["
+	public let intermediateTag : String? = "]("
+	public let closingTag : String? = ")"
+	public let maxTags : Int = 1
+	public let minTags : Int = 1
+	public init() { }
+}
+
+public struct AsteriskRule : Rule {
+	public let escapeCharacter : Character? = "\\"
+	public let openTag : String = "*"
+	public let intermediateTag : String? = nil
+	public let closingTag : String? = nil
+	public let maxTags : Int = 3
+	public let minTags : Int = 1
+	public init() { }
+}

+ 1 - 0
Playground/SwiftyMarkdown.playground/Sources/Swifty Tokeniser.swift

@@ -74,6 +74,7 @@ public struct Token {
 	public let inputString : String
 	public var metadataString : String? = nil
 	public var characterStyles : [CharacterStyling] = []
+	public var group : Int = 0
 	public var count : Int = 0
 	public var shouldSkip : Bool = false
 	public var outputString : String {

+ 2 - 0
Playground/SwiftyMarkdown.playground/contents.xcplayground

@@ -5,5 +5,7 @@
         <page name='Line Processing'/>
         <page name='Tokenising'/>
         <page name='Attributed String'/>
+        <page name='SKLabelNode'/>
+        <page name='Groups'/>
     </pages>
 </playground>

+ 46 - 7
README.md

@@ -92,6 +92,12 @@ label.attributedText = md.attributedString()
     [Links](http://voyagetravelapps.com/)
     ![Images](<Name of asset in bundle>)
     
+    [Referenced Links][1]
+    ![Referenced Images][2]
+    
+    [1]: http://voyagetravelapps.com/
+    [2]: <Name of asset in bundle>
+    
     > Blockquotes
 	
 	- Bulleted
@@ -292,13 +298,46 @@ enum CharacterStyle : CharacterStyling {
 }
 
 static public var characterRules = [
-	CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1),
-	CharacterRule(openTag: "`", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.code]], maxTags: 1),
-	CharacterRule(openTag: "*", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3),
-	CharacterRule(openTag: "_", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
+    CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open), otherTags: [
+			CharacterRuleTag(tag: "]", type: .close),
+			CharacterRuleTag(tag: "[", type: .metadataOpen),
+			CharacterRuleTag(tag: "]", type: .metadataClose)
+	], styles: [1 : CharacterStyle.link], metadataLookup: true, definesBoundary: true),
+	CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingTags: true, balancedTags: true),
+	CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2),
+	CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2)
 ]
 ```
 
+These Character Rules are defined by SwiftyMarkdown:
+
+	public struct CharacterRule : CustomStringConvertible {
+
+		public let primaryTag : CharacterRuleTag
+		public let tags : [CharacterRuleTag]
+		public let escapeCharacters : [Character]
+		public let styles : [Int : CharacterStyling]
+		public let minTags : Int
+		public let maxTags : Int
+		public var metadataLookup : Bool = false
+		public var definesBoundary = false
+		public var shouldCancelRemainingRules = false
+		public var balancedTags = false
+	}
+
+1. `primaryTag`: Each rule must have at least one tag and it can be one of `repeating`, `open`, `close`, `metadataOpen`, or `metadataClose`. `repeating` tags are tags that have identical open and close characters (and often have more than 1 style depending on how many are in a group). For example, the `*` tag used in Markdown.
+2. `tags`: An array of other tags that the rule can look for. This is where you would put the `close` tag for a custom rule, for example.
+3. `escapeCharacters`: The characters that appear prior to any of the tag characters that tell the scanner to ignore the tag. 
+4. `styles`: The styles that should be applied to every character between the opening and closing tags. 
+5. `minTags`: The minimum number of repeating characters to be considered a successful match. For example, setting the `primaryTag` to `*` and the `minTag` to 2 would mean that `**foo**` would be a successful match wheras `*bar*` would not.
+6. `maxTags`: The maximum number of repeating characters to be considered a successful match. 
+7. `metadataLookup`: Used for Markdown reference links. Tells the scanner to try to look up the metadata from this dictionary, rather than from the inline result. 
+8. `definesBoundary`: In order for open and close tags to be successful, the `boundaryCount` for a given location in the string needs to be the same. Setting this property to `true` means that this rule will increase the `boundaryCount` for every character between its opening and closing tags. For example, the `[` rule defines a boundary. After it is applied, the string `*foo[bar*]` becomes `*foobar*` with a boundaryCount `00001111`. Applying the `*` rule results in the output `*foobar*` because the opening `*` tag and the closing `*` tag now have different `boundaryCount` values. It's basically a way to fix the `**[should not be bold**](url)` problem in Markdown. 
+9. `shouldCancelRemainingTags`: A successful match will mark every character between the opening and closing tags as complete, thereby preventing any further rules from being applied to those characters.
+10. `balancedTags`: This flag requires that the opening and closing tags be of exactly equal length. E.g. If this is set to true,  `**foo*` would result in `**foo*`. If it was false, the output would be `*foo`.
+
+
+
 #### Rule Subsets
 
 If you want to only support a small subset of Markdown, it's now easy to do. 
@@ -308,8 +347,8 @@ This example would only process strings with `*` and `_` characters, ignoring li
 SwiftyMarkdown.lineRules = []
 
 SwiftyMarkdown.characterRules = [
-	CharacterRule(openTag: "*", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3),
-	CharacterRule(openTag: "_", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
+	CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2),
+	CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2)
 ]
 ```
 
@@ -330,7 +369,7 @@ enum Characters : CharacterStyling {
 }
 
 let characterRules = [
-	CharacterRule(openTag: "%", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.elf]], maxTags: 1)
+	CharacterRule(primaryTag: CharacterRuleTag(tag: "%", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.elf])
 ]
 
 let processor = SwiftyTokeniser( with : characterRules )

+ 65 - 0
Resources/test.md

@@ -62,3 +62,68 @@ Header 2
 	1. Including indented lists
 		- Up to three levels
 1. Neat! 
+
+# SwiftyMarkdown 1.0
+
+SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a Swift-style syntax. It uses dynamic type to set the font size correctly with whatever font you'd like to use.
+
+## Fully Rebuilt For 2020!
+
+SwiftyMarkdown now features a more robust and reliable rules-based line processing and tokenisation engine. It has added support for images stored in the bundle (`![Image](<Name In bundle>)`), codeblocks, blockquotes, and unordered lists!
+
+Line-level attributes can now have a paragraph alignment applied to them (e.g. `h2.aligment = .center`), and links can be underlined by setting underlineLinks to `true`. 
+
+It also uses the system color `.label` as the default font color on iOS 13 and above for Dark Mode support out of the box. 
+
+## Installation
+
+### CocoaPods:
+
+`pod 'SwiftyMarkdown'`
+
+### SPM: 
+
+In Xcode, `File -> Swift Packages -> Add Package Dependency` and add the GitHub URL. 
+
+*italics* or _italics_
+**bold** or __bold__
+~~Linethrough~~Strikethroughs. 
+`code`
+
+# Header 1
+
+or
+
+Header 1
+====
+
+## Header 2
+
+or
+
+Header 2
+---
+
+### Header 3
+#### Header 4
+##### Header 5 #####
+###### Header 6 ######
+
+	Indented code blocks (spaces or tabs)
+
+[Links](http://voyagetravelapps.com/)
+![Images](<Name of asset in bundle>)
+
+> Blockquotes
+
+- Bulleted
+- Lists
+	- Including indented lists
+		- Up to three levels
+- Neat!
+
+1. Ordered
+1. Lists
+	1. Including indented lists
+		- Up to three levels
+1. Neat! 

+ 108 - 0
Sources/SwiftyMarkdown/CharacterRule.swift

@@ -0,0 +1,108 @@
+//
+//  CharacterRule.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+
+public enum SpaceAllowed {
+	case no
+	case bothSides
+	case oneSide
+	case leadingSide
+	case trailingSide
+}
+
+public enum Cancel {
+	case none
+	case allRemaining
+	case currentSet
+}
+
+public enum CharacterRuleTagType {
+	case open
+	case close
+	case metadataOpen
+	case metadataClose
+	case repeating
+}
+
+
+public struct CharacterRuleTag {
+	let tag : String
+	let type : CharacterRuleTagType
+	
+	public init( tag : String, type : CharacterRuleTagType ) {
+		self.tag = tag
+		self.type = type
+	}
+}
+
+public struct CharacterRule : CustomStringConvertible {
+	
+	public let primaryTag : CharacterRuleTag
+	public let tags : [CharacterRuleTag]
+	public let escapeCharacters : [Character]
+	public let styles : [Int : CharacterStyling]
+	public let minTags : Int
+	public let maxTags : Int
+	public var metadataLookup : Bool = false
+	public var isRepeatingTag : Bool {
+		return self.primaryTag.type == .repeating
+	}
+	public var definesBoundary = false
+	public var shouldCancelRemainingRules = false
+	public var balancedTags = false
+	
+	public var description: String {
+		return "Character Rule with Open tag: \(self.primaryTag.tag) and current styles : \(self.styles) "
+	}
+	
+	public func tag( for type : CharacterRuleTagType ) -> CharacterRuleTag? {
+		return self.tags.filter({ $0.type == type }).first ?? nil
+	}
+	
+	public init(primaryTag: CharacterRuleTag, otherTags: [CharacterRuleTag], escapeCharacters : [Character] = ["\\"], styles: [Int : CharacterStyling] = [:], minTags : Int = 1, maxTags : Int = 1, metadataLookup : Bool = false, definesBoundary : Bool = false, shouldCancelRemainingRules : Bool = false, balancedTags : Bool = false) {
+		self.primaryTag = primaryTag
+		self.tags = otherTags
+		self.escapeCharacters = escapeCharacters
+		self.styles = styles
+		self.metadataLookup = metadataLookup
+		self.definesBoundary = definesBoundary
+		self.shouldCancelRemainingRules = shouldCancelRemainingRules
+		self.minTags = maxTags < minTags ? maxTags : minTags
+		self.maxTags = minTags > maxTags ? minTags : maxTags
+		self.balancedTags = balancedTags
+	}
+}
+
+
+
+enum ElementType {
+	case tag
+	case escape
+	case string
+	case space
+	case newline
+	case metadata
+}
+
+struct Element {
+	let character : Character
+	var type : ElementType
+	var boundaryCount : Int = 0
+	var isComplete : Bool = false
+	var styles : [CharacterStyling] = []
+	var metadata : [String] = []
+}
+
+extension CharacterSet {
+    func containsUnicodeScalars(of character: Character) -> Bool {
+        return character.unicodeScalars.allSatisfy(contains(_:))
+    }
+}
+
+
+

+ 44 - 0
Sources/SwiftyMarkdown/PerfomanceLog.swift

@@ -0,0 +1,44 @@
+//
+//  PerfomanceLog.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+import os.log
+
+class PerformanceLog {
+	var timer : TimeInterval = 0
+	let enablePerfomanceLog : Bool
+	let log : OSLog
+	let identifier : String
+	
+	init( with environmentVariableName : String, identifier : String, log : OSLog  ) {
+		self.log = log
+		self.enablePerfomanceLog = (ProcessInfo.processInfo.environment[environmentVariableName] != nil)
+		self.identifier = identifier
+	}
+	
+	func start() {
+		guard enablePerfomanceLog else { return }
+		self.timer = Date().timeIntervalSinceReferenceDate
+		os_log("--- TIMER %{public}@ began", log: self.log, type: .info, self.identifier)
+	}
+	
+	func tag( with string : String) {
+		guard enablePerfomanceLog else { return }
+		if timer == 0 {
+			self.start()
+		}
+		os_log("TIMER %{public}@: %f %@", log: self.log, type: .info, self.identifier, Date().timeIntervalSinceReferenceDate - self.timer, string)
+	}
+	
+	func end() {
+		guard enablePerfomanceLog else { return }
+		self.timer = Date().timeIntervalSinceReferenceDate
+		os_log("--- TIMER %{public}@ finished. Total time: %f", log: self.log, type: .info, self.identifier, Date().timeIntervalSinceReferenceDate - self.timer)
+		self.timer = 0
+
+	}
+}

+ 21 - 1
Sources/SwiftyMarkdown/SwiftyLineProcessor.swift

@@ -7,6 +7,12 @@
 //
 
 import Foundation
+import os.log
+
+extension OSLog {
+	private static var subsystem = "SwiftyLineProcessor"
+	static let swiftyLineProcessorPerformance = OSLog(subsystem: subsystem, category: "Swifty Line Processor Performance")
+}
 
 public protocol LineStyling {
     var shouldTokeniseLine : Bool { get }
@@ -73,7 +79,9 @@ public class SwiftyLineProcessor {
     
     let lineRules : [LineRule]
 	let frontMatterRules : [FrontMatterRule]
-    
+	
+	let perfomanceLog = PerformanceLog(with: "SwiftyLineProcessorPerformanceLogging", identifier: "Line Processor", log: OSLog.swiftyLineProcessorPerformance)
+	    
 	public init( rules : [LineRule], defaultRule: LineStyling, frontMatterRules : [FrontMatterRule] = []) {
         self.lineRules = rules
         self.defaultType = defaultRule
@@ -117,6 +125,10 @@ public class SwiftyLineProcessor {
 				return nil
 			}
             
+			if !text.contains(element.token) {
+				continue
+			}
+			
             switch element.removeFrom {
             case .leading:
                 output = findLeadingLineElement(element, in: output)
@@ -199,9 +211,15 @@ public class SwiftyLineProcessor {
     public func process( _ string : String ) -> [SwiftyLine] {
         var foundAttributes : [SwiftyLine] = []
 		
+		
+		self.perfomanceLog.start()
+		
 		var lines = string.components(separatedBy: CharacterSet.newlines)
 		lines = self.processFrontMatter(lines)
 		
+		self.perfomanceLog.tag(with: "(Front matter completed)")
+		
+
         for  heading in lines {
             
             if processEmptyStrings == nil && heading.isEmpty {
@@ -220,6 +238,8 @@ public class SwiftyLineProcessor {
                 continue
             }
             foundAttributes.append(input)
+			
+			self.perfomanceLog.tag(with: "(line completed: \(heading)")
         }
         return foundAttributes
     }

+ 2 - 1
Sources/SwiftyMarkdown/SwiftyMarkdown+iOS.swift

@@ -165,7 +165,8 @@ extension SwiftyMarkdown {
 			return blockquotes.color
 		case .unorderedList, .unorderedListIndentFirstOrder, .unorderedListIndentSecondOrder, .orderedList, .orderedListIndentFirstOrder, .orderedListIndentSecondOrder:
 			return body.color
-
+		case .referencedLink:
+			return link.color
 		}
 	}
 	

+ 2 - 0
Sources/SwiftyMarkdown/SwiftyMarkdown+macOS.swift

@@ -137,6 +137,8 @@ extension SwiftyMarkdown {
 			return body.color
 		case .yaml:
 			return body.color
+		case .referencedLink:
+			return body.color
 		}
 	}
 	

+ 89 - 28
Sources/SwiftyMarkdown/SwiftyMarkdown.swift

@@ -5,23 +5,30 @@
 //  Created by Simon Fairbairn on 05/03/2016.
 //  Copyright © 2016 Voyage Travel Apps. All rights reserved.
 //
-
+import os.log
 #if os(macOS)
 import AppKit
 #else
 import UIKit
 #endif
 
-enum CharacterStyle : CharacterStyling {
+extension OSLog {
+	private static var subsystem = "SwiftyMarkdown"
+	static let swiftyMarkdownPerformance = OSLog(subsystem: subsystem, category: "Swifty Markdown Performance")
+}
+
+public enum CharacterStyle : CharacterStyling {
 	case none
 	case bold
 	case italic
 	case code
 	case link
 	case image
+	case referencedLink
+	case referencedImage
 	case strikethrough
 	
-	func isEqualTo(_ other: CharacterStyling) -> Bool {
+	public func isEqualTo(_ other: CharacterStyling) -> Bool {
 		guard let other = other as? CharacterStyle else {
 			return false
 		}
@@ -57,7 +64,7 @@ enum MarkdownLineStyle : LineStyling {
     case orderedList
 	case orderedListIndentFirstOrder
 	case orderedListIndentSecondOrder
-
+	case referencedLink
 	
     func styleIfFoundStyleAffectsPreviousLine() -> LineStyling? {
         switch self {
@@ -132,8 +139,12 @@ If that is not set, then the system default will be used.
 
 /// A class that takes a [Markdown](https://daringfireball.net/projects/markdown/) string or file and returns an NSAttributedString with the applied styles. Supports Dynamic Type.
 @objc open class SwiftyMarkdown: NSObject {
+	
+	static public var frontMatterRules = [
+		FrontMatterRule(openTag: "---", closeTag: "---", keyValueSeparator: ":")
+	]
+	
 	static public var lineRules = [
-		
 		LineRule(token: "=", type: MarkdownLineStyle.previousH1, removeFrom: .entireLine, changeAppliesTo: .previous),
 		LineRule(token: "-", type: MarkdownLineStyle.previousH2, removeFrom: .entireLine, changeAppliesTo: .previous),
 		LineRule(token: "\t\t- ", type: MarkdownLineStyle.unorderedListIndentSecondOrder, removeFrom: .leading, shouldTrim: false),
@@ -157,16 +168,30 @@ If that is not set, then the system default will be used.
 	]
 	
 	static public var characterRules = [
-		CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]], maxTags: 1),
-		CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1),
-		CharacterRule(openTag: "`", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.code]], maxTags: 1, cancels: .allRemaining),
-		CharacterRule(openTag: "~", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [2 : [CharacterStyle.strikethrough]], minTags: 2, maxTags: 2),
-		CharacterRule(openTag: "*", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3),
-		CharacterRule(openTag: "_", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
-	]
-	
-	static public var frontMatterRules = [
-		FrontMatterRule(openTag: "---", closeTag: "---", keyValueSeparator: ":")
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "![", type: .open), otherTags: [
+				CharacterRuleTag(tag: "]", type: .close),
+				CharacterRuleTag(tag: "[", type: .metadataOpen),
+				CharacterRuleTag(tag: "]", type: .metadataClose)
+		], styles: [1 : CharacterStyle.image], metadataLookup: true, definesBoundary: true),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "![", type: .open), otherTags: [
+				CharacterRuleTag(tag: "]", type: .close),
+				CharacterRuleTag(tag: "(", type: .metadataOpen),
+				CharacterRuleTag(tag: ")", type: .metadataClose)
+		], styles: [1 : CharacterStyle.image], metadataLookup: false, definesBoundary: true),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open), otherTags: [
+				CharacterRuleTag(tag: "]", type: .close),
+				CharacterRuleTag(tag: "[", type: .metadataOpen),
+				CharacterRuleTag(tag: "]", type: .metadataClose)
+		], styles: [1 : CharacterStyle.link], metadataLookup: true, definesBoundary: true),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open), otherTags: [
+				CharacterRuleTag(tag: "]", type: .close),
+				CharacterRuleTag(tag: "(", type: .metadataOpen),
+				CharacterRuleTag(tag: ")", type: .metadataClose)
+		], styles: [1 : CharacterStyle.link], metadataLookup: false, definesBoundary: true),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingTags: true, balancedTags: true),
+		CharacterRule(primaryTag:CharacterRuleTag(tag: "~", type: .repeating), otherTags : [], styles: [2 : CharacterStyle.strikethrough], minTags:2 , maxTags:2),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2)
 	]
 	
 	let lineProcessor = SwiftyLineProcessor(rules: SwiftyMarkdown.lineRules, defaultRule: MarkdownLineStyle.body, frontMatterRules: SwiftyMarkdown.frontMatterRules)
@@ -222,17 +247,18 @@ If that is not set, then the system default will be used.
 	
 	var currentType : MarkdownLineStyle = .body
 	
-	
 	var string : String
-	
-	let tagList = "!\\_*`[]()"
-	let validMarkdownTags = CharacterSet(charactersIn: "!\\_*`[]()")
 
 	var orderedListCount = 0
 	var orderedListIndentFirstOrderCount = 0
 	var orderedListIndentSecondOrderCount = 0
 	
+	var previouslyFoundTokens : [Token] = []
 	
+	var applyAttachments = true
+	
+	let perfomanceLog = PerformanceLog(with: "SwiftyMarkdownPerformanceLogging", identifier: "Swifty Markdown", log: .swiftyMarkdownPerformance)
+		
 	/**
 	
 	- parameter string: A string containing [Markdown](https://daringfireball.net/projects/markdown/) syntax to be converted to an NSAttributedString
@@ -347,28 +373,58 @@ If that is not set, then the system default will be used.
 	}
 	
 	
-	
 	/**
 	Generates an NSAttributedString from the string or URL passed at initialisation. Custom fonts or styles are applied to the appropriate elements when this method is called.
 	
 	- returns: An NSAttributedString with the styles applied
 	*/
 	open func attributedString(from markdownString : String? = nil) -> NSAttributedString {
+		
+		self.previouslyFoundTokens.removeAll()
+		self.perfomanceLog.start()
+		
 		if let existentMarkdownString = markdownString {
 			self.string = existentMarkdownString
 		}
 		let attributedString = NSMutableAttributedString(string: "")
 		self.lineProcessor.processEmptyStrings = MarkdownLineStyle.body
 		let foundAttributes : [SwiftyLine] = lineProcessor.process(self.string)
-
-		for (idx, line) in foundAttributes.enumerated() {
+		
+		let references : [SwiftyLine] = foundAttributes.filter({ $0.line.starts(with: "[") && $0.line.contains("]:") })
+		let referencesRemoved : [SwiftyLine] = foundAttributes.filter({ !($0.line.starts(with: "[") && $0.line.contains("]:") ) })
+		var keyValuePairs : [String : String] = [:]
+		for line in references {
+			let strings = line.line.components(separatedBy: "]:")
+			guard strings.count >= 2 else {
+				continue
+			}
+			var key : String = strings[0]
+			if !key.isEmpty {
+				let newstart = key.index(key.startIndex, offsetBy: 1)
+				let range : Range<String.Index> = newstart..<key.endIndex
+				key = String(key[range]).trimmingCharacters(in: .whitespacesAndNewlines)
+			}
+			keyValuePairs[key] = strings[1].trimmingCharacters(in: .whitespacesAndNewlines)
+		}
+		
+		self.perfomanceLog.tag(with: "(line processing complete)")
+		
+		self.tokeniser.metadataLookup = keyValuePairs
+		
+		for (idx, line) in referencesRemoved.enumerated() {
 			if idx > 0 {
 				attributedString.append(NSAttributedString(string: "\n"))
 			}
 			let finalTokens = self.tokeniser.process(line.line)
+			self.previouslyFoundTokens.append(contentsOf: finalTokens)
+			self.perfomanceLog.tag(with: "(tokenising complete for line \(idx)")
+			
 			attributedString.append(attributedStringFor(tokens: finalTokens, in: line))
 			
 		}
+		
+		self.perfomanceLog.end()
+		
 		return attributedString
 	}
 	
@@ -471,6 +527,8 @@ extension SwiftyMarkdown {
 			lineProperties = body
 		case .body:
 			lineProperties = body
+		case .referencedLink:
+			lineProperties = body
 		}
 		
 		if lineProperties.alignment != .left {
@@ -497,16 +555,16 @@ extension SwiftyMarkdown {
 				attributes[.foregroundColor] = self.bold.color
 			}
 			
-			if styles.contains(.link), let url = token.metadataString {
+			if let linkIdx = styles.firstIndex(of: .link), linkIdx < token.metadataStrings.count {
 				attributes[.foregroundColor] = self.link.color
 				attributes[.font] = self.font(for: line, characterOverride: .link)
-				attributes[.link] = url as AnyObject
+				attributes[.link] = token.metadataStrings[linkIdx] as AnyObject
 				
 				if underlineLinks {
 					attributes[.underlineStyle] = NSUnderlineStyle.single.rawValue as AnyObject
 				}
 			}
-			
+						
 			if styles.contains(.strikethrough) {
 				attributes[.font] = self.font(for: line, characterOverride: .strikethrough)
 				attributes[.strikethroughStyle] = NSUnderlineStyle.single.rawValue as AnyObject
@@ -514,15 +572,18 @@ extension SwiftyMarkdown {
 			}
 			
 			#if !os(watchOS)
-			if styles.contains(.image), let imageName = token.metadataString {
+			if let imgIdx = styles.firstIndex(of: .image), imgIdx < token.metadataStrings.count {
+				if !self.applyAttachments {
+					continue
+				}
 				#if !os(macOS)
 				let image1Attachment = NSTextAttachment()
-				image1Attachment.image = UIImage(named: imageName)
+				image1Attachment.image = UIImage(named: token.metadataStrings[imgIdx])
 				let str = NSAttributedString(attachment: image1Attachment)
 				finalAttributedString.append(str)
 				#elseif !os(watchOS)
 				let image1Attachment = NSTextAttachment()
-				image1Attachment.image = NSImage(named: imageName)
+				image1Attachment.image = NSImage(named: token.metadataStrings[imgIdx])
 				let str = NSAttributedString(attachment: image1Attachment)
 				finalAttributedString.append(str)
 				#endif

+ 59 - 0
Sources/SwiftyMarkdown/SwiftyScanner.swift

@@ -0,0 +1,59 @@
+//
+//  SwiftyScanner.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+import os.log
+
+extension OSLog {
+	private static var subsystem = "SwiftyScanner"
+	static let swiftyScannerTokenising = OSLog(subsystem: subsystem, category: "Swifty Scanner Tokenising")
+	static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Peformance")
+}
+
+/// Swifty Scanning Protocol
+public protocol SwiftyScanning {
+	var metadataLookup : [String : String] { get set }
+	func scan( _ string : String, with rule : CharacterRule) -> [Token]
+	func scan( _ tokens : [Token], with rule : CharacterRule) -> [Token]
+}
+
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+class SwiftyScanner : SwiftyScanning {
+	var metadataLookup: [String : String] = [:]
+	
+	init() {
+		
+	}
+	
+	func scan(_ string: String, with rule: CharacterRule) -> [Token] {
+		return []
+	}
+	
+	func scan(_ tokens: [Token], with rule: CharacterRule) -> [Token] {
+		return tokens
+	}
+
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+	
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
+}

+ 565 - 0
Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift

@@ -0,0 +1,565 @@
+//
+//  File.swift
+//  
+//
+//  Created by Simon Fairbairn on 04/04/2020.
+//
+
+//
+//  SwiftyScanner.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+import os.log
+
+
+extension OSLog {
+	private static var subsystem = "SwiftyScanner"
+	static let swiftyScannerScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner")
+	static let swiftyScannerScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance")
+}
+
+enum RepeatingTagType {
+	case open
+	case either
+	case close
+	case neither
+}
+
+struct TagGroup {
+	let groupID  = UUID().uuidString
+	var tagRanges : [ClosedRange<Int>]
+	var tagType : RepeatingTagType = .open
+	var count = 1
+}
+
+class SwiftyScannerNonRepeating {
+	var elements : [Element]
+	let rule : CharacterRule
+	let metadata : [String : String]
+	var pointer : Int = 0
+	
+	var spaceAndNewLine = CharacterSet.whitespacesAndNewlines
+	var tagGroups : [TagGroup] = []
+	
+	var isMetadataOpen = false
+	
+	
+	var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil)
+	
+	let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance)
+	let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScannerScanner)
+		
+	
+	
+	enum Position {
+		case forward(Int)
+		case backward(Int)
+	}
+	
+	init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) {
+		self.elements = elements
+		self.rule = rule
+		self.currentPerfomanceLog.start()
+		self.metadata = metadata
+	}
+	
+	func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? {
+		
+		let newIdx : Int
+		var isForward = true
+		switch newPosition {
+		case .backward(let positions):
+			isForward = false
+			newIdx = pointer - positions
+			if newIdx < 0 {
+				return nil
+			}
+		case .forward(let positions):
+			newIdx = pointer + positions
+			if newIdx >= self.elements.count {
+				return nil
+			}
+		}
+		
+		
+		let range : ClosedRange<Int> = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer
+		return Array(self.elements[range])
+	}
+	
+	
+	func element( for position : Position ) -> Element? {
+		let newIdx : Int
+		switch position {
+		case .backward(let positions):
+			newIdx = pointer - positions
+			if newIdx < 0 {
+				return nil
+			}
+		case .forward(let positions):
+			newIdx = pointer + positions
+			if newIdx >= self.elements.count {
+				return nil
+			}
+		}
+		return self.elements[newIdx]
+	}
+	
+	
+	func positionIsEqualTo( character : Character, direction : Position ) -> Bool {
+		guard let validElement = self.element(for: direction) else {
+			return false
+		}
+		return validElement.character == character
+	}
+	
+	func positionContains( characters : [Character], direction : Position ) -> Bool {
+		guard let validElement = self.element(for: direction) else {
+			return false
+		}
+		return characters.contains(validElement.character)
+	}
+	
+	func isEscaped() -> Bool {
+		let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1))
+		if isEscaped {
+			self.elements[self.pointer - 1].type = .escape
+		}
+		return isEscaped
+	}
+	
+	func range( for tag : String? ) -> ClosedRange<Int>? {
+
+		guard let tag = tag else {
+			return nil
+		}
+		
+		guard let openChar = tag.first else {
+			return nil
+		}
+		
+		if self.pointer == self.elements.count {
+			return nil
+		}
+		
+		if self.elements[self.pointer].character != openChar {
+			return nil
+		}
+		
+		if isEscaped() {
+			return nil
+		}
+		
+		let range : ClosedRange<Int>
+		if tag.count > 1 {
+			guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else {
+				return nil
+			}
+			// If it's already a tag, then it should be ignored
+			if elements.filter({ $0.type != .string }).count > 0 {
+				return nil
+			}
+			if elements.map( { String($0.character) }).joined() != tag {
+				return nil
+			}
+			let endIdx = (self.pointer + tag.count - 1)
+			for i in self.pointer...endIdx {
+				self.elements[i].type = .tag
+			}
+			range = self.pointer...endIdx
+			self.pointer += tag.count
+		} else {
+			// If it's already a tag, then it should be ignored
+			if self.elements[self.pointer].type != .string {
+				return nil
+			}
+			self.elements[self.pointer].type = .tag
+			range = self.pointer...self.pointer
+			self.pointer += 1
+		}
+		return range
+	}
+	
+	
+	func resetTagGroup( withID id : String ) {
+		if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) {
+			for range in self.tagGroups[idx].tagRanges {
+				self.resetTag(in: range)
+			}
+			self.tagGroups.remove(at: idx)
+		}
+		self.isMetadataOpen = false
+	}
+	
+	func resetTag( in range : ClosedRange<Int>) {
+		for idx in range {
+			self.elements[idx].type = .string
+		}
+	}
+	
+	func resetLastTag( for range : inout [ClosedRange<Int>]) {
+		guard let last = range.last else {
+			return
+		}
+		for idx in last {
+			self.elements[idx].type = .string
+		}
+	}
+	
+	func closeTag( _ tag : String, withGroupID id : String ) {
+
+		guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else {
+			return
+		}
+
+		var metadataString = ""
+		if self.isMetadataOpen {
+			let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+			let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+			
+			if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) {
+				if self.enableLog {
+					os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
+				}
+			} else {
+				for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) {
+					self.elements[idx].type = .metadata
+					if self.rule.definesBoundary {
+						self.elements[idx].boundaryCount += 1
+					}
+				}
+				
+				
+				let key = self.elements[metadataOpenRange.upperBound + 1..<metadataCloseRange.lowerBound].map( { String( $0.character )}).joined()
+				if self.rule.metadataLookup {
+					metadataString = self.metadata[key] ?? ""
+				} else {
+					metadataString = key
+				}
+			}
+		}
+		
+		let closeRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+		let openRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+
+		if self.rule.balancedTags && closeRange.count != openRange.count {
+			self.tagGroups[tagIdx].tagRanges.append(openRange)
+			self.tagGroups[tagIdx].tagRanges.append(closeRange)
+			return
+		}
+
+		var shouldRemove = true
+		var styles : [CharacterStyling] = []
+		if openRange.upperBound + 1 == (closeRange.lowerBound) {
+			if self.enableLog {
+				os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
+			}
+		} else {
+			var remainingTags = min(openRange.upperBound - openRange.lowerBound, closeRange.upperBound - closeRange.lowerBound) + 1
+			while remainingTags > 0 {
+				if remainingTags >= self.rule.maxTags {
+					remainingTags -= self.rule.maxTags
+					if let style = self.rule.styles[ self.rule.maxTags ] {
+						if !styles.contains(where: { $0.isEqualTo(style)}) {
+							styles.append(style)
+						}
+					}
+				}
+				if let style = self.rule.styles[remainingTags] {
+					remainingTags -= remainingTags
+					if !styles.contains(where: { $0.isEqualTo(style)}) {
+						styles.append(style)
+					}
+				}
+			}
+			
+			for idx in (openRange.upperBound)...(closeRange.lowerBound) {
+				self.elements[idx].styles.append(contentsOf: styles)
+				self.elements[idx].metadata.append(metadataString)
+				if self.rule.definesBoundary {
+					self.elements[idx].boundaryCount += 1
+				}
+				if self.rule.shouldCancelRemainingRules {
+					self.elements[idx].boundaryCount = 1000
+				}
+			}
+			
+			if self.rule.isRepeatingTag {
+				let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound)
+				switch difference {
+				case 1...:
+					shouldRemove = false
+					self.tagGroups[tagIdx].count = difference
+					self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound )
+				case ...(-1):
+					for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound {
+						self.elements[idx].type = .string
+					}
+				default:
+					break
+				}
+			}
+			
+		}
+		if shouldRemove {
+			self.tagGroups.removeAll(where: { $0.groupID == id })
+		}
+		self.isMetadataOpen = false
+	}
+	
+	func emptyRanges( _ ranges : inout [ClosedRange<Int>] ) {
+		while !ranges.isEmpty {
+			self.resetLastTag(for: &ranges)
+			ranges.removeLast()
+		}
+	}
+	
+	func scanNonRepeatingTags() {
+		var groupID = ""
+		let closeTag = self.rule.tag(for: .close)?.tag
+		let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag
+		let metadataClose = self.rule.tag(for: .metadataClose)?.tag
+		
+		while self.pointer < self.elements.count {
+			if self.enableLog {
+				os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character))
+			}
+			
+			if let range = self.range(for: metadataClose) {
+				if self.isMetadataOpen {
+					guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else {
+						self.pointer += 1
+						continue
+					}
+					
+					guard !self.tagGroups.isEmpty else {
+						self.resetTagGroup(withID: groupID)
+						continue
+					}
+				
+					guard self.isMetadataOpen else {
+						
+						self.resetTagGroup(withID: groupID)
+						continue
+					}
+					if self.enableLog {
+						os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
+					}
+					self.tagGroups[groupIdx].tagRanges.append(range)
+					self.closeTag(closeTag!, withGroupID: groupID)
+					self.isMetadataOpen = false
+					continue
+				} else {
+					self.resetTag(in: range)
+					self.pointer -= metadataClose!.count
+				}
+
+			}
+			
+			if let openRange = self.range(for: self.rule.primaryTag.tag) {
+				if self.isMetadataOpen {
+					self.resetTagGroup(withID: groupID)
+				}
+				
+				let tagGroup = TagGroup(tagRanges: [openRange])
+				groupID = tagGroup.groupID
+				if self.enableLog {
+					os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
+				}
+				if self.rule.isRepeatingTag {
+					
+				}
+				
+				self.tagGroups.append(tagGroup)
+				continue
+			}
+	
+			if let range = self.range(for: closeTag) {
+				guard !self.tagGroups.isEmpty else {
+					if self.enableLog {
+						os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScannerScanner, type:.info)
+					}
+					self.resetTag(in: range)
+					continue
+				}
+				self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
+				groupID = self.tagGroups[self.tagGroups.count - 1].groupID
+				if self.enableLog {
+					os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
+				}
+				guard metadataOpen != nil else {
+					if self.enableLog {
+						os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
+					}
+					self.closeTag(closeTag!, withGroupID: groupID)
+					continue
+				}
+				
+				guard self.pointer != self.elements.count else {
+					continue
+				}
+				
+				guard let range = self.range(for: metadataOpen) else {
+					if self.enableLog {
+						os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
+					}
+					self.resetTagGroup(withID: groupID)
+					continue
+				}
+				self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
+				self.isMetadataOpen = true
+				continue
+			}
+			
+
+			if let range = self.range(for: metadataOpen) {
+				if self.enableLog {
+					os_log("Multiple open metadata tags found!", log: OSLog.swiftyScannerScanner, type:.info , groupID)
+				}
+				self.resetTag(in: range)
+				self.resetTagGroup(withID: groupID)
+				self.isMetadataOpen = false
+				continue
+			}
+			self.pointer += 1
+		}
+	}
+	
+	func scanRepeatingTags() {
+				
+		var groupID = ""
+		let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined()
+		let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters))
+		while self.pointer < self.elements.count {
+			if self.enableLog {
+				os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character))
+			}
+			
+			if var openRange = self.range(for: self.rule.primaryTag.tag) {
+				
+				if self.elements[openRange].first?.boundaryCount == 1000 {
+					self.resetTag(in: openRange)
+					continue
+				}
+				
+				var count = 1
+				var tagType : RepeatingTagType = .open
+				if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1))  {
+					if !unionSet.containsUnicodeScalars(of: prevElement.character) {
+						tagType = .either
+					}
+				} else {
+					tagType = .open
+				}
+				
+				while let nextRange = self.range(for: self.rule.primaryTag.tag)  {
+					count += 1
+					openRange = openRange.lowerBound...nextRange.upperBound
+				}
+				
+				if self.rule.minTags > 1 {
+					if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags {
+						self.resetTag(in: openRange)
+						os_log("Tag does not meet minimum length", log: .swiftyScannerScanner, type: .info)
+						continue
+					}
+				}
+				
+				var validTagGroup = true
+				if let nextElement = self.element(for: .forward(0)) {
+					if unionSet.containsUnicodeScalars(of: nextElement.character) {
+						if tagType == .either {
+							tagType = .close
+						} else {
+							validTagGroup = tagType != .open
+						}
+					}
+				} else {
+					if tagType == .either {
+						tagType = .close
+					} else {
+						validTagGroup = tagType != .open
+					}
+				}
+				
+				if !validTagGroup {
+					if self.enableLog {
+						os_log("Tag has whitespace on both sides", log: .swiftyScannerScanner, type: .info)
+					}
+					self.resetTag(in: openRange)
+					continue
+				}
+				
+				if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) {
+					if tagType == .either {
+						if tagGroups[idx].count == count {
+							self.tagGroups[idx].tagRanges.append(openRange)
+							self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
+							
+							if let last = self.tagGroups.last {
+								groupID = last.groupID
+							}
+							
+							continue
+						}
+					} else {
+						if let prevRange = tagGroups[idx].tagRanges.first {
+							if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount {
+								self.tagGroups[idx].tagRanges.append(openRange)
+								self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
+							}
+						}
+						continue
+					}
+				}
+				var tagGroup = TagGroup(tagRanges: [openRange])
+				groupID = tagGroup.groupID
+				tagGroup.tagType = tagType
+				tagGroup.count = count
+				
+				if self.enableLog {
+					os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner,  type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID)
+				}
+				
+				self.tagGroups.append(tagGroup)
+				continue
+			}
+	
+			self.pointer += 1
+		}
+	}
+	
+	
+	func scan() -> [Element] {
+		
+		guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else {
+			return self.elements
+		}
+		
+		self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)")
+		
+		if self.enableLog {
+			os_log("RULE: %@", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
+		}
+		
+		if self.rule.isRepeatingTag {
+			self.scanRepeatingTags()
+		} else {
+			self.scanNonRepeatingTags()
+		}
+		
+		for tagGroup in self.tagGroups {
+			self.resetTagGroup(withID: tagGroup.groupID)
+		}
+		
+		if self.enableLog {
+			for element in self.elements {
+				print(element)
+			}
+		}
+		return self.elements
+	}
+}

+ 442 - 742
Sources/SwiftyMarkdown/SwiftyTokeniser.swift

@@ -12,116 +12,7 @@ extension OSLog {
 	private static var subsystem = "SwiftyTokeniser"
 	static let tokenising = OSLog(subsystem: subsystem, category: "Tokenising")
 	static let styling = OSLog(subsystem: subsystem, category: "Styling")
-}
-
-// Tag definition
-public protocol CharacterStyling {
-	func isEqualTo( _ other : CharacterStyling ) -> Bool
-}
-
-public enum SpaceAllowed {
-	case no
-	case bothSides
-	case oneSide
-	case leadingSide
-	case trailingSide
-}
-
-public enum Cancel {
-    case none
-    case allRemaining
-    case currentSet
-}
-
-public struct CharacterRule : CustomStringConvertible {
-	public let openTag : String
-	public let intermediateTag : String?
-	public let closingTag : String?
-	public let escapeCharacter : Character?
-	public let styles : [Int : [CharacterStyling]]
-	public var minTags : Int = 1
-	public var maxTags : Int = 1
-	public var spacesAllowed : SpaceAllowed = .oneSide
-	public var cancels : Cancel = .none
-	
-	public var description: String {
-		return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
-	}
-	
-	public init(openTag: String, intermediateTag: String? = nil, closingTag: String? = nil, escapeCharacter: Character? = nil, styles: [Int : [CharacterStyling]] = [:], minTags : Int = 1, maxTags : Int = 1, cancels : Cancel = .none) {
-		self.openTag = openTag
-		self.intermediateTag = intermediateTag
-		self.closingTag = closingTag
-		self.escapeCharacter = escapeCharacter
-		self.styles = styles
-		self.minTags = minTags
-		self.maxTags = maxTags
-		self.cancels = cancels
-	}
-}
-
-// Token definition
-public enum TokenType {
-	case repeatingTag
-	case openTag
-	case intermediateTag
-	case closeTag
-	case string
-	case escape
-	case replacement
-}
-
-
-
-public struct Token {
-	public let id = UUID().uuidString
-	public let type : TokenType
-	public let inputString : String
-	public fileprivate(set) var metadataString : String? = nil
-	public fileprivate(set) var characterStyles : [CharacterStyling] = []
-	public fileprivate(set) var count : Int = 0
-	public fileprivate(set) var shouldSkip : Bool = false
-	public fileprivate(set) var tokenIndex : Int = -1
-	public fileprivate(set) var isProcessed : Bool = false
-	public fileprivate(set) var isMetadata : Bool = false
-	public var outputString : String {
-		get {
-			switch self.type {
-			case .repeatingTag:
-				if count <= 0 {
-					return ""
-				} else {
-					let range = inputString.startIndex..<inputString.index(inputString.startIndex, offsetBy: self.count)
-					return String(inputString[range])
-				}
-			case .openTag, .closeTag, .intermediateTag:
-				return (self.isProcessed || self.isMetadata) ? "" : inputString
-			case .escape, .string:
-				return (self.isProcessed || self.isMetadata) ? "" : inputString
-			case .replacement:
-				return self.inputString
-			}
-		}
-	}
-	public init( type : TokenType, inputString : String, characterStyles : [CharacterStyling] = []) {
-		self.type = type
-		self.inputString = inputString
-		self.characterStyles = characterStyles
-	}
-	
-	func newToken( fromSubstring string: String,  isReplacement : Bool) -> Token {
-		var newToken = Token(type: (isReplacement) ? .replacement : .string , inputString: string, characterStyles: self.characterStyles)
-		newToken.metadataString = self.metadataString
-		newToken.isMetadata = self.isMetadata
-		newToken.isProcessed = self.isProcessed
-		return newToken
-	}
-}
-
-extension Sequence where Iterator.Element == Token {
-    var oslogDisplay: String {
-		return "[\"\(self.map( {  ($0.outputString.isEmpty) ? "\($0.type): \($0.inputString)" : $0.outputString }).joined(separator: "\", \""))\"]"
-    }
+	static let performance = OSLog(subsystem: subsystem, category: "Peformance")
 }
 
 public class SwiftyTokeniser {
@@ -129,9 +20,24 @@ public class SwiftyTokeniser {
 	var replacements : [String : [Token]] = [:]
 	
 	var enableLog = (ProcessInfo.processInfo.environment["SwiftyTokeniserLogging"] != nil)
+	let totalPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Total Run Time", log: OSLog.performance)
+	let currentPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Current", log: OSLog.performance)
+		
+	var scanner : SwiftyScanning!
+	public var metadataLookup : [String : String] = [:]
+	
+	let newlines = CharacterSet.newlines
+	let spaces = CharacterSet.whitespaces
+
 	
 	public init( with rules : [CharacterRule] ) {
 		self.rules = rules
+		
+		self.totalPerfomanceLog.start()
+	}
+	
+	deinit {
+		self.totalPerfomanceLog.end()
 	}
 	
 	
@@ -147,675 +53,469 @@ public class SwiftyTokeniser {
 	///
 	/// - Parameter inputString: A string to have the CharacterRules in `self.rules` applied to
 	public func process( _ inputString : String ) -> [Token] {
+		var currentTokens = [Token(type: .string, inputString: inputString)]
 		guard rules.count > 0 else {
-			return [Token(type: .string, inputString: inputString)]
+			return currentTokens
 		}
-
-		var currentTokens : [Token] = []
 		var mutableRules = self.rules
 		
+		if inputString.isEmpty {
+			return [Token(type: .string, inputString: "", characterStyles: [])]
+		}
 		
-		
-		while !mutableRules.isEmpty {
-			let nextRule = mutableRules.removeFirst()
-			
-			if enableLog {
-				os_log("------------------------------", log: .tokenising, type: .info)
-				os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
-			}
+		self.currentPerfomanceLog.start()
 	
-			if currentTokens.isEmpty {
-				// This means it's the first time through
-				currentTokens = self.applyStyles(to: self.scan(inputString, with: nextRule), usingRule: nextRule)
+		var elementArray : [Element] = []
+		for char in inputString {
+			if newlines.containsUnicodeScalars(of: char) {
+				let element = Element(character: char, type: .newline)
+				elementArray.append(element)
 				continue
 			}
-			
-			var outerStringTokens : [Token] = []
-			var innerStringTokens : [Token] = []
-			var isOuter = true
-			for idx in 0..<currentTokens.count {
-				let nextToken = currentTokens[idx]
-				if nextToken.type == .openTag && nextToken.isProcessed {
-					isOuter = false
-				}
-				if nextToken.type == .closeTag {
-					let ref = UUID().uuidString
-					outerStringTokens.append(Token(type: .replacement, inputString: ref))
-					innerStringTokens.append(nextToken)
-					self.replacements[ref] = self.handleReplacementTokens(innerStringTokens, with: nextRule)
-					innerStringTokens.removeAll()
-					isOuter = true
-					continue
-				}
-				(isOuter) ? outerStringTokens.append(nextToken) : innerStringTokens.append(nextToken)
+			if spaces.containsUnicodeScalars(of: char) {
+				let element = Element(character: char, type: .space)
+				elementArray.append(element)
+				continue
 			}
+			let element = Element(character: char, type: .string)
+			elementArray.append(element)
+		}
+		
+		while !mutableRules.isEmpty {
+			let nextRule = mutableRules.removeFirst()
 			
-			currentTokens = self.handleReplacementTokens(outerStringTokens, with: nextRule)
-			
-			var finalTokens : [Token] = []
-			for token in currentTokens {
-				guard token.type == .replacement else {
-					finalTokens.append(token)
-					continue
-				}
-				if let hasReplacement = self.replacements[token.inputString] {
-					if enableLog {
-						os_log("Found replacement for %@", log: .tokenising, type: .info, token.inputString)
-					}
-					
-					for var repToken in hasReplacement {
-						guard repToken.type == .string else {
-							finalTokens.append(repToken)
-							continue
-						}
-						for style in token.characterStyles {
-							if !repToken.characterStyles.contains(where: { $0.isEqualTo(style)}) {
-								repToken.characterStyles.append(contentsOf: token.characterStyles)
-							}
-						}
-						
-						finalTokens.append(repToken)
-					}
-				}
+			if nextRule.isRepeatingTag {
+				self.scanner = SwiftyScanner()
+				self.scanner.metadataLookup = self.metadataLookup
 			}
-			currentTokens = finalTokens
-			
-			
-			// Each string could have additional tokens within it, so they have to be scanned as well with the current rule.
-			// The one string token might then be exploded into multiple more tokens
-		}
 
-		if enableLog {
-			os_log("=====RULE PROCESSING COMPLETE=====", log: .tokenising, type: .info)
-			os_log("==================================", log: .tokenising, type: .info)
-		}
-		
-		return currentTokens
-	}
-	
-	
-	
-	/// In order to reinsert the original replacements into the new string token, the replacements
-	/// need to be searched for in the incoming string one by one.
-	///
-	/// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles
-	/// are passed over into the newly created tokens.
-	///
-	/// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with
-	/// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens:
-	///
-	/// - `string`: "This string "
-	/// - `replacement`: "AAAAA-BBBBB-CCCCC"
-	/// - `string`: " replacements"
-	///
-	/// Each of these need to have a character style of `bold`.
-	///
-	/// - Parameters:
-	///   - replacements: An array of `replacement` tokens
-	///   - token: The new `string` token that may contain replacement IDs contained in the `replacements` array
-	func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] {
-		guard !stringToken.outputString.isEmpty && !replacements.isEmpty else {
-			return [stringToken]
-		}
-		var outputTokens : [Token] = []
-		let scanner = Scanner(string: stringToken.outputString)
-		scanner.charactersToBeSkipped = nil
-		
-		// Remove any replacements that don't appear in the incoming string
-		var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) })
-		
-		var testString = "\n"
-		while !scanner.isAtEnd {
-			var outputString : String = ""
-			if repTokens.count > 0 {
-				testString = repTokens.removeFirst().inputString
+			if enableLog {
+				os_log("------------------------------", log: .tokenising, type: .info)
+				os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
 			}
+			self.currentPerfomanceLog.tag(with: "(start rule %@)")
 			
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				if let nextString = scanner.scanUpToString(testString) {
-					outputString = nextString
-					outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
-					if let outputToken = scanner.scanString(testString) {
-						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-					}
-				} else if let outputToken = scanner.scanString(testString) {
-					outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-				}
-			} else {
-				var oldString : NSString? = nil
-				var tokenString : NSString? = nil
-				scanner.scanUpTo(testString, into: &oldString)
-				if let nextString = oldString {
-					outputString = nextString as String
-					outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
-					scanner.scanString(testString, into: &tokenString)
-					if let outputToken = tokenString as String? {
-						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-					}
-				} else {
-					scanner.scanString(testString, into: &tokenString)
-					if let outputToken = tokenString as String? {
-						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-					}
-				}
-			}
-		}
-		return outputTokens
-	}
-	
-	
-	/// This function is necessary because a previously tokenised string might have
-	///
-	/// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance.
-	///
-	/// The incoming tokens will look like this:
-	///
-	/// - `string`: "A \*\*Bold"
-	/// - `replacement` : "AAAAA-BBBBB-CCCCC"
-	/// - `string`: " with a trailing string**"
-	///
-	///	However, because the scanner can only tokenise individual strings, passing in the string values
-	///	of these tokens individually and applying the styles will not correctly detect the starting and
-	///	ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC",
-	///	and finally " with a trailing string\*\*")
-	///
-	///	The strings need to be combined, so that they form a single string:
-	///	A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*.
-	///	This string is then parsed and tokenised so that it looks like this:
-	///
-	/// - `string`: "A "
-	///	- `repeatingTag`: "\*\*"
-	///	- `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string"
-	///	- `repeatingTag`: "\*\*"
-	///
-	///	Finally, the replacements from the original incoming token array are searched for and pulled out
-	///	of this new string, so the final result looks like this:
-	///
-	/// - `string`: "A "
-	///	- `repeatingTag`: "\*\*"
-	///	- `string`: "Bold "
-	///	- `replacement`: "AAAAA-BBBBB-CCCCC"
-	///	- `string`: " with a trailing string"
-	///	- `repeatingTag`: "\*\*"
-	///
-	/// - Parameters:
-	///   - tokens: The tokens to be combined, scanned, re-tokenised, and merged
-	///   - rule: The character rule currently being applied
-	func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
-		guard tokens.count > 0 else {
-			return []
+			let scanner = SwiftyScannerNonRepeating(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup)
+			elementArray = scanner.scan()
 		}
 		
-		let combinedString = tokens.map({ $0.outputString }).joined()
+		var output : [Token] = []
 		
-		let nextTokens = self.scan(combinedString, with: rule)
-		var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
 		
-		/// It's necessary here to check to see if the first token (which will always represent the styles
-		/// to be applied from previous scans) has any existing metadata or character styles and apply them
-		/// to *all* the string and replacement tokens found by the new scan.
-		for idx in 0..<replacedTokens.count {
-			guard replacedTokens[idx].type == .string || replacedTokens[idx].type == .replacement else {
-				continue
-			}
-			if tokens.first!.metadataString != nil && replacedTokens[idx].metadataString == nil {
-				replacedTokens[idx].metadataString = tokens.first!.metadataString
+		func empty( _ string : inout String, into tokens : inout [Token] )  {
+			guard !string.isEmpty else {
+				return
 			}
-			replacedTokens[idx].characterStyles.append(contentsOf: tokens.first!.characterStyles)
+			var token = Token(type: .string, inputString: string)
+			token.metadataStrings.append(contentsOf: lastElement.metadata) 
+			token.characterStyles = lastElement.styles
+			string.removeAll()
+			tokens.append(token)
 		}
 		
-		// Swap the original replacement tokens back in
-		let replacements = tokens.filter({ $0.type == .replacement })
-		var outputTokens : [Token] = []
-		for token in replacedTokens {
-			guard token.type == .string else {
-				outputTokens.append(token)
-				continue
-			}
-			outputTokens.append(contentsOf: self.reinsertReplacements(replacements, from: token))
-		}
 		
-		return outputTokens
-	}
-	
-	
-	
-	/// This function ensures that only concurrent `string` and `replacement` tokens are processed together.
-	///
-	/// i.e. If there is an existing `repeatingTag` token between two strings, then those strings will be
-	/// processed individually. This prevents incorrect parsing of strings like "\*\*\_Should only be bold\*\*\_"
-	///
-	/// - Parameters:
-	///   - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
-	///   - rule: The current rule being processed
-	func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
-	
-		// Only combine string and replacements that are next to each other.
-		var newTokenSet : [Token] = []
-		var currentTokenSet : [Token] = []
-		for i in 0..<incomingTokens.count {
-			guard incomingTokens[i].type == .string || incomingTokens[i].type == .replacement else {
-				newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-				newTokenSet.append(incomingTokens[i])
-				currentTokenSet.removeAll()
+		var lastElement = elementArray.first!
+		var accumulatedString = ""
+		for element in elementArray {
+			guard element.type != .escape else {
 				continue
 			}
-			guard !incomingTokens[i].isProcessed && !incomingTokens[i].isMetadata && !incomingTokens[i].shouldSkip else {
-				newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-				newTokenSet.append(incomingTokens[i])
-				currentTokenSet.removeAll()
+			
+			guard element.type == .string || element.type == .space || element.type == .newline else {
+				empty(&accumulatedString, into: &output)
 				continue
 			}
-			currentTokenSet.append(incomingTokens[i])
-		}
-		newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-
-		return newTokenSet
-	}
-	
-	
-	func handleClosingTagFromOpenTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule ) {
-		
-		guard rule.closingTag != nil else {
-			return
-		}
-		guard let closeTokenIdx = tokens.firstIndex(where: { $0.type == .closeTag && !$0.isProcessed }) else {
-			return
-		}
-		
-		var metadataIndex = index
-		// If there's an intermediate tag, get the index of that
-		if rule.intermediateTag != nil {
-			guard let nextTokenIdx = tokens.firstIndex(where: { $0.type == .intermediateTag  && !$0.isProcessed }) else {
-				return
-			}
-			metadataIndex = nextTokenIdx
-			let styles : [CharacterStyling] = rule.styles[1] ?? []
-			for i in index..<nextTokenIdx {
-				for style in styles {
-					if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
-						tokens[i].characterStyles.append(style)
-					}
-				}
-			}
-		}
-
-		var metadataString : String = ""
-		for i in metadataIndex..<closeTokenIdx {
-			if tokens[i].type == .string {
-				metadataString.append(tokens[i].outputString)
-				tokens[i].isMetadata = true
+			if lastElement.styles as? [CharacterStyle] != element.styles as? [CharacterStyle] {
+				empty(&accumulatedString, into: &output)
 			}
+			accumulatedString.append(element.character)
+			lastElement = element
 		}
+		empty(&accumulatedString, into: &output)
 		
-		for i in index..<metadataIndex {
-			if tokens[i].type == .string {
-				tokens[i].metadataString = metadataString
-			}
-		}
-		
-		tokens[closeTokenIdx].isProcessed = true
-		tokens[metadataIndex].isProcessed = true
-		tokens[index].isProcessed = true
-	}
-	
-	
-	/// This is here to manage how opening tags are matched with closing tags when they're all the same
-	/// character.
-	///
-	/// Of course, because Markdown is about as loose as a spec can be while still being considered any
-	/// kind of spec, the number of times this character repeats causes different effects. Then there
-	/// is the ill-defined way it should work if the number of opening and closing tags are different.
-	///
-	/// - Parameters:
-	///   - index: The index of the current token in the loop
-	///   - tokens: An inout variable of the loop tokens of interest
-	///   - rule: The character rule being applied
-	func handleClosingTagFromRepeatingTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule) {
-		let theToken = tokens[index]
+		self.currentPerfomanceLog.tag(with: "(finished all rules)")
 		
 		if enableLog {
-		os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
-		}
-		
-		guard theToken.count > 0 else {
-			return
-		}
-		
-		let startIdx = index
-		var endIdx : Int? = nil
-		
-		let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
-		// Try to find exact match first
-		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed }) {
-			endIdx = nextTokenIdx
-		}
-		
-		if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id  && !$0.isProcessed }) {
-			endIdx = nextTokenIdx
-		}
-		guard let existentEnd = endIdx else {
-			return
-		}
-		
-		
-		let styles : [CharacterStyling] = rule.styles[maxCount] ?? []
-		for i in startIdx..<existentEnd {
-			for style in styles {
-				if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
-					tokens[i].characterStyles.append(style)
-				}
-			}
-			if rule.cancels == .allRemaining {
-				tokens[i].shouldSkip = true
-			}
-		}
-		
-		let maxEnd = (tokens[existentEnd].count > rule.maxTags) ? rule.maxTags : tokens[existentEnd].count
-		tokens[index].count = theToken.count - maxEnd
-		tokens[existentEnd].count = tokens[existentEnd].count - maxEnd
-		if maxEnd < rule.maxTags {
-			self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule)
-		} else {
-			tokens[existentEnd].isProcessed = true
-			tokens[index].isProcessed = true
+			os_log("=====RULE PROCESSING COMPLETE=====", log: .tokenising, type: .info)
+			os_log("==================================", log: .tokenising, type: .info)
 		}
-		
-		
+		return output
 	}
 	
-	func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] {
-		var mutableTokens : [Token] = tokens
-		
-		if enableLog {
-			os_log("Applying styles to tokens: %@", log: .tokenising, type: .info,  tokens.oslogDisplay )
-		}
-		for idx in 0..<mutableTokens.count {
-			let token = mutableTokens[idx]
-			switch token.type {
-			case .escape:
-			if enableLog {
-				os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
-			}
-			case .repeatingTag:
-				self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
-			case .openTag:
-				let theToken = mutableTokens[idx]
-				if enableLog {
-					os_log("Found repeating tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
-				}
-								
-				guard rule.closingTag != nil else {
-					
-					// If there's an intermediate tag, get the index of that
-					
-					// Get the index of the closing tag
-					
-					continue
-				}
-				self.handleClosingTagFromOpenTag(withIndex: idx, in: &mutableTokens, following: rule)
-				
-				
-			case .intermediateTag:
-				let theToken = mutableTokens[idx]
-				if enableLog {
-					os_log("Found intermediate tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
-				}
-				
-			case .closeTag:
-				let theToken = mutableTokens[idx]
-					if enableLog {
-						os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
-				}
-				
-			case .string:
-				let theToken = mutableTokens[idx]
-				if enableLog {
-					if theToken.isMetadata {
-						os_log("Found Metadata: %@", log: .tokenising, type: .info, theToken.inputString )
-					} else {
-						os_log("Found String: %@", log: .tokenising, type: .info, theToken.inputString )
-					}
-					if let hasMetadata = theToken.metadataString {
-						os_log("...with metadata: %@", log: .tokenising, type: .info, hasMetadata )
-					}
-				}
-				
-			case .replacement:
-				if enableLog {
-					os_log("Found replacement with ID: %@", log: .tokenising, type: .info, mutableTokens[idx].inputString )
-				}
-			}
-		}
-		return mutableTokens
-	}
 	
+//
+//
+//	/// In order to reinsert the original replacements into the new string token, the replacements
+//	/// need to be searched for in the incoming string one by one.
+//	///
+//	/// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles
+//	/// are passed over into the newly created tokens.
+//	///
+//	/// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with
+//	/// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens:
+//	///
+//	/// - `string`: "This string "
+//	/// - `replacement`: "AAAAA-BBBBB-CCCCC"
+//	/// - `string`: " replacements"
+//	///
+//	/// Each of these need to have a character style of `bold`.
+//	///
+//	/// - Parameters:
+//	///   - replacements: An array of `replacement` tokens
+//	///   - token: The new `string` token that may contain replacement IDs contained in the `replacements` array
+//	func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] {
+//		guard !stringToken.outputString.isEmpty && !replacements.isEmpty else {
+//			return [stringToken]
+//		}
+//		var outputTokens : [Token] = []
+//		let scanner = Scanner(string: stringToken.outputString)
+//		scanner.charactersToBeSkipped = nil
+//
+//		// Remove any replacements that don't appear in the incoming string
+//		var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) })
+//
+//		var testString = "\n"
+//		while !scanner.isAtEnd {
+//			var outputString : String = ""
+//			if repTokens.count > 0 {
+//				testString = repTokens.removeFirst().inputString
+//			}
+//
+//			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+//				if let nextString = scanner.scanUpToString(testString) {
+//					outputString = nextString
+//					outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
+//					if let outputToken = scanner.scanString(testString) {
+//						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
+//					}
+//				} else if let outputToken = scanner.scanString(testString) {
+//					outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
+//				}
+//			} else {
+//				var oldString : NSString? = nil
+//				var tokenString : NSString? = nil
+//				scanner.scanUpTo(testString, into: &oldString)
+//				if let nextString = oldString {
+//					outputString = nextString as String
+//					outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
+//					scanner.scanString(testString, into: &tokenString)
+//					if let outputToken = tokenString as String? {
+//						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
+//					}
+//				} else {
+//					scanner.scanString(testString, into: &tokenString)
+//					if let outputToken = tokenString as String? {
+//						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
+//					}
+//				}
+//			}
+//		}
+//		return outputTokens
+//	}
+//
+//
+//	/// This function is necessary because a previously tokenised string might have
+//	///
+//	/// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance.
+//	///
+//	/// The incoming tokens will look like this:
+//	///
+//	/// - `string`: "A \*\*Bold"
+//	/// - `replacement` : "AAAAA-BBBBB-CCCCC"
+//	/// - `string`: " with a trailing string**"
+//	///
+//	///	However, because the scanner can only tokenise individual strings, passing in the string values
+//	///	of these tokens individually and applying the styles will not correctly detect the starting and
+//	///	ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC",
+//	///	and finally " with a trailing string\*\*")
+//	///
+//	///	The strings need to be combined, so that they form a single string:
+//	///	A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*.
+//	///	This string is then parsed and tokenised so that it looks like this:
+//	///
+//	/// - `string`: "A "
+//	///	- `repeatingTag`: "\*\*"
+//	///	- `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string"
+//	///	- `repeatingTag`: "\*\*"
+//	///
+//	///	Finally, the replacements from the original incoming token array are searched for and pulled out
+//	///	of this new string, so the final result looks like this:
+//	///
+//	/// - `string`: "A "
+//	///	- `repeatingTag`: "\*\*"
+//	///	- `string`: "Bold "
+//	///	- `replacement`: "AAAAA-BBBBB-CCCCC"
+//	///	- `string`: " with a trailing string"
+//	///	- `repeatingTag`: "\*\*"
+//	///
+//	/// - Parameters:
+//	///   - tokens: The tokens to be combined, scanned, re-tokenised, and merged
+//	///   - rule: The character rule currently being applied
+//	func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
+//		guard tokens.count > 0 else {
+//			return []
+//		}
+//
+//		let combinedString = tokens.map({ $0.outputString }).joined()
+//
+//		let nextTokens = self.scanner.scan(combinedString, with: rule)
+//		var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
+//
+//		/// It's necessary here to check to see if the first token (which will always represent the styles
+//		/// to be applied from previous scans) has any existing metadata or character styles and apply them
+//		/// to *all* the string and replacement tokens found by the new scan.
+//		for idx in 0..<replacedTokens.count {
+//			guard replacedTokens[idx].type == .string || replacedTokens[idx].type == .replacement else {
+//				continue
+//			}
+//			if tokens.first!.metadataString != nil && replacedTokens[idx].metadataString == nil {
+//				replacedTokens[idx].metadataString = tokens.first!.metadataString
+//			}
+//			replacedTokens[idx].characterStyles.append(contentsOf: tokens.first!.characterStyles)
+//		}
+//
+//		// Swap the original replacement tokens back in
+//		let replacements = tokens.filter({ $0.type == .replacement })
+//		var outputTokens : [Token] = []
+//		for token in replacedTokens {
+//			guard token.type == .string else {
+//				outputTokens.append(token)
+//				continue
+//			}
+//			outputTokens.append(contentsOf: self.reinsertReplacements(replacements, from: token))
+//		}
+//
+//		return outputTokens
+//	}
+//
+//
+//
+//	/// This function ensures that only concurrent `string` and `replacement` tokens are processed together.
+//	///
+//	/// i.e. If there is an existing `repeatingTag` token between two strings, then those strings will be
+//	/// processed individually. This prevents incorrect parsing of strings like "\*\*\_Should only be bold\*\*\_"
+//	///
+//	/// - Parameters:
+//	///   - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
+//	///   - rule: The current rule being processed
+//	func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
+//
+//		// Only combine string and replacements that are next to each other.
+//		var newTokenSet : [Token] = []
+//		var currentTokenSet : [Token] = []
+//		for i in 0..<incomingTokens.count {
+//			guard incomingTokens[i].type == .string || incomingTokens[i].type == .replacement else {
+//				newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
+//				newTokenSet.append(incomingTokens[i])
+//				currentTokenSet.removeAll()
+//				continue
+//			}
+//			guard !incomingTokens[i].isProcessed && !incomingTokens[i].isMetadata && !incomingTokens[i].shouldSkip else {
+//				newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
+//				newTokenSet.append(incomingTokens[i])
+//				currentTokenSet.removeAll()
+//				continue
+//			}
+//			currentTokenSet.append(incomingTokens[i])
+//		}
+//		newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
+//
+//		return newTokenSet
+//	}
+//
+//
+//	func handleClosingTagFromOpenTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule ) {
+//
+//		guard rule.closeTag != nil else {
+//			return
+//		}
+//		guard let closeTokenIdx = tokens.firstIndex(where: { $0.type == .closeTag && !$0.isProcessed }) else {
+//			return
+//		}
+//
+//		var metadataIndex = index
+//		// If there's an intermediate tag, get the index of that
+//		if rule.intermediateTag != nil {
+//			guard let nextTokenIdx = tokens.firstIndex(where: { $0.type == .intermediateTag  && !$0.isProcessed }) else {
+//				return
+//			}
+//			metadataIndex = nextTokenIdx
+//			let styles : [CharacterStyling] = rule.styles[1] ?? []
+//			for i in index..<nextTokenIdx {
+//				for style in styles {
+//					if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
+//						tokens[i].characterStyles.append(style)
+//					}
+//				}
+//			}
+//		}
+//
+//		var metadataString : String = ""
+//		for i in metadataIndex..<closeTokenIdx {
+//			if tokens[i].type == .string {
+//				metadataString.append(tokens[i].outputString)
+//				tokens[i].isMetadata = true
+//			}
+//		}
+//
+//		for i in index..<metadataIndex {
+//			if tokens[i].type == .string {
+//				tokens[i].metadataString = metadataString
+//			}
+//		}
+//
+//		tokens[closeTokenIdx].isProcessed = true
+//		tokens[metadataIndex].isProcessed = true
+//		tokens[index].isProcessed = true
+//	}
+//
+//
+//	/// This is here to manage how opening tags are matched with closing tags when they're all the same
+//	/// character.
+//	///
+//	/// Of course, because Markdown is about as loose as a spec can be while still being considered any
+//	/// kind of spec, the number of times this character repeats causes different effects. Then there
+//	/// is the ill-defined way it should work if the number of opening and closing tags are different.
+//	///
+//	/// - Parameters:
+//	///   - index: The index of the current token in the loop
+//	///   - tokens: An inout variable of the loop tokens of interest
+//	///   - rule: The character rule being applied
+//	func handleClosingTagFromRepeatingTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule) {
+//		let theToken = tokens[index]
+//
+//		if enableLog {
+//			os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
+//		}
+//
+//		guard theToken.count > 0 else {
+//			return
+//		}
+//
+//		let startIdx = index
+//		var endIdx : Int? = nil
+//
+//		let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
+//		// Try to find exact match first
+//		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
+//			endIdx = nextTokenIdx
+//		}
+//
+//		if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id  && !$0.isProcessed }) {
+//			endIdx = nextTokenIdx
+//		}
+//		guard let existentEnd = endIdx else {
+//			return
+//		}
+//
+//
+//		let styles : [CharacterStyling] = rule.styles[maxCount] ?? []
+//		for i in startIdx..<existentEnd {
+//			for style in styles {
+//				if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
+//					tokens[i].characterStyles.append(style)
+//				}
+//			}
+//			if rule.cancels == .allRemaining {
+//				tokens[i].shouldSkip = true
+//			}
+//		}
+//
+//		let maxEnd = (tokens[existentEnd].count > rule.maxTags) ? rule.maxTags : tokens[existentEnd].count
+//		tokens[index].count = theToken.count - maxEnd
+//		tokens[existentEnd].count = tokens[existentEnd].count - maxEnd
+//		if maxEnd < rule.maxTags {
+//			self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule)
+//		} else {
+//			tokens[existentEnd].isProcessed = true
+//			tokens[index].isProcessed = true
+//		}
+//
+//
+//	}
+//
+//	func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] {
+//		var mutableTokens : [Token] = tokens
+//
+//		if enableLog {
+//			os_log("Applying styles to tokens: %@", log: .tokenising, type: .info,  tokens.oslogDisplay )
+//		}
+//		for idx in 0..<mutableTokens.count {
+//			let token = mutableTokens[idx]
+//			switch token.type {
+//			case .escape:
+//				if enableLog {
+//					os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
+//				}
+//			case .repeatingTag:
+//				let theToken = mutableTokens[idx]
+//				self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
+//				if enableLog {
+//					os_log("Found repeating tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
+//				}
+//			case .openTag:
+//				let theToken = mutableTokens[idx]
+//				if enableLog {
+//					os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
+//				}
+//
+//				guard rule.closingTag != nil else {
+//
+//					// If there's an intermediate tag, get the index of that
+//
+//					// Get the index of the closing tag
+//
+//					continue
+//				}
+//				self.handleClosingTagFromOpenTag(withIndex: idx, in: &mutableTokens, following: rule)
+//
+//
+//			case .intermediateTag:
+//				let theToken = mutableTokens[idx]
+//				if enableLog {
+//					os_log("Found intermediate tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
+//				}
+//
+//			case .closeTag:
+//				let theToken = mutableTokens[idx]
+//				if enableLog {
+//					os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
+//				}
+//
+//			case .string:
+//				let theToken = mutableTokens[idx]
+//				if enableLog {
+//					if theToken.isMetadata {
+//						os_log("Found Metadata: %@", log: .tokenising, type: .info, theToken.inputString )
+//					} else {
+//						os_log("Found String: %@", log: .tokenising, type: .info, theToken.inputString )
+//					}
+//					if let hasMetadata = theToken.metadataString {
+//						os_log("...with metadata: %@", log: .tokenising, type: .info, hasMetadata )
+//					}
+//				}
+//
+//			case .replacement:
+//				if enableLog {
+//					os_log("Found replacement with ID: %@", log: .tokenising, type: .info, mutableTokens[idx].inputString )
+//				}
+//			}
+//		}
+//		return mutableTokens
+//	}
+//
+//
+//
 	
-	func scan( _ string : String, with rule : CharacterRule) -> [Token] {
-		let scanner = Scanner(string: string)
-		scanner.charactersToBeSkipped = nil
-		var tokens : [Token] = []
-		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
-		if let existentEscape = rule.escapeCharacter {
-			set.insert(charactersIn: String(existentEscape))
-		}
-		
-		var openTagFound = false
-		var openingString = ""
-		while !scanner.isAtEnd {
-			
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				if let start = scanner.scanUpToCharacters(from: set) {
-					openingString.append(start)
-				}
-			} else {
-				var string : NSString?
-				scanner.scanUpToCharacters(from: set, into: &string)
-				if let existentString = string as String? {
-					openingString.append(existentString)
-				}
-				// Fallback on earlier versions
-			}
-			
-			let lastChar : String?
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				lastChar = ( scanner.currentIndex > string.startIndex ) ? String(string[string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
-			} else {
-				if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
-					lastChar = ( scanLocation > string.startIndex ) ? String(string[string.index(before: scanLocation)..<scanLocation]) : nil
-				} else {
-					lastChar = nil
-				}
-				
-			}
-			let maybeFoundChars : String?
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				maybeFoundChars = scanner.scanCharacters(from: set )
-			} else {
-				var string : NSString?
-				scanner.scanCharacters(from: set, into: &string)
-				maybeFoundChars = string as String?
-			}
-			
-			let nextChar : String?
-			if #available(iOS 13.0, OSX 10.15,  watchOS 6.0,tvOS 13.0, *) {
-				 nextChar = (scanner.currentIndex != string.endIndex) ? String(string[scanner.currentIndex]) : nil
-			} else {
-				if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
-					nextChar = (scanLocation != string.endIndex) ? String(string[scanLocation]) : nil
-				} else {
-					nextChar = nil
-				}
-				
-			}
-			
-			guard let foundChars = maybeFoundChars else {
-				tokens.append(Token(type: .string, inputString: "\(openingString)"))
-				openingString = ""
-				continue
-			}
-			
-			if foundChars == rule.openTag && foundChars.count < rule.minTags {
-				openingString.append(foundChars)
-				continue
-			}
-			
-			if !validateSpacing(nextCharacter: nextChar, previousCharacter: lastChar, with: rule) {
-				let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
-				var escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
-				if let hasIntermediateTag = rule.intermediateTag {
-					escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
-				}
-				if let existentClosingTag = rule.closingTag {
-					escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
-				}
-				
-				openingString.append(escaped)
-				continue
-			}
+}
 
-			var cumulativeString = ""
-			var openString = ""
-			var intermediateString = ""
-			var closedString = ""
-			var maybeEscapeNext = false
-			
-			
-			func addToken( for type : TokenType ) {
-				var inputString : String
-				switch type {
-				case .openTag:
-					inputString = openString
-				case .intermediateTag:
-					inputString = intermediateString
-				case .closeTag:
-					inputString = closedString
-				default:
-					inputString = ""
-				}
-				guard !inputString.isEmpty else {
-					return
-				}
-				guard inputString.count >= rule.minTags else {
-					return
-				}
-				
-				if !openingString.isEmpty {
-					tokens.append(Token(type: .string, inputString: "\(openingString)"))
-					openingString = ""
-				}
-				let actualType : TokenType = ( rule.intermediateTag == nil && rule.closingTag == nil ) ? .repeatingTag : type
-				
-				var token = Token(type: actualType, inputString: inputString)
-				if rule.closingTag == nil {
-					token.count = inputString.count
-				}
-				
-				tokens.append(token)
-				
-				switch type {
-				case .openTag:
-					openString = ""
-				case .intermediateTag:
-					intermediateString = ""
-				case .closeTag:
-					closedString = ""
-				default:
-					break
-				}
-			}
-			
-			// Here I am going through and adding the characters in the found set to a cumulative string.
-			// If there is an escape character, then the loop stops and any open tags are tokenised.
-			for char in foundChars {
-				cumulativeString.append(char)
-				if maybeEscapeNext {
-					
-					var escaped = cumulativeString
-					if String(char) == rule.openTag || String(char) == rule.intermediateTag || String(char) == rule.closingTag {
-						escaped = String(cumulativeString.replacingOccurrences(of: String(rule.escapeCharacter ?? Character("")), with: ""))
-					}
-					
-					openingString.append(escaped)
-					cumulativeString = ""
-					maybeEscapeNext = false
-				}
-				if let existentEscape = rule.escapeCharacter {
-					if cumulativeString == String(existentEscape) {
-						maybeEscapeNext = true
-						addToken(for: .openTag)
-						addToken(for: .intermediateTag)
-						addToken(for: .closeTag)
-						continue
-					}
-				}
-				
-				
-				if cumulativeString == rule.openTag {
-					openString.append(char)
-					cumulativeString = ""
-					openTagFound = true
-				} else if cumulativeString == rule.intermediateTag, openTagFound {
-					intermediateString.append(cumulativeString)
-					cumulativeString = ""
-				} else if cumulativeString == rule.closingTag, openTagFound {
-					closedString.append(char)
-					cumulativeString = ""
-					openTagFound = false
-				}
-			}
-			// If we're here, it means that an escape character was found but without a corresponding
-			// tag, which means it might belong to a different rule.
-			// It should be added to the next group of regular characters
-			
-			addToken(for: .openTag)
-			addToken(for: .intermediateTag)
-			addToken(for: .closeTag)
-			openingString.append( cumulativeString )
-		}
-		
-		if !openingString.isEmpty {
-			tokens.append(Token(type: .string, inputString: "\(openingString)"))
-		}
-		
-		return tokens
-	}
-	
-	func validateSpacing( nextCharacter : String?, previousCharacter : String?, with rule : CharacterRule ) -> Bool {
-		switch rule.spacesAllowed {
-		case .leadingSide:
-			guard nextCharacter != nil else {
-				return true
-			}
-			if nextCharacter == " "  {
-				return false
-			}
-		case .trailingSide:
-			guard previousCharacter != nil else {
-				return true
-			}
-			if previousCharacter == " " {
-				return false
-			}
-		case .no:
-			switch (previousCharacter, nextCharacter) {
-			case (nil, nil), ( " ", _ ), (  _, " " ):
-				return false
-			default:
-				return true
-			}
-		
-		case .oneSide:
-			switch (previousCharacter, nextCharacter) {
-			case  (nil, " " ), (" ", nil), (" ", " " ):
-				return false
-			default:
-				return true
-			}
-		default:
-			break
+
+extension String {
+	func repeating( _ max : Int ) -> String {
+		var output = self
+		for _ in 1..<max {
+			output += self
 		}
-		return true
+		return output
 	}
-	
 }

+ 81 - 0
Sources/SwiftyMarkdown/Token.swift

@@ -0,0 +1,81 @@
+//
+//  Token.swift
+//  SwiftyMarkdown
+//
+//  Created by Simon Fairbairn on 04/02/2020.
+//
+
+import Foundation
+
+// Tag definition
+public protocol CharacterStyling {
+	func isEqualTo( _ other : CharacterStyling ) -> Bool
+}
+
+// Token definition
+public enum TokenType {
+	case repeatingTag
+	case openTag
+	case intermediateTag
+	case closeTag
+	case string
+	case escape
+	case replacement
+}
+
+public struct Token {
+	public let id = UUID().uuidString
+	public let type : TokenType
+	public let inputString : String
+	public var metadataStrings : [String] = []
+	public internal(set) var group : Int = 0
+	public internal(set) var characterStyles : [CharacterStyling] = []
+	public internal(set) var count : Int = 0
+	public internal(set) var shouldSkip : Bool = false
+	public internal(set) var tokenIndex : Int = -1
+	public internal(set) var isProcessed : Bool = false
+	public internal(set) var isMetadata : Bool = false
+	public var children : [Token] = []
+	
+	public var outputString : String {
+		get {
+			switch self.type {
+			case .repeatingTag:
+				if count <= 0 {
+					return ""
+				} else {
+					let range = inputString.startIndex..<inputString.index(inputString.startIndex, offsetBy: self.count)
+					return String(inputString[range])
+				}
+			case .openTag, .closeTag, .intermediateTag:
+				return (self.isProcessed || self.isMetadata) ? "" : inputString
+			case .escape, .string:
+				return (self.isProcessed || self.isMetadata) ? "" : inputString
+			case .replacement:
+				return self.inputString
+			}
+		}
+	}
+	public init( type : TokenType, inputString : String, characterStyles : [CharacterStyling] = []) {
+		self.type = type
+		self.inputString = inputString
+		self.characterStyles = characterStyles
+		if type == .repeatingTag {
+			self.count = inputString.count
+		}
+	}
+	
+	func newToken( fromSubstring string: String,  isReplacement : Bool) -> Token {
+		var newToken = Token(type: (isReplacement) ? .replacement : .string , inputString: string, characterStyles: self.characterStyles)
+		newToken.metadataStrings = self.metadataStrings
+		newToken.isMetadata = self.isMetadata
+		newToken.isProcessed = self.isProcessed
+		return newToken
+	}
+}
+
+extension Sequence where Iterator.Element == Token {
+	var oslogDisplay: String {
+		return "[\"\(self.map( {  ($0.outputString.isEmpty) ? "\($0.type): \($0.inputString)" : $0.outputString }).joined(separator: "\", \""))\"]"
+	}
+}

+ 560 - 717
SwiftyMarkdown.xcodeproj/project.pbxproj

@@ -1,720 +1,563 @@
 // !$*UTF8*$!
 {
-   archiveVersion = "1";
-   objectVersion = "46";
-   objects = {
-      "OBJ_1" = {
-         isa = "PBXProject";
-         attributes = {
-            LastSwiftMigration = "9999";
-            LastUpgradeCheck = "9999";
-         };
-         buildConfigurationList = "OBJ_2";
-         compatibilityVersion = "Xcode 3.2";
-         developmentRegion = "en";
-         hasScannedForEncodings = "0";
-         knownRegions = (
-            "en"
-         );
-         mainGroup = "OBJ_5";
-         productRefGroup = "OBJ_23";
-         projectDirPath = ".";
-         targets = (
-            "SwiftyMarkdown::SwiftyMarkdown",
-            "SwiftyMarkdown::SwiftPMPackageDescription",
-            "SwiftyMarkdown::SwiftyMarkdownPackageTests::ProductTarget",
-            "SwiftyMarkdown::SwiftyMarkdownTests"
-         );
-      };
-      "OBJ_10" = {
-         isa = "PBXFileReference";
-         path = "SwiftyLineProcessor.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_11" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdown+iOS.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_12" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdown+macOS.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_13" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdown.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_14" = {
-         isa = "PBXFileReference";
-         path = "SwiftyTokeniser.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_15" = {
-         isa = "PBXGroup";
-         children = (
-            "OBJ_16"
-         );
-         name = "Tests";
-         path = "";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_16" = {
-         isa = "PBXGroup";
-         children = (
-            "OBJ_17",
-            "OBJ_18",
-            "OBJ_19",
-            "OBJ_20",
-            "OBJ_21",
-            "OBJ_22"
-         );
-         name = "SwiftyMarkdownTests";
-         path = "Tests/SwiftyMarkdownTests";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_17" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdownAttributedStringTests copy.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_18" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdownCharacterTests.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_19" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdownLineTests.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_2" = {
-         isa = "XCConfigurationList";
-         buildConfigurations = (
-            "OBJ_3",
-            "OBJ_4"
-         );
-         defaultConfigurationIsVisible = "0";
-         defaultConfigurationName = "Release";
-      };
-      "OBJ_20" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdownLinkTests.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_21" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdownPerformanceTests.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_22" = {
-         isa = "PBXFileReference";
-         path = "XCTest+SwiftyMarkdown.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_23" = {
-         isa = "PBXGroup";
-         children = (
-            "SwiftyMarkdown::SwiftyMarkdown::Product",
-            "SwiftyMarkdown::SwiftyMarkdownTests::Product"
-         );
-         name = "Products";
-         path = "";
-         sourceTree = "BUILT_PRODUCTS_DIR";
-      };
-      "OBJ_26" = {
-         isa = "PBXFileReference";
-         path = "Playground";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_27" = {
-         isa = "PBXFileReference";
-         path = "Example";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_28" = {
-         isa = "PBXFileReference";
-         path = "Resources";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_29" = {
-         isa = "PBXFileReference";
-         path = "fastlane";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_3" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            CLANG_ENABLE_OBJC_ARC = "YES";
-            COMBINE_HIDPI_IMAGES = "YES";
-            COPY_PHASE_STRIP = "NO";
-            DEBUG_INFORMATION_FORMAT = "dwarf";
-            DYLIB_INSTALL_NAME_BASE = "@rpath";
-            ENABLE_NS_ASSERTIONS = "YES";
-            GCC_OPTIMIZATION_LEVEL = "0";
-            GCC_PREPROCESSOR_DEFINITIONS = (
-               "$(inherited)",
-               "SWIFT_PACKAGE=1",
-               "DEBUG=1"
-            );
-            MACOSX_DEPLOYMENT_TARGET = "10.10";
-            ONLY_ACTIVE_ARCH = "YES";
-            OTHER_SWIFT_FLAGS = (
-               "$(inherited)",
-               "-DXcode"
-            );
-            PRODUCT_NAME = "$(TARGET_NAME)";
-            SDKROOT = "macosx";
-            SUPPORTED_PLATFORMS = (
-               "macosx",
-               "iphoneos",
-               "iphonesimulator",
-               "appletvos",
-               "appletvsimulator",
-               "watchos",
-               "watchsimulator"
-            );
-            SWIFT_ACTIVE_COMPILATION_CONDITIONS = (
-               "$(inherited)",
-               "SWIFT_PACKAGE",
-               "DEBUG"
-            );
-            SWIFT_OPTIMIZATION_LEVEL = "-Onone";
-            USE_HEADERMAP = "NO";
-         };
-         name = "Debug";
-      };
-      "OBJ_30" = {
-         isa = "PBXFileReference";
-         path = "LICENSE";
-         sourceTree = "<group>";
-      };
-      "OBJ_31" = {
-         isa = "PBXFileReference";
-         path = "README.md";
-         sourceTree = "<group>";
-      };
-      "OBJ_32" = {
-         isa = "PBXFileReference";
-         path = "Gemfile";
-         sourceTree = "<group>";
-      };
-      "OBJ_33" = {
-         isa = "PBXFileReference";
-         path = "Gemfile.lock";
-         sourceTree = "<group>";
-      };
-      "OBJ_34" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdown.podspec";
-         sourceTree = "<group>";
-      };
-      "OBJ_36" = {
-         isa = "XCConfigurationList";
-         buildConfigurations = (
-            "OBJ_37",
-            "OBJ_38"
-         );
-         defaultConfigurationIsVisible = "0";
-         defaultConfigurationName = "Release";
-      };
-      "OBJ_37" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            ENABLE_TESTABILITY = "YES";
-            FRAMEWORK_SEARCH_PATHS = (
-               "$(inherited)",
-               "$(PLATFORM_DIR)/Developer/Library/Frameworks"
-            );
-            HEADER_SEARCH_PATHS = (
-               "$(inherited)"
-            );
-            INFOPLIST_FILE = "SwiftyMarkdown.xcodeproj/SwiftyMarkdown_Info.plist";
-            IPHONEOS_DEPLOYMENT_TARGET = "11.0";
-            LD_RUNPATH_SEARCH_PATHS = (
-               "$(inherited)",
-               "$(TOOLCHAIN_DIR)/usr/lib/swift/macosx"
-            );
-            MACOSX_DEPLOYMENT_TARGET = "10.12";
-            OTHER_CFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_LDFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_SWIFT_FLAGS = (
-               "$(inherited)"
-            );
-            PRODUCT_BUNDLE_IDENTIFIER = "SwiftyMarkdown";
-            PRODUCT_MODULE_NAME = "$(TARGET_NAME:c99extidentifier)";
-            PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
-            SKIP_INSTALL = "YES";
-            SWIFT_ACTIVE_COMPILATION_CONDITIONS = (
-               "$(inherited)"
-            );
-            SWIFT_VERSION = "5.0";
-            TARGET_NAME = "SwiftyMarkdown";
-            TVOS_DEPLOYMENT_TARGET = "11.0";
-            WATCHOS_DEPLOYMENT_TARGET = "4.0";
-         };
-         name = "Debug";
-      };
-      "OBJ_38" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            ENABLE_TESTABILITY = "YES";
-            FRAMEWORK_SEARCH_PATHS = (
-               "$(inherited)",
-               "$(PLATFORM_DIR)/Developer/Library/Frameworks"
-            );
-            HEADER_SEARCH_PATHS = (
-               "$(inherited)"
-            );
-            INFOPLIST_FILE = "SwiftyMarkdown.xcodeproj/SwiftyMarkdown_Info.plist";
-            IPHONEOS_DEPLOYMENT_TARGET = "11.0";
-            LD_RUNPATH_SEARCH_PATHS = (
-               "$(inherited)",
-               "$(TOOLCHAIN_DIR)/usr/lib/swift/macosx"
-            );
-            MACOSX_DEPLOYMENT_TARGET = "10.12";
-            OTHER_CFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_LDFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_SWIFT_FLAGS = (
-               "$(inherited)"
-            );
-            PRODUCT_BUNDLE_IDENTIFIER = "SwiftyMarkdown";
-            PRODUCT_MODULE_NAME = "$(TARGET_NAME:c99extidentifier)";
-            PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
-            SKIP_INSTALL = "YES";
-            SWIFT_ACTIVE_COMPILATION_CONDITIONS = (
-               "$(inherited)"
-            );
-            SWIFT_VERSION = "5.0";
-            TARGET_NAME = "SwiftyMarkdown";
-            TVOS_DEPLOYMENT_TARGET = "11.0";
-            WATCHOS_DEPLOYMENT_TARGET = "4.0";
-         };
-         name = "Release";
-      };
-      "OBJ_39" = {
-         isa = "PBXSourcesBuildPhase";
-         files = (
-            "OBJ_40",
-            "OBJ_41",
-            "OBJ_42",
-            "OBJ_43",
-            "OBJ_44",
-            "OBJ_45"
-         );
-      };
-      "OBJ_4" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            CLANG_ENABLE_OBJC_ARC = "YES";
-            COMBINE_HIDPI_IMAGES = "YES";
-            COPY_PHASE_STRIP = "YES";
-            DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-            DYLIB_INSTALL_NAME_BASE = "@rpath";
-            GCC_OPTIMIZATION_LEVEL = "s";
-            GCC_PREPROCESSOR_DEFINITIONS = (
-               "$(inherited)",
-               "SWIFT_PACKAGE=1"
-            );
-            MACOSX_DEPLOYMENT_TARGET = "10.10";
-            OTHER_SWIFT_FLAGS = (
-               "$(inherited)",
-               "-DXcode"
-            );
-            PRODUCT_NAME = "$(TARGET_NAME)";
-            SDKROOT = "macosx";
-            SUPPORTED_PLATFORMS = (
-               "macosx",
-               "iphoneos",
-               "iphonesimulator",
-               "appletvos",
-               "appletvsimulator",
-               "watchos",
-               "watchsimulator"
-            );
-            SWIFT_ACTIVE_COMPILATION_CONDITIONS = (
-               "$(inherited)",
-               "SWIFT_PACKAGE"
-            );
-            SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule";
-            USE_HEADERMAP = "NO";
-         };
-         name = "Release";
-      };
-      "OBJ_40" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_9";
-      };
-      "OBJ_41" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_10";
-      };
-      "OBJ_42" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_11";
-      };
-      "OBJ_43" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_12";
-      };
-      "OBJ_44" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_13";
-      };
-      "OBJ_45" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_14";
-      };
-      "OBJ_46" = {
-         isa = "PBXFrameworksBuildPhase";
-         files = (
-         );
-      };
-      "OBJ_48" = {
-         isa = "XCConfigurationList";
-         buildConfigurations = (
-            "OBJ_49",
-            "OBJ_50"
-         );
-         defaultConfigurationIsVisible = "0";
-         defaultConfigurationName = "Release";
-      };
-      "OBJ_49" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            LD = "/usr/bin/true";
-            OTHER_SWIFT_FLAGS = (
-               "-swift-version",
-               "5",
-               "-I",
-               "$(TOOLCHAIN_DIR)/usr/lib/swift/pm/4_2",
-               "-target",
-               "x86_64-apple-macosx10.10",
-               "-sdk",
-               "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk",
-               "-package-description-version",
-               "5.1"
-            );
-            SWIFT_VERSION = "5.0";
-         };
-         name = "Debug";
-      };
-      "OBJ_5" = {
-         isa = "PBXGroup";
-         children = (
-            "OBJ_6",
-            "OBJ_7",
-            "OBJ_15",
-            "OBJ_23",
-            "OBJ_26",
-            "OBJ_27",
-            "OBJ_28",
-            "OBJ_29",
-            "OBJ_30",
-            "OBJ_31",
-            "OBJ_32",
-            "OBJ_33",
-            "OBJ_34"
-         );
-         path = "";
-         sourceTree = "<group>";
-      };
-      "OBJ_50" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            LD = "/usr/bin/true";
-            OTHER_SWIFT_FLAGS = (
-               "-swift-version",
-               "5",
-               "-I",
-               "$(TOOLCHAIN_DIR)/usr/lib/swift/pm/4_2",
-               "-target",
-               "x86_64-apple-macosx10.10",
-               "-sdk",
-               "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk",
-               "-package-description-version",
-               "5.1"
-            );
-            SWIFT_VERSION = "5.0";
-         };
-         name = "Release";
-      };
-      "OBJ_51" = {
-         isa = "PBXSourcesBuildPhase";
-         files = (
-            "OBJ_52"
-         );
-      };
-      "OBJ_52" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_6";
-      };
-      "OBJ_54" = {
-         isa = "XCConfigurationList";
-         buildConfigurations = (
-            "OBJ_55",
-            "OBJ_56"
-         );
-         defaultConfigurationIsVisible = "0";
-         defaultConfigurationName = "Release";
-      };
-      "OBJ_55" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-         };
-         name = "Debug";
-      };
-      "OBJ_56" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-         };
-         name = "Release";
-      };
-      "OBJ_57" = {
-         isa = "PBXTargetDependency";
-         target = "SwiftyMarkdown::SwiftyMarkdownTests";
-      };
-      "OBJ_59" = {
-         isa = "XCConfigurationList";
-         buildConfigurations = (
-            "OBJ_60",
-            "OBJ_61"
-         );
-         defaultConfigurationIsVisible = "0";
-         defaultConfigurationName = "Release";
-      };
-      "OBJ_6" = {
-         isa = "PBXFileReference";
-         explicitFileType = "sourcecode.swift";
-         path = "Package.swift";
-         sourceTree = "<group>";
-      };
-      "OBJ_60" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            CLANG_ENABLE_MODULES = "YES";
-            EMBEDDED_CONTENT_CONTAINS_SWIFT = "YES";
-            FRAMEWORK_SEARCH_PATHS = (
-               "$(inherited)",
-               "$(PLATFORM_DIR)/Developer/Library/Frameworks"
-            );
-            HEADER_SEARCH_PATHS = (
-               "$(inherited)"
-            );
-            INFOPLIST_FILE = "SwiftyMarkdown.xcodeproj/SwiftyMarkdownTests_Info.plist";
-            IPHONEOS_DEPLOYMENT_TARGET = "11.0";
-            LD_RUNPATH_SEARCH_PATHS = (
-               "$(inherited)",
-               "@loader_path/../Frameworks",
-               "@loader_path/Frameworks"
-            );
-            MACOSX_DEPLOYMENT_TARGET = "10.12";
-            OTHER_CFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_LDFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_SWIFT_FLAGS = (
-               "$(inherited)"
-            );
-            SWIFT_ACTIVE_COMPILATION_CONDITIONS = (
-               "$(inherited)"
-            );
-            SWIFT_VERSION = "5.0";
-            TARGET_NAME = "SwiftyMarkdownTests";
-            TVOS_DEPLOYMENT_TARGET = "11.0";
-            WATCHOS_DEPLOYMENT_TARGET = "4.0";
-         };
-         name = "Debug";
-      };
-      "OBJ_61" = {
-         isa = "XCBuildConfiguration";
-         buildSettings = {
-            CLANG_ENABLE_MODULES = "YES";
-            EMBEDDED_CONTENT_CONTAINS_SWIFT = "YES";
-            FRAMEWORK_SEARCH_PATHS = (
-               "$(inherited)",
-               "$(PLATFORM_DIR)/Developer/Library/Frameworks"
-            );
-            HEADER_SEARCH_PATHS = (
-               "$(inherited)"
-            );
-            INFOPLIST_FILE = "SwiftyMarkdown.xcodeproj/SwiftyMarkdownTests_Info.plist";
-            IPHONEOS_DEPLOYMENT_TARGET = "11.0";
-            LD_RUNPATH_SEARCH_PATHS = (
-               "$(inherited)",
-               "@loader_path/../Frameworks",
-               "@loader_path/Frameworks"
-            );
-            MACOSX_DEPLOYMENT_TARGET = "10.12";
-            OTHER_CFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_LDFLAGS = (
-               "$(inherited)"
-            );
-            OTHER_SWIFT_FLAGS = (
-               "$(inherited)"
-            );
-            SWIFT_ACTIVE_COMPILATION_CONDITIONS = (
-               "$(inherited)"
-            );
-            SWIFT_VERSION = "5.0";
-            TARGET_NAME = "SwiftyMarkdownTests";
-            TVOS_DEPLOYMENT_TARGET = "11.0";
-            WATCHOS_DEPLOYMENT_TARGET = "4.0";
-         };
-         name = "Release";
-      };
-      "OBJ_62" = {
-         isa = "PBXSourcesBuildPhase";
-         files = (
-            "OBJ_63",
-            "OBJ_64",
-            "OBJ_65",
-            "OBJ_66",
-            "OBJ_67",
-            "OBJ_68"
-         );
-      };
-      "OBJ_63" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_17";
-      };
-      "OBJ_64" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_18";
-      };
-      "OBJ_65" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_19";
-      };
-      "OBJ_66" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_20";
-      };
-      "OBJ_67" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_21";
-      };
-      "OBJ_68" = {
-         isa = "PBXBuildFile";
-         fileRef = "OBJ_22";
-      };
-      "OBJ_69" = {
-         isa = "PBXFrameworksBuildPhase";
-         files = (
-            "OBJ_70"
-         );
-      };
-      "OBJ_7" = {
-         isa = "PBXGroup";
-         children = (
-            "OBJ_8"
-         );
-         name = "Sources";
-         path = "";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_70" = {
-         isa = "PBXBuildFile";
-         fileRef = "SwiftyMarkdown::SwiftyMarkdown::Product";
-      };
-      "OBJ_71" = {
-         isa = "PBXTargetDependency";
-         target = "SwiftyMarkdown::SwiftyMarkdown";
-      };
-      "OBJ_8" = {
-         isa = "PBXGroup";
-         children = (
-            "OBJ_9",
-            "OBJ_10",
-            "OBJ_11",
-            "OBJ_12",
-            "OBJ_13",
-            "OBJ_14"
-         );
-         name = "SwiftyMarkdown";
-         path = "Sources/SwiftyMarkdown";
-         sourceTree = "SOURCE_ROOT";
-      };
-      "OBJ_9" = {
-         isa = "PBXFileReference";
-         path = "String+SwiftyMarkdown.swift";
-         sourceTree = "<group>";
-      };
-      "SwiftyMarkdown::SwiftPMPackageDescription" = {
-         isa = "PBXNativeTarget";
-         buildConfigurationList = "OBJ_48";
-         buildPhases = (
-            "OBJ_51"
-         );
-         dependencies = (
-         );
-         name = "SwiftyMarkdownPackageDescription";
-         productName = "SwiftyMarkdownPackageDescription";
-         productType = "com.apple.product-type.framework";
-      };
-      "SwiftyMarkdown::SwiftyMarkdown" = {
-         isa = "PBXNativeTarget";
-         buildConfigurationList = "OBJ_36";
-         buildPhases = (
-            "OBJ_39",
-            "OBJ_46"
-         );
-         dependencies = (
-         );
-         name = "SwiftyMarkdown";
-         productName = "SwiftyMarkdown";
-         productReference = "SwiftyMarkdown::SwiftyMarkdown::Product";
-         productType = "com.apple.product-type.framework";
-      };
-      "SwiftyMarkdown::SwiftyMarkdown::Product" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdown.framework";
-         sourceTree = "BUILT_PRODUCTS_DIR";
-      };
-      "SwiftyMarkdown::SwiftyMarkdownPackageTests::ProductTarget" = {
-         isa = "PBXAggregateTarget";
-         buildConfigurationList = "OBJ_54";
-         buildPhases = (
-         );
-         dependencies = (
-            "OBJ_57"
-         );
-         name = "SwiftyMarkdownPackageTests";
-         productName = "SwiftyMarkdownPackageTests";
-      };
-      "SwiftyMarkdown::SwiftyMarkdownTests" = {
-         isa = "PBXNativeTarget";
-         buildConfigurationList = "OBJ_59";
-         buildPhases = (
-            "OBJ_62",
-            "OBJ_69"
-         );
-         dependencies = (
-            "OBJ_71"
-         );
-         name = "SwiftyMarkdownTests";
-         productName = "SwiftyMarkdownTests";
-         productReference = "SwiftyMarkdown::SwiftyMarkdownTests::Product";
-         productType = "com.apple.product-type.bundle.unit-test";
-      };
-      "SwiftyMarkdown::SwiftyMarkdownTests::Product" = {
-         isa = "PBXFileReference";
-         path = "SwiftyMarkdownTests.xctest";
-         sourceTree = "BUILT_PRODUCTS_DIR";
-      };
-   };
-   rootObject = "OBJ_1";
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 46;
+	objects = {
+
+/* Begin PBXAggregateTarget section */
+		"SwiftyMarkdown::SwiftyMarkdownPackageTests::ProductTarget" /* SwiftyMarkdownPackageTests */ = {
+			isa = PBXAggregateTarget;
+			buildConfigurationList = OBJ_54 /* Build configuration list for PBXAggregateTarget "SwiftyMarkdownPackageTests" */;
+			buildPhases = (
+			);
+			dependencies = (
+				OBJ_57 /* PBXTargetDependency */,
+			);
+			name = SwiftyMarkdownPackageTests;
+			productName = SwiftyMarkdownPackageTests;
+		};
+/* End PBXAggregateTarget section */
+
+/* Begin PBXBuildFile section */
+		F4ACB6CB23E8A5C500EA665D /* SwiftyScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6CA23E8A5C500EA665D /* SwiftyScanner.swift */; };
+		F4ACB6CE23E8A88400EA665D /* Token.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6CD23E8A88400EA665D /* Token.swift */; };
+		F4ACB6D023E8A8A500EA665D /* CharacterRule.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6CF23E8A8A500EA665D /* CharacterRule.swift */; };
+		F4ACB6D223E8B08400EA665D /* PerfomanceLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4ACB6D123E8B08400EA665D /* PerfomanceLog.swift */; };
+		F4C95126243ECB320059AB15 /* SwiftyScannerNonRepeating.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4C95125243ECB320059AB15 /* SwiftyScannerNonRepeating.swift */; };
+		OBJ_40 /* String+SwiftyMarkdown.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_9 /* String+SwiftyMarkdown.swift */; };
+		OBJ_41 /* SwiftyLineProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_10 /* SwiftyLineProcessor.swift */; };
+		OBJ_42 /* SwiftyMarkdown+iOS.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_11 /* SwiftyMarkdown+iOS.swift */; };
+		OBJ_43 /* SwiftyMarkdown+macOS.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_12 /* SwiftyMarkdown+macOS.swift */; };
+		OBJ_44 /* SwiftyMarkdown.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_13 /* SwiftyMarkdown.swift */; };
+		OBJ_45 /* SwiftyTokeniser.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_14 /* SwiftyTokeniser.swift */; };
+		OBJ_52 /* Package.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_6 /* Package.swift */; };
+		OBJ_63 /* SwiftyMarkdownAttributedStringTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_17 /* SwiftyMarkdownAttributedStringTests.swift */; };
+		OBJ_64 /* SwiftyMarkdownCharacterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_18 /* SwiftyMarkdownCharacterTests.swift */; };
+		OBJ_65 /* SwiftyMarkdownLineTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_19 /* SwiftyMarkdownLineTests.swift */; };
+		OBJ_66 /* SwiftyMarkdownLinkTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_20 /* SwiftyMarkdownLinkTests.swift */; };
+		OBJ_67 /* SwiftyMarkdownPerformanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_21 /* SwiftyMarkdownPerformanceTests.swift */; };
+		OBJ_68 /* XCTest+SwiftyMarkdown.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_22 /* XCTest+SwiftyMarkdown.swift */; };
+		OBJ_70 /* SwiftyMarkdown.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = "SwiftyMarkdown::SwiftyMarkdown::Product" /* SwiftyMarkdown.framework */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXContainerItemProxy section */
+		F4ACB6CC23E8A5C600EA665D /* PBXContainerItemProxy */ = {
+			isa = PBXContainerItemProxy;
+			containerPortal = OBJ_1 /* Project object */;
+			proxyType = 1;
+			remoteGlobalIDString = "SwiftyMarkdown::SwiftyMarkdownTests";
+			remoteInfo = SwiftyMarkdownTests;
+		};
+		F4B37A0723E507C900833479 /* PBXContainerItemProxy */ = {
+			isa = PBXContainerItemProxy;
+			containerPortal = OBJ_1 /* Project object */;
+			proxyType = 1;
+			remoteGlobalIDString = "SwiftyMarkdown::SwiftyMarkdown";
+			remoteInfo = SwiftyMarkdown;
+		};
+/* End PBXContainerItemProxy section */
+
+/* Begin PBXFileReference section */
+		F4ACB6CA23E8A5C500EA665D /* SwiftyScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyScanner.swift; sourceTree = "<group>"; };
+		F4ACB6CD23E8A88400EA665D /* Token.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Token.swift; sourceTree = "<group>"; };
+		F4ACB6CF23E8A8A500EA665D /* CharacterRule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CharacterRule.swift; sourceTree = "<group>"; };
+		F4ACB6D123E8B08400EA665D /* PerfomanceLog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerfomanceLog.swift; sourceTree = "<group>"; };
+		F4C95125243ECB320059AB15 /* SwiftyScannerNonRepeating.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SwiftyScannerNonRepeating.swift; sourceTree = "<group>"; };
+		OBJ_10 /* SwiftyLineProcessor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyLineProcessor.swift; sourceTree = "<group>"; };
+		OBJ_11 /* SwiftyMarkdown+iOS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SwiftyMarkdown+iOS.swift"; sourceTree = "<group>"; };
+		OBJ_12 /* SwiftyMarkdown+macOS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SwiftyMarkdown+macOS.swift"; sourceTree = "<group>"; };
+		OBJ_13 /* SwiftyMarkdown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyMarkdown.swift; sourceTree = "<group>"; };
+		OBJ_14 /* SwiftyTokeniser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyTokeniser.swift; sourceTree = "<group>"; };
+		OBJ_17 /* SwiftyMarkdownAttributedStringTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyMarkdownAttributedStringTests.swift; sourceTree = "<group>"; };
+		OBJ_18 /* SwiftyMarkdownCharacterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyMarkdownCharacterTests.swift; sourceTree = "<group>"; };
+		OBJ_19 /* SwiftyMarkdownLineTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyMarkdownLineTests.swift; sourceTree = "<group>"; };
+		OBJ_20 /* SwiftyMarkdownLinkTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyMarkdownLinkTests.swift; sourceTree = "<group>"; };
+		OBJ_21 /* SwiftyMarkdownPerformanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftyMarkdownPerformanceTests.swift; sourceTree = "<group>"; };
+		OBJ_22 /* XCTest+SwiftyMarkdown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "XCTest+SwiftyMarkdown.swift"; sourceTree = "<group>"; };
+		OBJ_26 /* Playground */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Playground; sourceTree = SOURCE_ROOT; };
+		OBJ_27 /* Example */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Example; sourceTree = SOURCE_ROOT; };
+		OBJ_28 /* Resources */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Resources; sourceTree = SOURCE_ROOT; };
+		OBJ_29 /* fastlane */ = {isa = PBXFileReference; lastKnownFileType = folder; path = fastlane; sourceTree = SOURCE_ROOT; };
+		OBJ_30 /* LICENSE */ = {isa = PBXFileReference; lastKnownFileType = text; path = LICENSE; sourceTree = "<group>"; };
+		OBJ_31 /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = "<group>"; };
+		OBJ_32 /* Gemfile */ = {isa = PBXFileReference; lastKnownFileType = text; path = Gemfile; sourceTree = "<group>"; };
+		OBJ_33 /* Gemfile.lock */ = {isa = PBXFileReference; lastKnownFileType = text; path = Gemfile.lock; sourceTree = "<group>"; };
+		OBJ_34 /* SwiftyMarkdown.podspec */ = {isa = PBXFileReference; lastKnownFileType = text; path = SwiftyMarkdown.podspec; sourceTree = "<group>"; };
+		OBJ_6 /* Package.swift */ = {isa = PBXFileReference; explicitFileType = sourcecode.swift; path = Package.swift; sourceTree = "<group>"; };
+		OBJ_9 /* String+SwiftyMarkdown.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+SwiftyMarkdown.swift"; sourceTree = "<group>"; };
+		"SwiftyMarkdown::SwiftyMarkdown::Product" /* SwiftyMarkdown.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftyMarkdown.framework; sourceTree = BUILT_PRODUCTS_DIR; };
+		"SwiftyMarkdown::SwiftyMarkdownTests::Product" /* SwiftyMarkdownTests.xctest */ = {isa = PBXFileReference; lastKnownFileType = file; path = SwiftyMarkdownTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		OBJ_46 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 0;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+		OBJ_69 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 0;
+			files = (
+				OBJ_70 /* SwiftyMarkdown.framework in Frameworks */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		OBJ_15 /* Tests */ = {
+			isa = PBXGroup;
+			children = (
+				OBJ_16 /* SwiftyMarkdownTests */,
+			);
+			name = Tests;
+			sourceTree = SOURCE_ROOT;
+		};
+		OBJ_16 /* SwiftyMarkdownTests */ = {
+			isa = PBXGroup;
+			children = (
+				OBJ_17 /* SwiftyMarkdownAttributedStringTests.swift */,
+				OBJ_18 /* SwiftyMarkdownCharacterTests.swift */,
+				OBJ_19 /* SwiftyMarkdownLineTests.swift */,
+				OBJ_20 /* SwiftyMarkdownLinkTests.swift */,
+				OBJ_21 /* SwiftyMarkdownPerformanceTests.swift */,
+				OBJ_22 /* XCTest+SwiftyMarkdown.swift */,
+			);
+			name = SwiftyMarkdownTests;
+			path = Tests/SwiftyMarkdownTests;
+			sourceTree = SOURCE_ROOT;
+		};
+		OBJ_23 /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				"SwiftyMarkdown::SwiftyMarkdown::Product" /* SwiftyMarkdown.framework */,
+				"SwiftyMarkdown::SwiftyMarkdownTests::Product" /* SwiftyMarkdownTests.xctest */,
+			);
+			name = Products;
+			sourceTree = BUILT_PRODUCTS_DIR;
+		};
+		OBJ_5 = {
+			isa = PBXGroup;
+			children = (
+				OBJ_6 /* Package.swift */,
+				OBJ_7 /* Sources */,
+				OBJ_15 /* Tests */,
+				OBJ_23 /* Products */,
+				OBJ_26 /* Playground */,
+				OBJ_27 /* Example */,
+				OBJ_28 /* Resources */,
+				OBJ_29 /* fastlane */,
+				OBJ_30 /* LICENSE */,
+				OBJ_31 /* README.md */,
+				OBJ_32 /* Gemfile */,
+				OBJ_33 /* Gemfile.lock */,
+				OBJ_34 /* SwiftyMarkdown.podspec */,
+			);
+			sourceTree = "<group>";
+		};
+		OBJ_7 /* Sources */ = {
+			isa = PBXGroup;
+			children = (
+				OBJ_8 /* SwiftyMarkdown */,
+			);
+			name = Sources;
+			sourceTree = SOURCE_ROOT;
+		};
+		OBJ_8 /* SwiftyMarkdown */ = {
+			isa = PBXGroup;
+			children = (
+				OBJ_9 /* String+SwiftyMarkdown.swift */,
+				OBJ_10 /* SwiftyLineProcessor.swift */,
+				OBJ_11 /* SwiftyMarkdown+iOS.swift */,
+				OBJ_12 /* SwiftyMarkdown+macOS.swift */,
+				OBJ_13 /* SwiftyMarkdown.swift */,
+				OBJ_14 /* SwiftyTokeniser.swift */,
+				F4ACB6CD23E8A88400EA665D /* Token.swift */,
+				F4ACB6CA23E8A5C500EA665D /* SwiftyScanner.swift */,
+				F4C95125243ECB320059AB15 /* SwiftyScannerNonRepeating.swift */,
+				F4ACB6CF23E8A8A500EA665D /* CharacterRule.swift */,
+				F4ACB6D123E8B08400EA665D /* PerfomanceLog.swift */,
+			);
+			name = SwiftyMarkdown;
+			path = Sources/SwiftyMarkdown;
+			sourceTree = SOURCE_ROOT;
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		"SwiftyMarkdown::SwiftPMPackageDescription" /* SwiftyMarkdownPackageDescription */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = OBJ_48 /* Build configuration list for PBXNativeTarget "SwiftyMarkdownPackageDescription" */;
+			buildPhases = (
+				OBJ_51 /* Sources */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = SwiftyMarkdownPackageDescription;
+			productName = SwiftyMarkdownPackageDescription;
+			productType = "com.apple.product-type.framework";
+		};
+		"SwiftyMarkdown::SwiftyMarkdown" /* SwiftyMarkdown */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = OBJ_36 /* Build configuration list for PBXNativeTarget "SwiftyMarkdown" */;
+			buildPhases = (
+				OBJ_39 /* Sources */,
+				OBJ_46 /* Frameworks */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = SwiftyMarkdown;
+			productName = SwiftyMarkdown;
+			productReference = "SwiftyMarkdown::SwiftyMarkdown::Product" /* SwiftyMarkdown.framework */;
+			productType = "com.apple.product-type.framework";
+		};
+		"SwiftyMarkdown::SwiftyMarkdownTests" /* SwiftyMarkdownTests */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = OBJ_59 /* Build configuration list for PBXNativeTarget "SwiftyMarkdownTests" */;
+			buildPhases = (
+				OBJ_62 /* Sources */,
+				OBJ_69 /* Frameworks */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+				OBJ_71 /* PBXTargetDependency */,
+			);
+			name = SwiftyMarkdownTests;
+			productName = SwiftyMarkdownTests;
+			productReference = "SwiftyMarkdown::SwiftyMarkdownTests::Product" /* SwiftyMarkdownTests.xctest */;
+			productType = "com.apple.product-type.bundle.unit-test";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		OBJ_1 /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastSwiftMigration = 9999;
+				LastUpgradeCheck = 9999;
+			};
+			buildConfigurationList = OBJ_2 /* Build configuration list for PBXProject "SwiftyMarkdown" */;
+			compatibilityVersion = "Xcode 3.2";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+			);
+			mainGroup = OBJ_5;
+			productRefGroup = OBJ_23 /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				"SwiftyMarkdown::SwiftyMarkdown" /* SwiftyMarkdown */,
+				"SwiftyMarkdown::SwiftPMPackageDescription" /* SwiftyMarkdownPackageDescription */,
+				"SwiftyMarkdown::SwiftyMarkdownPackageTests::ProductTarget" /* SwiftyMarkdownPackageTests */,
+				"SwiftyMarkdown::SwiftyMarkdownTests" /* SwiftyMarkdownTests */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXSourcesBuildPhase section */
+		OBJ_39 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 0;
+			files = (
+				OBJ_40 /* String+SwiftyMarkdown.swift in Sources */,
+				OBJ_41 /* SwiftyLineProcessor.swift in Sources */,
+				F4ACB6D223E8B08400EA665D /* PerfomanceLog.swift in Sources */,
+				F4ACB6CB23E8A5C500EA665D /* SwiftyScanner.swift in Sources */,
+				OBJ_42 /* SwiftyMarkdown+iOS.swift in Sources */,
+				OBJ_43 /* SwiftyMarkdown+macOS.swift in Sources */,
+				F4C95126243ECB320059AB15 /* SwiftyScannerNonRepeating.swift in Sources */,
+				OBJ_44 /* SwiftyMarkdown.swift in Sources */,
+				OBJ_45 /* SwiftyTokeniser.swift in Sources */,
+				F4ACB6D023E8A8A500EA665D /* CharacterRule.swift in Sources */,
+				F4ACB6CE23E8A88400EA665D /* Token.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+		OBJ_51 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 0;
+			files = (
+				OBJ_52 /* Package.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+		OBJ_62 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 0;
+			files = (
+				OBJ_63 /* SwiftyMarkdownAttributedStringTests.swift in Sources */,
+				OBJ_64 /* SwiftyMarkdownCharacterTests.swift in Sources */,
+				OBJ_65 /* SwiftyMarkdownLineTests.swift in Sources */,
+				OBJ_66 /* SwiftyMarkdownLinkTests.swift in Sources */,
+				OBJ_67 /* SwiftyMarkdownPerformanceTests.swift in Sources */,
+				OBJ_68 /* XCTest+SwiftyMarkdown.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXTargetDependency section */
+		OBJ_57 /* PBXTargetDependency */ = {
+			isa = PBXTargetDependency;
+			target = "SwiftyMarkdown::SwiftyMarkdownTests" /* SwiftyMarkdownTests */;
+			targetProxy = F4ACB6CC23E8A5C600EA665D /* PBXContainerItemProxy */;
+		};
+		OBJ_71 /* PBXTargetDependency */ = {
+			isa = PBXTargetDependency;
+			target = "SwiftyMarkdown::SwiftyMarkdown" /* SwiftyMarkdown */;
+			targetProxy = F4B37A0723E507C900833479 /* PBXContainerItemProxy */;
+		};
+/* End PBXTargetDependency section */
+
+/* Begin XCBuildConfiguration section */
+		OBJ_3 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CLANG_ENABLE_OBJC_ARC = YES;
+				COMBINE_HIDPI_IMAGES = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				DYLIB_INSTALL_NAME_BASE = "@rpath";
+				ENABLE_NS_ASSERTIONS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"SWIFT_PACKAGE=1",
+					"DEBUG=1",
+				);
+				MACOSX_DEPLOYMENT_TARGET = 10.10;
+				ONLY_ACTIVE_ARCH = YES;
+				OTHER_SWIFT_FLAGS = "$(inherited) -DXcode";
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SDKROOT = macosx;
+				SUPPORTED_PLATFORMS = "macosx iphoneos iphonesimulator appletvos appletvsimulator watchos watchsimulator";
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited) SWIFT_PACKAGE DEBUG";
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+				USE_HEADERMAP = NO;
+			};
+			name = Debug;
+		};
+		OBJ_37 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ENABLE_TESTABILITY = YES;
+				FRAMEWORK_SEARCH_PATHS = (
+					"$(inherited)",
+					"$(PLATFORM_DIR)/Developer/Library/Frameworks",
+				);
+				HEADER_SEARCH_PATHS = "$(inherited)";
+				INFOPLIST_FILE = SwiftyMarkdown.xcodeproj/SwiftyMarkdown_Info.plist;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) $(TOOLCHAIN_DIR)/usr/lib/swift/macosx";
+				MACOSX_DEPLOYMENT_TARGET = 10.12;
+				OTHER_CFLAGS = "$(inherited)";
+				OTHER_LDFLAGS = "$(inherited)";
+				OTHER_SWIFT_FLAGS = "$(inherited)";
+				PRODUCT_BUNDLE_IDENTIFIER = SwiftyMarkdown;
+				PRODUCT_MODULE_NAME = "$(TARGET_NAME:c99extidentifier)";
+				PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
+				SKIP_INSTALL = YES;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited)";
+				SWIFT_VERSION = 5.0;
+				TARGET_NAME = SwiftyMarkdown;
+				TVOS_DEPLOYMENT_TARGET = 11.0;
+				WATCHOS_DEPLOYMENT_TARGET = 4.0;
+			};
+			name = Debug;
+		};
+		OBJ_38 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ENABLE_TESTABILITY = YES;
+				FRAMEWORK_SEARCH_PATHS = (
+					"$(inherited)",
+					"$(PLATFORM_DIR)/Developer/Library/Frameworks",
+				);
+				HEADER_SEARCH_PATHS = "$(inherited)";
+				INFOPLIST_FILE = SwiftyMarkdown.xcodeproj/SwiftyMarkdown_Info.plist;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) $(TOOLCHAIN_DIR)/usr/lib/swift/macosx";
+				MACOSX_DEPLOYMENT_TARGET = 10.12;
+				OTHER_CFLAGS = "$(inherited)";
+				OTHER_LDFLAGS = "$(inherited)";
+				OTHER_SWIFT_FLAGS = "$(inherited)";
+				PRODUCT_BUNDLE_IDENTIFIER = SwiftyMarkdown;
+				PRODUCT_MODULE_NAME = "$(TARGET_NAME:c99extidentifier)";
+				PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
+				SKIP_INSTALL = YES;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited)";
+				SWIFT_VERSION = 5.0;
+				TARGET_NAME = SwiftyMarkdown;
+				TVOS_DEPLOYMENT_TARGET = 11.0;
+				WATCHOS_DEPLOYMENT_TARGET = 4.0;
+			};
+			name = Release;
+		};
+		OBJ_4 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CLANG_ENABLE_OBJC_ARC = YES;
+				COMBINE_HIDPI_IMAGES = YES;
+				COPY_PHASE_STRIP = YES;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				DYLIB_INSTALL_NAME_BASE = "@rpath";
+				GCC_OPTIMIZATION_LEVEL = s;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"SWIFT_PACKAGE=1",
+				);
+				MACOSX_DEPLOYMENT_TARGET = 10.10;
+				OTHER_SWIFT_FLAGS = "$(inherited) -DXcode";
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SDKROOT = macosx;
+				SUPPORTED_PLATFORMS = "macosx iphoneos iphonesimulator appletvos appletvsimulator watchos watchsimulator";
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited) SWIFT_PACKAGE";
+				SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule";
+				USE_HEADERMAP = NO;
+			};
+			name = Release;
+		};
+		OBJ_49 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				LD = /usr/bin/true;
+				OTHER_SWIFT_FLAGS = "-swift-version 5 -I $(TOOLCHAIN_DIR)/usr/lib/swift/pm/4_2 -target x86_64-apple-macosx10.10 -sdk /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk -package-description-version 5.1";
+				SWIFT_VERSION = 5.0;
+			};
+			name = Debug;
+		};
+		OBJ_50 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				LD = /usr/bin/true;
+				OTHER_SWIFT_FLAGS = "-swift-version 5 -I $(TOOLCHAIN_DIR)/usr/lib/swift/pm/4_2 -target x86_64-apple-macosx10.10 -sdk /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk -package-description-version 5.1";
+				SWIFT_VERSION = 5.0;
+			};
+			name = Release;
+		};
+		OBJ_55 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+			};
+			name = Debug;
+		};
+		OBJ_56 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+			};
+			name = Release;
+		};
+		OBJ_60 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CLANG_ENABLE_MODULES = YES;
+				EMBEDDED_CONTENT_CONTAINS_SWIFT = YES;
+				FRAMEWORK_SEARCH_PATHS = (
+					"$(inherited)",
+					"$(PLATFORM_DIR)/Developer/Library/Frameworks",
+				);
+				HEADER_SEARCH_PATHS = "$(inherited)";
+				INFOPLIST_FILE = SwiftyMarkdown.xcodeproj/SwiftyMarkdownTests_Info.plist;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @loader_path/../Frameworks @loader_path/Frameworks";
+				MACOSX_DEPLOYMENT_TARGET = 10.12;
+				OTHER_CFLAGS = "$(inherited)";
+				OTHER_LDFLAGS = "$(inherited)";
+				OTHER_SWIFT_FLAGS = "$(inherited)";
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited)";
+				SWIFT_VERSION = 5.0;
+				TARGET_NAME = SwiftyMarkdownTests;
+				TVOS_DEPLOYMENT_TARGET = 11.0;
+				WATCHOS_DEPLOYMENT_TARGET = 4.0;
+			};
+			name = Debug;
+		};
+		OBJ_61 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CLANG_ENABLE_MODULES = YES;
+				EMBEDDED_CONTENT_CONTAINS_SWIFT = YES;
+				FRAMEWORK_SEARCH_PATHS = (
+					"$(inherited)",
+					"$(PLATFORM_DIR)/Developer/Library/Frameworks",
+				);
+				HEADER_SEARCH_PATHS = "$(inherited)";
+				INFOPLIST_FILE = SwiftyMarkdown.xcodeproj/SwiftyMarkdownTests_Info.plist;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @loader_path/../Frameworks @loader_path/Frameworks";
+				MACOSX_DEPLOYMENT_TARGET = 10.12;
+				OTHER_CFLAGS = "$(inherited)";
+				OTHER_LDFLAGS = "$(inherited)";
+				OTHER_SWIFT_FLAGS = "$(inherited)";
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited)";
+				SWIFT_VERSION = 5.0;
+				TARGET_NAME = SwiftyMarkdownTests;
+				TVOS_DEPLOYMENT_TARGET = 11.0;
+				WATCHOS_DEPLOYMENT_TARGET = 4.0;
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		OBJ_2 /* Build configuration list for PBXProject "SwiftyMarkdown" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				OBJ_3 /* Debug */,
+				OBJ_4 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		OBJ_36 /* Build configuration list for PBXNativeTarget "SwiftyMarkdown" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				OBJ_37 /* Debug */,
+				OBJ_38 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		OBJ_48 /* Build configuration list for PBXNativeTarget "SwiftyMarkdownPackageDescription" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				OBJ_49 /* Debug */,
+				OBJ_50 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		OBJ_54 /* Build configuration list for PBXAggregateTarget "SwiftyMarkdownPackageTests" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				OBJ_55 /* Debug */,
+				OBJ_56 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		OBJ_59 /* Build configuration list for PBXNativeTarget "SwiftyMarkdownTests" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				OBJ_60 /* Debug */,
+				OBJ_61 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = OBJ_1 /* Project object */;
 }

+ 42 - 0
SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/88991ED5-B954-422F-B610-BDC9A4AEC008.plist

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>classNames</key>
+	<dict>
+		<key>SwiftyMarkdownPerformanceTests</key>
+		<dict>
+			<key>testThatFilesAreProcessedQuickly()</key>
+			<dict>
+				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
+				<dict>
+					<key>baselineAverage</key>
+					<real>0.1</real>
+					<key>baselineIntegrationDisplayName</key>
+					<string>Local Baseline</string>
+				</dict>
+			</dict>
+			<key>testThatStringsAreProcessedQuickly()</key>
+			<dict>
+				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
+				<dict>
+					<key>baselineAverage</key>
+					<real>0.1</real>
+					<key>baselineIntegrationDisplayName</key>
+					<string>Local Baseline</string>
+				</dict>
+			</dict>
+			<key>testThatVeryLongStringsAreProcessedQuickly()</key>
+			<dict>
+				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
+				<dict>
+					<key>baselineAverage</key>
+					<real>0.1</real>
+					<key>baselineIntegrationDisplayName</key>
+					<string>Local Baseline</string>
+				</dict>
+			</dict>
+		</dict>
+	</dict>
+</dict>
+</plist>

+ 3 - 1
SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/AD1DF83E-20BC-4E7E-8C14-683818ED0A26.plist

@@ -11,9 +11,11 @@
 				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
 				<dict>
 					<key>baselineAverage</key>
-					<real>0.0217</real>
+					<real>0.212</real>
 					<key>baselineIntegrationDisplayName</key>
 					<string>Local Baseline</string>
+					<key>maxPercentRelativeStandardDeviation</key>
+					<real>10</real>
 				</dict>
 			</dict>
 			<key>testThatStringsAreProcessedQuickly()</key>

+ 24 - 0
SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/Info.plist

@@ -4,6 +4,30 @@
 <dict>
 	<key>runDestinationsByUUID</key>
 	<dict>
+		<key>88991ED5-B954-422F-B610-BDC9A4AEC008</key>
+		<dict>
+			<key>localComputer</key>
+			<dict>
+				<key>busSpeedInMHz</key>
+				<integer>400</integer>
+				<key>cpuCount</key>
+				<integer>1</integer>
+				<key>cpuKind</key>
+				<string>8-Core Intel Core i9</string>
+				<key>cpuSpeedInMHz</key>
+				<integer>2400</integer>
+				<key>logicalCPUCoresPerPackage</key>
+				<integer>16</integer>
+				<key>modelCode</key>
+				<string>MacBookPro16,1</string>
+				<key>physicalCPUCoresPerPackage</key>
+				<integer>8</integer>
+				<key>platformIdentifier</key>
+				<string>com.apple.platform.macosx</string>
+			</dict>
+			<key>targetArchitecture</key>
+			<string>x86_64h</string>
+		</dict>
 		<key>AD1DF83E-20BC-4E7E-8C14-683818ED0A26</key>
 		<dict>
 			<key>localComputer</key>

+ 37 - 0
SwiftyMarkdown.xcodeproj/xcshareddata/xcschemes/SwiftyMarkdown-Package.xcscheme

@@ -50,6 +50,43 @@
       debugDocumentVersioning = "YES"
       debugServiceExtension = "internal"
       allowLocationSimulation = "YES">
+      <EnvironmentVariables>
+         <EnvironmentVariable
+            key = "SwiftyTokeniserLogging"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyScannerScanner"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyScannerScannerPerformanceLogging"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyLineProcessorPerformanceLogging"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyScannerPerformanceLogging"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyTokeniserPerformanceLogging"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "SwiftyMarkdownPerformanceLogging"
+            value = ""
+            isEnabled = "NO">
+         </EnvironmentVariable>
+      </EnvironmentVariables>
    </LaunchAction>
    <ProfileAction
       buildConfiguration = "Release"

+ 0 - 0
Tests/SwiftyMarkdownTests/SwiftyMarkdownAttributedStringTests copy.swift → Tests/SwiftyMarkdownTests/SwiftyMarkdownAttributedStringTests.swift


+ 453 - 93
Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift

@@ -9,26 +9,90 @@
 @testable import SwiftyMarkdown
 import XCTest
 
-class SwiftyMarkdownCharacterTests: XCTestCase {
+class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
-	func testIsolatedCase() {
-		let challenge = TokenTest(input: "\\~\\~removed\\~\\~crossed-out string. ~This should be ignored~", output: "~~removed~~crossed-out string. ~This should be ignored~", tokens: [
-			Token(type: .string, inputString: "~~removed~~crossed-out string. ~This should be ignored~", characterStyles: [])
+	func off_testIsolatedCase() {
+		
+		challenge = TokenTest(input: "*\\***\\****b*\\***\\****\\", output: "***b***\\", tokens : [
+			Token(type: .string, inputString: "*", characterStyles: [CharacterStyle.italic]),
+			Token(type: .string, inputString: "*b**", characterStyles: [CharacterStyle.bold, CharacterStyle.italic]),
+			Token(type: .string, inputString: "\\", characterStyles: [])
 		])
-		let results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		return
+		
+		challenge = TokenTest(input: """
+		An asterisk: *
+		Line break
+		""", output: """
+		An asterisk: *
+		Line break
+		""", tokens: [
+			Token(type: .string, inputString: "An asterisk: *", characterStyles: []),
+			Token(type: .string, inputString: "Line break", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(results.stringTokens.count, challenge.tokens.count )
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		return
+			
+			challenge = TokenTest(input: "A [referenced link][link]\n[link]: https://www.neverendingvoyage.com/", output: "A referenced link", tokens: [
+				Token(type: .string, inputString: "A ", characterStyles: []),
+				Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
+			])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		
+		
+		challenge = TokenTest(input: "A [referenced link][link]\n[notLink]: https://www.neverendingvoyage.com/", output: "A [referenced link][link]", tokens: [
+			Token(type: .string, inputString: "A [referenced link][link]", characterStyles: [])
+		])
+		results = self.attempt(challenge, rules: [.links, .images, .referencedLinks])
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 0)
 		
 	}
 	
 	func testThatBoldTraitsAreRecognised() {
-		var challenge = TokenTest(input: "**A bold string**", output: "A bold string",  tokens: [
+		challenge = TokenTest(input: "**A bold string**", output: "A bold string",  tokens: [
 			Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -38,8 +102,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " word", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -47,8 +117,29 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "**A normal string**", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "\\\\*\\*A normal \\\\ string\\*\\*", output: "\\**A normal \\\\ string**", tokens: [
+			Token(type: .string, inputString: "\\**A normal \\\\ string**", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -56,8 +147,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "A string with double **escaped** asterisks", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -66,8 +163,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "One escaped, one not at either end*", characterStyles: [CharacterStyle.italic]),
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -77,19 +180,31 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " asterisk, one not at either end", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testThatCodeTraitsAreRecognised() {
-		var challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)",  tokens: [
-			Token(type: .string, inputString: "Code (**should** not process internal tags) ", characterStyles: [CharacterStyle.code])
+		challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)",  tokens: [
+			Token(type: .string, inputString: "Code (**should** not process internal tags)", characterStyles: [CharacterStyle.code])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -99,8 +214,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " (should not be indented)", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -112,8 +233,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "instances", characterStyles: [CharacterStyle.code])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -121,8 +248,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "`A normal string`", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -130,8 +263,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "A string with `escaped` backticks", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -139,20 +278,47 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "A lonely backtick: `", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "Two backticks followed by a full stop ``.", output: "Two backticks followed by a full stop ``.", tokens: [
+			Token(type: .string, inputString: "Two backticks followed by a full stop ``.", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testThatItalicTraitsAreParsedCorrectly() {
 		
-		var challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
+		challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
 			Token(type: .string, inputString: "An italicised string", characterStyles: [CharacterStyle.italic])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -162,8 +328,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " text", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -178,8 +350,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "styles", characterStyles: [CharacterStyle.italic])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -188,8 +366,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "_A normal string_", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -197,8 +381,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "A string with _escaped_ underscores", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -213,21 +403,26 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "Line break", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.stringTokens.count, challenge.tokens.count )
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
 	}
 	
 	func testThatStrikethroughTraitsAreRecognised() {
-		var challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
+		challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
 			Token(type: .string, inputString: "An", characterStyles: [CharacterStyle.strikethrough]),
 			Token(type: .string, inputString: "A crossed-out string", characterStyles: [])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		
 		challenge = TokenTest(input: "A **Bold** string and a ~~removed~~crossed-out string", output: "A Bold string and a removedcrossed-out string", tokens: [
@@ -238,38 +433,56 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "crossed-out string", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		
 		challenge = TokenTest(input: "\\~\\~removed\\~\\~crossed-out string. ~This should be ignored~", output: "~~removed~~crossed-out string. ~This should be ignored~", tokens: [
 			Token(type: .string, inputString: "~~removed~~crossed-out string. ~This should be ignored~", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		
 	}
 	
 	func testThatMixedTraitsAreRecognised() {
 		
-		var challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
+		challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
 			Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold]),
-			Token(type: .string, inputString: "with a ", characterStyles: []),
+			Token(type: .string, inputString: " with a ", characterStyles: []),
 			Token(type: .string, inputString: "mix", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: " ", characterStyles: []),
 			Token(type: .string, inputString: "of", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: " bold ", characterStyles: []),
 			Token(type: .string, inputString: "styles", characterStyles: [CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
-		challenge = TokenTest(input: "_An italic string_, **follwed by a bold one**, `with some code`, \\*\\*and some\\*\\* \\_escaped\\_ \\`characters\\`, `ending` *with* __more__ variety.", output: "An italic string, follwed by a bold one, with some code, **and some** _escaped_ `characters`, ending with more variety.", tokens : [
+		challenge = TokenTest(input: "_An italic string_, **followed by a bold one**, `with some code`, \\*\\*and some\\*\\* \\_escaped\\_ \\`characters\\`, `ending` *with* __more__ variety.", output: "An italic string, followed by a bold one, with some code, **and some** _escaped_ `characters`, ending with more variety.", tokens : [
 			Token(type: .string, inputString: "An italic string", characterStyles: [CharacterStyle.italic]),
 			Token(type: .string, inputString: ", ", characterStyles: []),
 			Token(type: .string, inputString: "followed by a bold one", characterStyles: [CharacterStyle.bold]),
@@ -284,30 +497,103 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " variety.", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+	}
+	
+	func testForExtremeEscapeCombinations() {
+		
+		challenge = TokenTest(input: "\\****b\\****", output: "*b*", tokens : [
+			Token(type: .string, inputString: "*", characterStyles: []),
+			Token(type: .string, inputString: "b*", characterStyles: [CharacterStyle.bold, CharacterStyle.italic])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "**\\**b*\\***", output: "*b*", tokens : [
+			Token(type: .string, inputString: "*", characterStyles: [CharacterStyle.bold]),
+			Token(type: .string, inputString: "b", characterStyles: [CharacterStyle.italic, CharacterStyle.bold]),
+			Token(type: .string, inputString: "*", characterStyles: [CharacterStyle.bold]),
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+//		challenge = TokenTest(input: "Before *\\***\\****A bold string*\\***\\****\\ After", output: "Before ***A bold string***\\ After", tokens : [
+//			Token(type: .string, inputString: "Before ", characterStyles: []),
+//			Token(type: .string, inputString: "*", characterStyles: [CharacterStyle.italic]),
+//			Token(type: .string, inputString: "**", characterStyles: [CharacterStyle.bold]),
+//			Token(type: .string, inputString: "A bold string**", characterStyles: [CharacterStyle.bold, CharacterStyle.italic]),
+//			Token(type: .string, inputString: "\\ After", characterStyles: [])
+//		])
+//		results = self.attempt(challenge)
+//		if results.stringTokens.count == challenge.tokens.count {
+//			for (idx, token) in results.stringTokens.enumerated() {
+//				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+//				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+//			}
+//		} else {
+//			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+//		}
+//		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+//		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testThatExtraCharactersAreHandles() {
-		var challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string",  tokens: [
+		challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string",  tokens: [
 			Token(type: .string, inputString: "A bold italic string", characterStyles: [CharacterStyle.bold, CharacterStyle.italic])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
-		challenge = TokenTest(input: "A string with a ****bold italic**** word", output: "A string with a *bold italic* word",  tokens: [
+		challenge = TokenTest(input: "A string with a ****bold**** word", output: "A string with a bold word",  tokens: [
 			Token(type: .string, inputString: "A string with a ", characterStyles: []),
-			Token(type: .string, inputString: "*bold italic*", characterStyles: [CharacterStyle.bold, CharacterStyle.italic]),
+			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: " word", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -317,8 +603,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " word", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -329,21 +621,64 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: " word", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
 		challenge = TokenTest(input: "A string with a **bold*italic*bold** word", output: "A string with a bolditalicbold word",  tokens: [
 			Token(type: .string, inputString: "A string with a ", characterStyles: []),
 			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
-			Token(type: .string, inputString: "italic", characterStyles: [CharacterStyle.bold, CharacterStyle.italic]),
+			Token(type: .string, inputString: "italic", characterStyles: [CharacterStyle.italic, CharacterStyle.bold]),
 			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: " word", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "A string with ```code`", output: "A string with ```code`", tokens : [
+			Token(type: .string, inputString: "A string with ```code`", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "A string with ```code```", output: "A string with code", tokens : [
+			Token(type: .string, inputString: "A string with ", characterStyles: []),
+			Token(type: .string, inputString: "code", characterStyles: [CharacterStyle.code])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -365,14 +700,20 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 	
 	func offtestAdvancedEscaping() {
 		
-		var challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
+		challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
 			Token(type: .string, inputString: "**", characterStyles: []),
 			Token(type: .string, inputString: "A normal string", characterStyles: [CharacterStyle.italic]),
 			Token(type: .string, inputString: "**", characterStyles: [])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
@@ -382,8 +723,14 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 			Token(type: .string, inputString: "** asterisks", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
@@ -396,10 +743,9 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 		let asteriskComma = "An asterisk followed by a full stop: *, *"
 		
 		let backtickSpace = "A backtick followed by a space: `"
-		let backtickFullStop = "Two backticks followed by a full stop: ``."
 		
 		let underscoreSpace = "An underscore followed by a space: _"
-
+		
 		let backtickComma = "A backtick followed by a space: `, `"
 		let underscoreComma = "An underscore followed by a space: _, _"
 		
@@ -407,6 +753,7 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 		let underscoreWithItalic = "An _italic_ word followed by an underscore _ "
 		
 		var md = SwiftyMarkdown(string: backtickSpace)
+		SwiftyMarkdown.characterRules = self.defaultRules
 		XCTAssertEqual(md.attributedString().string, backtickSpace)
 		
 		md = SwiftyMarkdown(string: underscoreSpace)
@@ -415,9 +762,6 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 		md = SwiftyMarkdown(string: asteriskFullStop)
 		XCTAssertEqual(md.attributedString().string, asteriskFullStop)
 		
-		md = SwiftyMarkdown(string: backtickFullStop)
-		XCTAssertEqual(md.attributedString().string, backtickFullStop)
-		
 		md = SwiftyMarkdown(string: underscoreFullStop)
 		XCTAssertEqual(md.attributedString().string, underscoreFullStop)
 		
@@ -441,6 +785,22 @@ class SwiftyMarkdownCharacterTests: XCTestCase {
 		
 	}
 	
-	
+	func testReportedCrashingStrings() {
+		challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
+			Token(type: .string, inputString: "\\!bang", characterStyles: [CharacterStyle.link, CharacterStyle.bold])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+	}
 	
 }

+ 1 - 6
Tests/SwiftyMarkdownTests/SwiftyMarkdownLineTests.swift

@@ -189,12 +189,7 @@ A break
         
     }
 	
-	func testReportedCrashingStrings() {
-		let text = "[**\\!bang**](https://duckduckgo.com/bang) "
-		let expected = "\\!bang"
-		let output = SwiftyMarkdown(string: text).attributedString().string
-		XCTAssertEqual(output, expected)
-	}
+
 	
 	func testThatYAMLMetadataIsRemoved() {
 		let yaml = StringTest(input: "---\nlayout: page\ntitle: \"Trail Wallet FAQ\"\ndate: 2015-04-22 10:59\ncomments: true\nsharing: true\nliking: false\nfooter: true\nsidebar: false\n---\n# Finally some Markdown!\n\nWith A Heading\n---", expectedOutput: "Finally some Markdown!\n\nWith A Heading")

+ 632 - 77
Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift

@@ -9,35 +9,142 @@
 @testable import SwiftyMarkdown
 import XCTest
 
-class SwiftyMarkdownLinkTests: XCTestCase {
+class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 	
-	func testForLinks() {
+	func testSingleLinkPositions() {
+		challenge = TokenTest(input: "[a](b)", output: "a", tokens: [
+			Token(type: .string, inputString: "a", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge, rules: [.links])
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		if let existentOpen = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) }).first {
+			XCTAssertEqual(existentOpen.metadataStrings.first, "b")
+		} else {
+			XCTFail("Failed to find an open link tag")
+		}
 		
-		var challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
-			Token(type: .string, inputString: "Link at start", characterStyles: [CharacterStyle.link])
+		challenge = TokenTest(input: "[Link at](http://voyagetravelapps.com/) start", output: "Link at start", tokens: [
+			Token(type: .string, inputString: "Link at", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: " start")
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		if let existentOpen = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) }).first {
-			XCTAssertEqual(existentOpen.metadataString, "http://voyagetravelapps.com/")
+			XCTAssertEqual(existentOpen.metadataStrings.first, "http://voyagetravelapps.com/")
 		} else {
 			XCTFail("Failed to find an open link tag")
 		}
-
 		
-		challenge = TokenTest(input: "A [Link](http://voyagetravelapps.com/)", output: "A Link", tokens: [
+		challenge = TokenTest(input: "A [link at end](http://voyagetravelapps.com/)", output: "A link at end", tokens: [
 			Token(type: .string, inputString: "A ", characterStyles: []),
-			Token(type: .string, inputString: "Link", characterStyles: [CharacterStyle.link])
+			Token(type: .string, inputString: "link at end", characterStyles: [CharacterStyle.link])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
+		challenge = TokenTest(input: "A [link in the](http://voyagetravelapps.com/) middle", output: "A link in the middle", tokens: [
+			Token(type: .string, inputString: "A ", characterStyles: []),
+			Token(type: .string, inputString: "link in the", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: " middle", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+	}
+	
+	func testEscapedLinks() {
+		challenge = TokenTest(input: "\\[a](b)", output: "[a](b)", tokens: [
+			Token(type: .string, inputString: "[a](b)", characterStyles: [])
+		])
+		results = self.attempt(challenge, rules: [.images, .referencedLinks, .links])
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "![a](b)", output: "!a", tokens: [
+			Token(type: .string, inputString: "!", characterStyles: []),
+			Token(type: .string, inputString: "a", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge, rules: [.links])
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+	}
+	
+	func testMultipleLinkPositions() {
+		
+		challenge = TokenTest(input: "[Link 1](http://voyagetravelapps.com/)[Link 2](https://www.neverendingvoyage.com/)", output: "Link 1Link 2", tokens: [
+			Token(type: .string, inputString: "Link 1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: "Link 2", characterStyles: [CharacterStyle.link])
+		])
+		
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		if links.count == 2 {
+			XCTAssertEqual(links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+			XCTAssertEqual(links[1].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
 		
 		challenge = TokenTest(input: "[Link 1](http://voyagetravelapps.com/), [Link 2](https://www.neverendingvoyage.com/)", output: "Link 1, Link 2", tokens: [
 			Token(type: .string, inputString: "Link 1", characterStyles: [CharacterStyle.link]),
@@ -46,134 +153,582 @@ class SwiftyMarkdownLinkTests: XCTestCase {
 		])
 		
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
-		var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
-		XCTAssertEqual(links.count, 2)
-		XCTAssertEqual(links[0].metadataString, "http://voyagetravelapps.com/")
-		XCTAssertEqual(links[1].metadataString, "https://www.neverendingvoyage.com/")
+		links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		if links.count == 2 {
+			XCTAssertEqual(links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+			XCTAssertEqual(links[1].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
+		
+		challenge = TokenTest(input: "String at start [Link 1](http://voyagetravelapps.com/), [Link 2](https://www.neverendingvoyage.com/)", output: "String at start Link 1, Link 2", tokens: [
+			Token(type: .string, inputString: "String at start ", characterStyles: []),
+			Token(type: .string, inputString: "Link 1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: ", ", characterStyles: []),
+			Token(type: .string, inputString: "Link 2", characterStyles: [CharacterStyle.link])
+		])
+		
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		if links.count == 2 {
+			XCTAssertEqual(links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+			XCTAssertEqual(links[1].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
+		
+		challenge = TokenTest(input: "String at start [Link 1](http://voyagetravelapps.com/)[Link 2](https://www.neverendingvoyage.com/)", output: "String at start Link 1Link 2", tokens: [
+			Token(type: .string, inputString: "String at start ", characterStyles: []),
+			Token(type: .string, inputString: "Link 1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: "Link 2", characterStyles: [CharacterStyle.link])
+		])
+		
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		if links.count == 2 {
+			XCTAssertEqual(links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+			XCTAssertEqual(links[1].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
+		
+	}
+	
+	
+	func testForAlternativeURLs() {
+		
 		
 		challenge = TokenTest(input: "Email us at [simon@voyagetravelapps.com](mailto:simon@voyagetravelapps.com) Twitter [@VoyageTravelApp](twitter://user?screen_name=VoyageTravelApp)", output: "Email us at simon@voyagetravelapps.com Twitter @VoyageTravelApp", tokens: [
 			Token(type: .string, inputString: "Email us at ", characterStyles: []),
 			Token(type: .string, inputString: "simon@voyagetravelapps.com", characterStyles: [CharacterStyle.link]),
-			Token(type: .string, inputString: " Twitter", characterStyles: []),
+			Token(type: .string, inputString: " Twitter ", characterStyles: []),
 			Token(type: .string, inputString: "@VoyageTravelApp", characterStyles: [CharacterStyle.link])
 		])
 		
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		let links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		if links.count == 2 {
+			XCTAssertEqual(links[0].metadataStrings.first, "mailto:simon@voyagetravelapps.com")
+			XCTAssertEqual(links[1].metadataStrings.first, "twitter://user?screen_name=VoyageTravelApp")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
+	}
+		
+	func testForLinksMixedWithTokenCharacters() {
+		
+		challenge = TokenTest(input: "Link ([Surrounded by parentheses](https://www.neverendingvoyage.com/))", output: "Link (Surrounded by parentheses)", tokens: [
+			Token(type: .string, inputString: "Link (", characterStyles: []),
+			Token(type: .string, inputString: "Surrounded by parentheses", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: ")", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		if links.count == 1 {
+			XCTAssertEqual(links[0].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
+		
+		challenge = TokenTest(input: "[[Surrounded by square brackets](https://www.neverendingvoyage.com/)]", output: "[Surrounded by square brackets]", tokens: [
+			Token(type: .string, inputString: "[", characterStyles: []),
+			Token(type: .string, inputString: "Surrounded by square brackets", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: "]", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
-		XCTAssertEqual(links.count, 2)
-		XCTAssertEqual(links[0].metadataString, "mailto:simon@voyagetravelapps.com")
-		XCTAssertEqual(links[1].metadataString, "twitter://user?screen_name=VoyageTravelApp")
+		if links.count == 1 {
+			XCTAssertEqual(links[0].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
+		}
+		
+	}
 	
+	func testMalformedLinks() {
+		
+		challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/", output: "[Link with missing parenthesis](http://voyagetravelapps.com/", tokens: [
+			Token(type: .string, inputString: "[Link with missing parenthesis](http://voyagetravelapps.com/", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(results.stringTokens.count, challenge.tokens.count )
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "A [Link](http://voyagetravelapps.com/", output: "A [Link](http://voyagetravelapps.com/", tokens: [
+			Token(type: .string, inputString: "A [Link](http://voyagetravelapps.com/", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "[A link](((url)", output: "[A link](((url)", tokens: [
+			Token(type: .string, inputString: "[A link](((url)", characterStyles: [])
+		])
+		results = self.attempt(challenge, rules: [.images, .links])
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 0)
+		
+		challenge = TokenTest(input: "[[a](((b)](c)", output: "[a](((b)", tokens: [
+			Token(type: .string, inputString: "[a](((b)", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge, rules: [.images, .links])
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "c")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		
+		
 		challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
-			Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
+			Token(type: .string, inputString: "[Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "[Link with [second opening](http://voyagetravelapps.com/)", output: "[Link with second opening", tokens: [
+			Token(type: .string, inputString: "[Link with ", characterStyles: []),
+			Token(type: .string, inputString: "second opening", characterStyles: [CharacterStyle.link])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
 		
 		challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
-			Token(type: .string, inputString: "A ", characterStyles: []),
-			Token(type: .string, inputString: "[Link(http://voyagetravelapps.com/)", characterStyles: [])
+			Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/)", characterStyles: [])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
 		
-		challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/", output: "[Link with missing parenthesis](http://voyagetravelapps.com/", tokens: [
-			Token(type: .string, inputString: "[Link with missing parenthesis](", characterStyles: []),
-			Token(type: .string, inputString: "http://voyagetravelapps.com/", characterStyles: [])
+	}
+	
+	func testMalformedLinksWithValidLinks() {
+		
+		challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/ followed by a [valid link](http://voyagetravelapps.com/)", output: "[Link with missing parenthesis](http://voyagetravelapps.com/ followed by a valid link", tokens: [
+			Token(type: .string, inputString: "[Link with missing parenthesis](http://voyagetravelapps.com/ followed by a ", characterStyles: []),
+			Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.stringTokens.count, challenge.tokens.count )
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
 		
-		challenge = TokenTest(input: "A [Link](http://voyagetravelapps.com/", output: "A [Link](http://voyagetravelapps.com/", tokens: [
-			Token(type: .string, inputString: "A ", characterStyles: []),
-			Token(type: .string, inputString: "[Link](", characterStyles: []),
-			Token(type: .string, inputString: "http://voyagetravelapps.com/", characterStyles: [])
+		challenge = TokenTest(input: "A [Link](http://voyagetravelapps.com/ followed by a [valid link](http://voyagetravelapps.com/)", output: "A [Link](http://voyagetravelapps.com/ followed by a valid link", tokens: [
+			Token(type: .string, inputString: "A [Link](http://voyagetravelapps.com/ followed by a ", characterStyles: []),
+			Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
 		
-		challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2",  tokens: [
-			Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
-			Token(type: .string, inputString: " ", characterStyles: []),
-			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
-			Token(type: .string, inputString: " ", characterStyles: []),
-			Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
+		challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/) followed by a [valid link](http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/) followed by a valid link", tokens: [
+			Token(type: .string, inputString: "[Link with missing square(http://voyagetravelapps.com/) followed by a ", characterStyles: []),
+			Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
-	
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/) followed by a [valid link](http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/) followed by a valid link", tokens: [
+			Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/) followed by a ", characterStyles: []),
+			Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		
 	}
 	
 	func testLinksWithOtherStyles() {
-		var challenge = TokenTest(input: "A **Bold [Link](http://voyagetravelapps.com/)**", output: "A Bold Link", tokens: [
+		challenge = TokenTest(input: "A **Bold [Link](http://voyagetravelapps.com/)**", output: "A Bold Link", tokens: [
 			Token(type: .string, inputString: "A ", characterStyles: []),
 			Token(type: .string, inputString: "Bold ", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: "Link", characterStyles: [CharacterStyle.link, CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-//		XCTAssertEqual(results.attributedString.string, challenge.output)
-		var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
-		XCTAssertEqual(links.count, 1)
-		if links.count == 1 {
-			XCTAssertEqual(links[0].metadataString, "http://voyagetravelapps.com/")
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "http://voyagetravelapps.com/")
 		} else {
-			XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
 		}
 		
 		challenge = TokenTest(input: "A Bold [**Link**](http://voyagetravelapps.com/)", output: "A Bold Link", tokens: [
 			Token(type: .string, inputString: "A Bold ", characterStyles: []),
-			Token(type: .string, inputString: "Link", characterStyles: [CharacterStyle.bold, CharacterStyle.link])
+			Token(type: .string, inputString: "Link", characterStyles: [ CharacterStyle.link, CharacterStyle.bold])
 		])
 		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2",  tokens: [
+			Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
-		links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
-		XCTAssertEqual(links.count, 1)
-		XCTAssertEqual(links[0].metadataString, "http://voyagetravelapps.com/")
 	}
 	
 	func testForImages() {
-		let challenge = TokenTest(input: "An ![Image](imageName)", output: "An Image", tokens: [
-			Token(type: .string, inputString: "An Image", characterStyles: []),
-			Token(type: .string, inputString: "", characterStyles: [CharacterStyle.image])
+		challenge = TokenTest(input: "An ![Image](imageName)", output: "An ", tokens: [
+			Token(type: .string, inputString: "An ", characterStyles: []),
+			Token(type: .string, inputString: "Image", characterStyles: [CharacterStyle.image])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		if results.images.count == 1 {
+			XCTAssertEqual(results.images[0].metadataStrings.first, "imageName")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.images.count)")
+		}
+		
+		challenge = TokenTest(input: "An [![Image](imageName)](https://www.neverendingvoyage.com/)", output: "An ", tokens: [
+			Token(type: .string, inputString: "An ", characterStyles: []),
+			Token(type: .string, inputString: "Image", characterStyles: [CharacterStyle.image, CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		if results.images.count == 1 {
+			XCTAssertEqual(results.images[0].metadataStrings.first, "imageName")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.images.count)")
+		}
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.last, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+	}
+	
+	func testForReferencedImages() {
+		challenge = TokenTest(input: "A ![referenced image][image]\n[image]: imageName", output: "A ", tokens: [
+			Token(type: .string, inputString: "A ", characterStyles: []),
+			Token(type: .string, inputString: "referenced image", characterStyles: [CharacterStyle.image])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		if results.images.count == 1 {
+			XCTAssertEqual(results.images[0].metadataStrings.first, "imageName")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+	}
+	
+	func testForReferencedLinks() {
+		challenge = TokenTest(input: "A [referenced link][link]\n[link]: https://www.neverendingvoyage.com/", output: "A referenced link", tokens: [
+			Token(type: .string, inputString: "A ", characterStyles: []),
+			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "A [referenced link][link]\n  [link]: https://www.neverendingvoyage.com/", output: "A referenced link", tokens: [
+			Token(type: .string, inputString: "A ", characterStyles: []),
+			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
 		])
-		let results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		let links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.image) ?? false) })
-		XCTAssertEqual(links.count, 1)
-		XCTAssertEqual(links[0].metadataString, "imageName")
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "https://www.neverendingvoyage.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "An *\\*italic\\** [referenced link][a]\n[a]: link", output: "An *italic* referenced link", tokens: [
+			Token(type: .string, inputString: "An ", characterStyles: []),
+			Token(type: .string, inputString: "*italic*", characterStyles: [CharacterStyle.italic]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge, rules: [.asterisks, .links, .referencedLinks])
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "link")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
 	}
 	
+	func testForMixedLinkStyles() {
+		challenge = TokenTest(input: "A [referenced link][link] and a [regular link](http://voyagetravelapps.com/)\n[link]: https://www.neverendingvoyage.com/", output: "A referenced link and a regular link", tokens: [
+			Token(type: .string, inputString: "A ", characterStyles: []),
+			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: " and a ", characterStyles: []),
+			Token(type: .string, inputString: "regular link", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		if results.stringTokens.count == challenge.tokens.count {
+			for (idx, token) in results.stringTokens.enumerated() {
+				XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
+				XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as?  [CharacterStyle])
+			}
+		} else {
+			XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
+		}
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		if results.links.count == 2 {
+			XCTAssertEqual(results.links[0].metadataStrings.first, "https://www.neverendingvoyage.com/")
+			XCTAssertEqual(results.links[1].metadataStrings.first, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+	}
 	
 }

+ 1 - 1
Tests/SwiftyMarkdownTests/SwiftyMarkdownPerformanceTests.swift

@@ -33,7 +33,7 @@ class SwiftyMarkdownPerformanceTests: XCTestCase {
 		}
 	}
 	
-	func testThatVeryLongStringsAreProcessedQuickly() {
+ 	func testThatVeryLongStringsAreProcessedQuickly() {
 		let string = "SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a *Swift*-style syntax. It uses **dynamic type** to set the font size correctly with [whatever](https://www.neverendingvoyage.com/) font you'd like to use. SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a *Swift*-style syntax. It uses **dynamic type** to set the font size correctly with [whatever](https://www.neverendingvoyage.com/) font you'd like to use. SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a *Swift*-style syntax. It uses **dynamic type** to set the font size correctly with [whatever](https://www.neverendingvoyage.com/) font you'd like to use. SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a *Swift*-style syntax. It uses **dynamic type** to set the font size correctly with [whatever](https://www.neverendingvoyage.com/) font you'd like to use. SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a *Swift*-style syntax. It uses **dynamic type** to set the font size correctly with [whatever](https://www.neverendingvoyage.com/) font you'd like to use. SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a *Swift*-style syntax. It uses **dynamic type** to set the font size correctly with [whatever](https://www.neverendingvoyage.com/) font you'd like to use."
 		let md = SwiftyMarkdown(string: string)
 		measure {

+ 74 - 9
Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift

@@ -9,23 +9,88 @@
 import XCTest
 @testable import SwiftyMarkdown
 
-extension XCTestCase {
+
+struct ChallengeReturn {
+	let tokens : [Token]
+	let stringTokens : [Token]
+	let links : [Token]
+	let images : [Token]
+	let attributedString : NSAttributedString
+	let foundStyles : [[CharacterStyle]]
+	let expectedStyles : [[CharacterStyle]]
+}
+
+enum Rule {
+	case asterisks
+	case backticks
+	case underscores
+	case images
+	case links
+	case referencedLinks
+	case referencedImages
+	case tildes
 	
-	func resourceURL(for filename : String ) -> URL {
-		let thisSourceFile = URL(fileURLWithPath: #file)
-		let thisDirectory = thisSourceFile.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
-		return thisDirectory.appendingPathComponent("Resources", isDirectory: true).appendingPathComponent(filename)
+	func asCharacterRule() -> CharacterRule {
+		switch self {
+		case .images:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "![" && !$0.metadataLookup  }).first!
+		case .links:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "[" && !$0.metadataLookup  }).first!
+		case .backticks:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "`" }).first!
+		case .tildes:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "~" }).first!
+		case .asterisks:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "*" }).first!
+		case .underscores:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "_" }).first!
+		case .referencedLinks:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "[" && $0.metadataLookup  }).first!
+		case .referencedImages:
+			return SwiftyMarkdown.characterRules.filter({ $0.primaryTag.tag == "![" && $0.metadataLookup  }).first!
+		}
 	}
+}
+
+class SwiftyMarkdownCharacterTests : XCTestCase {
+	let defaultRules = SwiftyMarkdown.characterRules
+	
+	var challenge : TokenTest!
+	var results : ChallengeReturn!
 	
-	func attempt( _ challenge : TokenTest ) -> (tokens : [Token], stringTokens: [Token], attributedString : NSAttributedString, foundStyles : [[CharacterStyle]], expectedStyles : [[CharacterStyle]] ) {
+	func attempt( _ challenge : TokenTest, rules : [Rule]? = nil ) -> ChallengeReturn {
+		if let validRules = rules {
+			SwiftyMarkdown.characterRules = validRules.map({ $0.asCharacterRule() })
+		} else {
+			SwiftyMarkdown.characterRules = self.defaultRules
+		}
+		
 		let md = SwiftyMarkdown(string: challenge.input)
-		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
-		let tokens = tokeniser.process(challenge.input)
+		md.applyAttachments = false
+		let attributedString = md.attributedString()
+		let tokens : [Token] = md.previouslyFoundTokens
 		let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
 		
 		let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
 		let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
 		
-		return (tokens, stringTokens, md.attributedString(), existentTokenStyles, expectedStyles)
+		let linkTokens = tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		let imageTokens = tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.image) ?? false) })
+		
+		return ChallengeReturn(tokens: tokens, stringTokens: stringTokens, links : linkTokens, images: imageTokens, attributedString:  attributedString, foundStyles: existentTokenStyles, expectedStyles : expectedStyles)
 	}
 }
+
+
+extension XCTestCase {
+	
+	func resourceURL(for filename : String ) -> URL {
+		let thisSourceFile = URL(fileURLWithPath: #file)
+		let thisDirectory = thisSourceFile.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
+		return thisDirectory.appendingPathComponent("Resources", isDirectory: true).appendingPathComponent(filename)
+	}
+	
+
+}
+
+