Cleans up codebase and class names

Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj

@@ -7,7 +7,6 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
-		F421DD991C8AF4E900B86D66 /* example.md in Resources */ = {isa = PBXBuildFile; fileRef = F421DD951C8AF34F00B86D66 /* example.md */; };
 		F4B4A44C23E4E17400550249 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B4A44B23E4E17400550249 /* AppDelegate.swift */; };
 		F4B4A44E23E4E17400550249 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B4A44D23E4E17400550249 /* ViewController.swift */; };
 		F4B4A45023E4E17400550249 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = F4B4A44F23E4E17400550249 /* Assets.xcassets */; };
@@ -20,6 +19,7 @@
 		F4CE98B61C8AEF7D00D735C1 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */; };
 		F4CE98C11C8AEF7D00D735C1 /* SwiftyMarkdownExampleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4CE98C01C8AEF7D00D735C1 /* SwiftyMarkdownExampleTests.swift */; };
 		F4CE98CC1C8AEF7D00D735C1 /* SwiftyMarkdownExampleUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4CE98CB1C8AEF7D00D735C1 /* SwiftyMarkdownExampleUITests.swift */; };
+		F4EAB653244179FE00206782 /* example.md in Resources */ = {isa = PBXBuildFile; fileRef = F4576C2E2437F67B0013E2B6 /* example.md */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -62,8 +62,7 @@
 /* End PBXCopyFilesBuildPhase section */
 
 /* Begin PBXFileReference section */
-		F421DD951C8AF34F00B86D66 /* example.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = "<group>"; };
-		F4576C2E2437F67B0013E2B6 /* example copy.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = "example copy.md"; sourceTree = "<group>"; };
+		F4576C2E2437F67B0013E2B6 /* example.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = "<group>"; };
 		F4B4A44923E4E17400550249 /* SwiftyMarkdownExample macOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "SwiftyMarkdownExample macOS.app"; sourceTree = BUILT_PRODUCTS_DIR; };
 		F4B4A44B23E4E17400550249 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
 		F4B4A44D23E4E17400550249 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
@@ -173,8 +172,7 @@
 				F4CE98B21C8AEF7D00D735C1 /* Assets.xcassets */,
 				F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */,
 				F4CE98B71C8AEF7D00D735C1 /* Info.plist */,
-				F4576C2E2437F67B0013E2B6 /* example copy.md */,
-				F421DD951C8AF34F00B86D66 /* example.md */,
+				F4576C2E2437F67B0013E2B6 /* example.md */,
 			);
 			path = SwiftyMarkdownExample;
 			sourceTree = "<group>";
@@ -345,8 +343,8 @@
 			buildActionMask = 2147483647;
 			files = (
 				F4CE98B61C8AEF7D00D735C1 /* LaunchScreen.storyboard in Resources */,
+				F4EAB653244179FE00206782 /* example.md in Resources */,
 				F4CE98B31C8AEF7D00D735C1 /* Assets.xcassets in Resources */,
-				F421DD991C8AF4E900B86D66 /* example.md in Resources */,
 				F4CE98B11C8AEF7D00D735C1 /* Main.storyboard in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

Example/SwiftyMarkdownExample/example copy.md

@@ -1,50 +0,0 @@
-# Swifty Markdown
-
-SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts.
-
-Show Images From Your App Bundle!
----
-![Image](bubble)
-
-Customise fonts and colors easily in a Swift-like way: 
-
-    md.code.fontName = "CourierNewPSMT"
-
-    md.h2.fontName = "AvenirNextCondensed-Medium"
-    md.h2.color = UIColor.redColor()
-    md.h2.alignment = .center
-
-It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings.
-
-It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/).
-
-> It also now supports blockquotes
-> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun!
-
-**Lists**
-
-- It Supports
-- Unordered
-- Lists
-	- Indented item with a longer string to make sure indentation is consistent
-		- Second level indent with a longer string to make sure indentation is consistent
-- List item with a longer string to make sure indentation is consistent
-
-1. And
-1. Ordered
-1. Lists
-	1. Indented item
-		1. Second level indent
-1. (Use `1.` as the list item identifier)
-1. List item
-1. List item
-	- Mix
-		- List styles
-1. List item with a longer string to make sure indentation is consistent
-1. List item
-1. List item
-1. List item
-1. List item
-
-
-

Example/SwiftyMarkdownExample/example.md

@@ -1 +1,50 @@
-[a](b)
+# Swifty Markdown
+
+SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts.
+
+Show Images From Your App Bundle!
+---
+![Image](bubble)
+
+Customise fonts and colors easily in a Swift-like way: 
+
+    md.code.fontName = "CourierNewPSMT"
+
+    md.h2.fontName = "AvenirNextCondensed-Medium"
+    md.h2.color = UIColor.redColor()
+    md.h2.alignment = .center
+
+It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings.
+
+It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/).
+
+> It also now supports blockquotes
+> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun!
+
+**Lists**
+
+- It Supports
+- Unordered
+- Lists
+	- Indented item with a longer string to make sure indentation is consistent
+		- Second level indent with a longer string to make sure indentation is consistent
+- List item with a longer string to make sure indentation is consistent
+
+1. And
+1. Ordered
+1. Lists
+	1. Indented item
+		1. Second level indent
+1. (Use `1.` as the list item identifier)
+1. List item
+1. List item
+	- Mix
+		- List styles
+1. List item with a longer string to make sure indentation is consistent
+1. List item
+1. List item
+1. List item
+1. List item
+
+
+

Sources/SwiftyMarkdown/SwiftyMarkdown.swift

@@ -188,7 +188,7 @@ If that is not set, then the system default will be used.
 				CharacterRuleTag(tag: "(", type: .metadataOpen),
 				CharacterRuleTag(tag: ")", type: .metadataClose)
 		], styles: [1 : CharacterStyle.link], metadataLookup: false, definesBoundary: true),
-		CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingTags: true, balancedTags: true),
+		CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingRules: true, balancedTags: true),
 		CharacterRule(primaryTag:CharacterRuleTag(tag: "~", type: .repeating), otherTags : [], styles: [2 : CharacterStyle.strikethrough], minTags:2 , maxTags:2),
 		CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2),
 		CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2)

Sources/SwiftyMarkdown/SwiftyScanner.swift

@@ -1,3 +1,10 @@
+//
+//  SwiftyScanner.swift
+//  
+//
+//  Created by Simon Fairbairn on 04/04/2020.
+//
+
 //
 //  SwiftyScanner.swift
 //  SwiftyMarkdown
@@ -10,50 +17,548 @@ import os.log
 
 extension OSLog {
 	private static var subsystem = "SwiftyScanner"
-	static let swiftyScannerTokenising = OSLog(subsystem: subsystem, category: "Swifty Scanner Tokenising")
-	static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Peformance")
+	static let swiftyScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner")
+	static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance")
 }
 
-/// Swifty Scanning Protocol
-public protocol SwiftyScanning {
-	var metadataLookup : [String : String] { get set }
-	func scan( _ string : String, with rule : CharacterRule) -> [Token]
-	func scan( _ tokens : [Token], with rule : CharacterRule) -> [Token]
+enum RepeatingTagType {
+	case open
+	case either
+	case close
+	case neither
 }
 
-enum TagState {
-	case none
-	case open
-	case intermediate
-	case closed
+struct TagGroup {
+	let groupID  = UUID().uuidString
+	var tagRanges : [ClosedRange<Int>]
+	var tagType : RepeatingTagType = .open
+	var count = 1
 }
 
-class SwiftyScanner : SwiftyScanning {
-	var metadataLookup: [String : String] = [:]
+class SwiftyScanner {
+	var elements : [Element]
+	let rule : CharacterRule
+	let metadata : [String : String]
+	var pointer : Int = 0
+	
+	var spaceAndNewLine = CharacterSet.whitespacesAndNewlines
+	var tagGroups : [TagGroup] = []
+	
+	var isMetadataOpen = false
 	
-	init() {
+	
+	var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil)
+	
+	let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance)
+	let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScanner)
 		
+	
+	
+	enum Position {
+		case forward(Int)
+		case backward(Int)
 	}
 	
-	func scan(_ string: String, with rule: CharacterRule) -> [Token] {
-		return []
+	init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) {
+		self.elements = elements
+		self.rule = rule
+		self.currentPerfomanceLog.start()
+		self.metadata = metadata
 	}
 	
-	func scan(_ tokens: [Token], with rule: CharacterRule) -> [Token] {
-		return tokens
+	func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? {
+		
+		let newIdx : Int
+		var isForward = true
+		switch newPosition {
+		case .backward(let positions):
+			isForward = false
+			newIdx = pointer - positions
+			if newIdx < 0 {
+				return nil
+			}
+		case .forward(let positions):
+			newIdx = pointer + positions
+			if newIdx >= self.elements.count {
+				return nil
+			}
+		}
+		
+		
+		let range : ClosedRange<Int> = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer
+		return Array(self.elements[range])
 	}
+	
+	
+	func element( for position : Position ) -> Element? {
+		let newIdx : Int
+		switch position {
+		case .backward(let positions):
+			newIdx = pointer - positions
+			if newIdx < 0 {
+				return nil
+			}
+		case .forward(let positions):
+			newIdx = pointer + positions
+			if newIdx >= self.elements.count {
+				return nil
+			}
+		}
+		return self.elements[newIdx]
+	}
+	
+	
+	func positionIsEqualTo( character : Character, direction : Position ) -> Bool {
+		guard let validElement = self.element(for: direction) else {
+			return false
+		}
+		return validElement.character == character
+	}
+	
+	func positionContains( characters : [Character], direction : Position ) -> Bool {
+		guard let validElement = self.element(for: direction) else {
+			return false
+		}
+		return characters.contains(validElement.character)
+	}
+	
+	func isEscaped() -> Bool {
+		let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1))
+		if isEscaped {
+			self.elements[self.pointer - 1].type = .escape
+		}
+		return isEscaped
+	}
+	
+	func range( for tag : String? ) -> ClosedRange<Int>? {
 
-}
+		guard let tag = tag else {
+			return nil
+		}
+		
+		guard let openChar = tag.first else {
+			return nil
+		}
+		
+		if self.pointer == self.elements.count {
+			return nil
+		}
+		
+		if self.elements[self.pointer].character != openChar {
+			return nil
+		}
+		
+		if isEscaped() {
+			return nil
+		}
+		
+		let range : ClosedRange<Int>
+		if tag.count > 1 {
+			guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else {
+				return nil
+			}
+			// If it's already a tag, then it should be ignored
+			if elements.filter({ $0.type != .string }).count > 0 {
+				return nil
+			}
+			if elements.map( { String($0.character) }).joined() != tag {
+				return nil
+			}
+			let endIdx = (self.pointer + tag.count - 1)
+			for i in self.pointer...endIdx {
+				self.elements[i].type = .tag
+			}
+			range = self.pointer...endIdx
+			self.pointer += tag.count
+		} else {
+			// If it's already a tag, then it should be ignored
+			if self.elements[self.pointer].type != .string {
+				return nil
+			}
+			self.elements[self.pointer].type = .tag
+			range = self.pointer...self.pointer
+			self.pointer += 1
+		}
+		return range
+	}
+	
+	
+	func resetTagGroup( withID id : String ) {
+		if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) {
+			for range in self.tagGroups[idx].tagRanges {
+				self.resetTag(in: range)
+			}
+			self.tagGroups.remove(at: idx)
+		}
+		self.isMetadataOpen = false
+	}
+	
+	func resetTag( in range : ClosedRange<Int>) {
+		for idx in range {
+			self.elements[idx].type = .string
+		}
+	}
+	
+	func resetLastTag( for range : inout [ClosedRange<Int>]) {
+		guard let last = range.last else {
+			return
+		}
+		for idx in last {
+			self.elements[idx].type = .string
+		}
+	}
+	
+	func closeTag( _ tag : String, withGroupID id : String ) {
+
+		guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else {
+			return
+		}
+
+		var metadataString = ""
+		if self.isMetadataOpen {
+			let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+			let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+			
+			if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) {
+				if self.enableLog {
+					os_log("Nothing between the tags", log: OSLog.swiftyScanner, type:.info , self.rule.description)
+				}
+			} else {
+				for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) {
+					self.elements[idx].type = .metadata
+					if self.rule.definesBoundary {
+						self.elements[idx].boundaryCount += 1
+					}
+				}
+				
+				
+				let key = self.elements[metadataOpenRange.upperBound + 1..<metadataCloseRange.lowerBound].map( { String( $0.character )}).joined()
+				if self.rule.metadataLookup {
+					metadataString = self.metadata[key] ?? ""
+				} else {
+					metadataString = key
+				}
+			}
+		}
+		
+		let closeRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+		let openRange = self.tagGroups[tagIdx].tagRanges.removeLast()
+
+		if self.rule.balancedTags && closeRange.count != openRange.count {
+			self.tagGroups[tagIdx].tagRanges.append(openRange)
+			self.tagGroups[tagIdx].tagRanges.append(closeRange)
+			return
+		}
+
+		var shouldRemove = true
+		var styles : [CharacterStyling] = []
+		if openRange.upperBound + 1 == (closeRange.lowerBound) {
+			if self.enableLog {
+				os_log("Nothing between the tags", log: OSLog.swiftyScanner, type:.info , self.rule.description)
+			}
+		} else {
+			var remainingTags = min(openRange.upperBound - openRange.lowerBound, closeRange.upperBound - closeRange.lowerBound) + 1
+			while remainingTags > 0 {
+				if remainingTags >= self.rule.maxTags {
+					remainingTags -= self.rule.maxTags
+					if let style = self.rule.styles[ self.rule.maxTags ] {
+						if !styles.contains(where: { $0.isEqualTo(style)}) {
+							styles.append(style)
+						}
+					}
+				}
+				if let style = self.rule.styles[remainingTags] {
+					remainingTags -= remainingTags
+					if !styles.contains(where: { $0.isEqualTo(style)}) {
+						styles.append(style)
+					}
+				}
+			}
+			
+			for idx in (openRange.upperBound)...(closeRange.lowerBound) {
+				self.elements[idx].styles.append(contentsOf: styles)
+				self.elements[idx].metadata.append(metadataString)
+				if self.rule.definesBoundary {
+					self.elements[idx].boundaryCount += 1
+				}
+				if self.rule.shouldCancelRemainingRules {
+					self.elements[idx].boundaryCount = 1000
+				}
+			}
+			
+			if self.rule.isRepeatingTag {
+				let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound)
+				switch difference {
+				case 1...:
+					shouldRemove = false
+					self.tagGroups[tagIdx].count = difference
+					self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound )
+				case ...(-1):
+					for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound {
+						self.elements[idx].type = .string
+					}
+				default:
+					break
+				}
+			}
+			
+		}
+		if shouldRemove {
+			self.tagGroups.removeAll(where: { $0.groupID == id })
+		}
+		self.isMetadataOpen = false
+	}
+	
+	func emptyRanges( _ ranges : inout [ClosedRange<Int>] ) {
+		while !ranges.isEmpty {
+			self.resetLastTag(for: &ranges)
+			ranges.removeLast()
+		}
+	}
+	
+	func scanNonRepeatingTags() {
+		var groupID = ""
+		let closeTag = self.rule.tag(for: .close)?.tag
+		let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag
+		let metadataClose = self.rule.tag(for: .metadataClose)?.tag
+		
+		while self.pointer < self.elements.count {
+			if self.enableLog {
+				os_log("CHARACTER: %@", log: OSLog.swiftyScanner, type:.info , String(self.elements[self.pointer].character))
+			}
+			
+			if let range = self.range(for: metadataClose) {
+				if self.isMetadataOpen {
+					guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else {
+						self.pointer += 1
+						continue
+					}
+					
+					guard !self.tagGroups.isEmpty else {
+						self.resetTagGroup(withID: groupID)
+						continue
+					}
+				
+					guard self.isMetadataOpen else {
+						
+						self.resetTagGroup(withID: groupID)
+						continue
+					}
+					if self.enableLog {
+						os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
+					}
+					self.tagGroups[groupIdx].tagRanges.append(range)
+					self.closeTag(closeTag!, withGroupID: groupID)
+					self.isMetadataOpen = false
+					continue
+				} else {
+					self.resetTag(in: range)
+					self.pointer -= metadataClose!.count
+				}
+
+			}
+			
+			if let openRange = self.range(for: self.rule.primaryTag.tag) {
+				if self.isMetadataOpen {
+					self.resetTagGroup(withID: groupID)
+				}
+				
+				let tagGroup = TagGroup(tagRanges: [openRange])
+				groupID = tagGroup.groupID
+				if self.enableLog {
+					os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
+				}
+				if self.rule.isRepeatingTag {
+					
+				}
+				
+				self.tagGroups.append(tagGroup)
+				continue
+			}
+	
+			if let range = self.range(for: closeTag) {
+				guard !self.tagGroups.isEmpty else {
+					if self.enableLog {
+						os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScanner, type:.info)
+					}
+					self.resetTag(in: range)
+					continue
+				}
+				self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
+				groupID = self.tagGroups[self.tagGroups.count - 1].groupID
+				if self.enableLog {
+					os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
+				}
+				guard metadataOpen != nil else {
+					if self.enableLog {
+						os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
+					}
+					self.closeTag(closeTag!, withGroupID: groupID)
+					continue
+				}
+				
+				guard self.pointer != self.elements.count else {
+					continue
+				}
+				
+				guard let range = self.range(for: metadataOpen) else {
+					if self.enableLog {
+						os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
+					}
+					self.resetTagGroup(withID: groupID)
+					continue
+				}
+				self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
+				self.isMetadataOpen = true
+				continue
+			}
+			
 
-struct TokenGroup {
-	enum TokenGroupType {
-		case string
-		case tag
-		case escape
+			if let range = self.range(for: metadataOpen) {
+				if self.enableLog {
+					os_log("Multiple open metadata tags found!", log: OSLog.swiftyScanner, type:.info , groupID)
+				}
+				self.resetTag(in: range)
+				self.resetTagGroup(withID: groupID)
+				self.isMetadataOpen = false
+				continue
+			}
+			self.pointer += 1
+		}
+	}
+	
+	func scanRepeatingTags() {
+				
+		var groupID = ""
+		let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined()
+		let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters))
+		while self.pointer < self.elements.count {
+			if self.enableLog {
+				os_log("CHARACTER: %@", log: OSLog.swiftyScanner, type:.info , String(self.elements[self.pointer].character))
+			}
+			
+			if var openRange = self.range(for: self.rule.primaryTag.tag) {
+				
+				if self.elements[openRange].first?.boundaryCount == 1000 {
+					self.resetTag(in: openRange)
+					continue
+				}
+				
+				var count = 1
+				var tagType : RepeatingTagType = .open
+				if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1))  {
+					if !unionSet.containsUnicodeScalars(of: prevElement.character) {
+						tagType = .either
+					}
+				} else {
+					tagType = .open
+				}
+				
+				while let nextRange = self.range(for: self.rule.primaryTag.tag)  {
+					count += 1
+					openRange = openRange.lowerBound...nextRange.upperBound
+				}
+				
+				if self.rule.minTags > 1 {
+					if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags {
+						self.resetTag(in: openRange)
+						os_log("Tag does not meet minimum length", log: .swiftyScanner, type: .info)
+						continue
+					}
+				}
+				
+				var validTagGroup = true
+				if let nextElement = self.element(for: .forward(0)) {
+					if unionSet.containsUnicodeScalars(of: nextElement.character) {
+						if tagType == .either {
+							tagType = .close
+						} else {
+							validTagGroup = tagType != .open
+						}
+					}
+				} else {
+					if tagType == .either {
+						tagType = .close
+					} else {
+						validTagGroup = tagType != .open
+					}
+				}
+				
+				if !validTagGroup {
+					if self.enableLog {
+						os_log("Tag has whitespace on both sides", log: .swiftyScanner, type: .info)
+					}
+					self.resetTag(in: openRange)
+					continue
+				}
+				
+				if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) {
+					if tagType == .either {
+						if tagGroups[idx].count == count {
+							self.tagGroups[idx].tagRanges.append(openRange)
+							self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
+							
+							if let last = self.tagGroups.last {
+								groupID = last.groupID
+							}
+							
+							continue
+						}
+					} else {
+						if let prevRange = tagGroups[idx].tagRanges.first {
+							if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount {
+								self.tagGroups[idx].tagRanges.append(openRange)
+								self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
+							}
+						}
+						continue
+					}
+				}
+				var tagGroup = TagGroup(tagRanges: [openRange])
+				groupID = tagGroup.groupID
+				tagGroup.tagType = tagType
+				tagGroup.count = count
+				
+				if self.enableLog {
+					os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScanner,  type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID)
+				}
+				
+				self.tagGroups.append(tagGroup)
+				continue
+			}
+	
+			self.pointer += 1
+		}
 	}
 	
-	let string : String
-	let isEscaped : Bool
-	let type : TokenGroupType
-	var state : TagState = .none
+	
+	func scan() -> [Element] {
+		
+		guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else {
+			return self.elements
+		}
+		
+		self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)")
+		
+		if self.enableLog {
+			os_log("RULE: %@", log: OSLog.swiftyScanner, type:.info , self.rule.description)
+		}
+		
+		if self.rule.isRepeatingTag {
+			self.scanRepeatingTags()
+		} else {
+			self.scanNonRepeatingTags()
+		}
+		
+		for tagGroup in self.tagGroups {
+			self.resetTagGroup(withID: tagGroup.groupID)
+		}
+		
+		if self.enableLog {
+			for element in self.elements {
+				print(element)
+			}
+		}
+		return self.elements
+	}
 }

Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift

@@ -1,565 +0,0 @@
-//
-//  File.swift
-//  
-//
-//  Created by Simon Fairbairn on 04/04/2020.
-//
-
-//
-//  SwiftyScanner.swift
-//  SwiftyMarkdown
-//
-//  Created by Simon Fairbairn on 04/02/2020.
-//
-
-import Foundation
-import os.log
-
-
-extension OSLog {
-	private static var subsystem = "SwiftyScanner"
-	static let swiftyScannerScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner")
-	static let swiftyScannerScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance")
-}
-
-enum RepeatingTagType {
-	case open
-	case either
-	case close
-	case neither
-}
-
-struct TagGroup {
-	let groupID  = UUID().uuidString
-	var tagRanges : [ClosedRange<Int>]
-	var tagType : RepeatingTagType = .open
-	var count = 1
-}
-
-class SwiftyScannerNonRepeating {
-	var elements : [Element]
-	let rule : CharacterRule
-	let metadata : [String : String]
-	var pointer : Int = 0
-	
-	var spaceAndNewLine = CharacterSet.whitespacesAndNewlines
-	var tagGroups : [TagGroup] = []
-	
-	var isMetadataOpen = false
-	
-	
-	var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil)
-	
-	let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance)
-	let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScannerScanner)
-		
-	
-	
-	enum Position {
-		case forward(Int)
-		case backward(Int)
-	}
-	
-	init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) {
-		self.elements = elements
-		self.rule = rule
-		self.currentPerfomanceLog.start()
-		self.metadata = metadata
-	}
-	
-	func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? {
-		
-		let newIdx : Int
-		var isForward = true
-		switch newPosition {
-		case .backward(let positions):
-			isForward = false
-			newIdx = pointer - positions
-			if newIdx < 0 {
-				return nil
-			}
-		case .forward(let positions):
-			newIdx = pointer + positions
-			if newIdx >= self.elements.count {
-				return nil
-			}
-		}
-		
-		
-		let range : ClosedRange<Int> = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer
-		return Array(self.elements[range])
-	}
-	
-	
-	func element( for position : Position ) -> Element? {
-		let newIdx : Int
-		switch position {
-		case .backward(let positions):
-			newIdx = pointer - positions
-			if newIdx < 0 {
-				return nil
-			}
-		case .forward(let positions):
-			newIdx = pointer + positions
-			if newIdx >= self.elements.count {
-				return nil
-			}
-		}
-		return self.elements[newIdx]
-	}
-	
-	
-	func positionIsEqualTo( character : Character, direction : Position ) -> Bool {
-		guard let validElement = self.element(for: direction) else {
-			return false
-		}
-		return validElement.character == character
-	}
-	
-	func positionContains( characters : [Character], direction : Position ) -> Bool {
-		guard let validElement = self.element(for: direction) else {
-			return false
-		}
-		return characters.contains(validElement.character)
-	}
-	
-	func isEscaped() -> Bool {
-		let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1))
-		if isEscaped {
-			self.elements[self.pointer - 1].type = .escape
-		}
-		return isEscaped
-	}
-	
-	func range( for tag : String? ) -> ClosedRange<Int>? {
-
-		guard let tag = tag else {
-			return nil
-		}
-		
-		guard let openChar = tag.first else {
-			return nil
-		}
-		
-		if self.pointer == self.elements.count {
-			return nil
-		}
-		
-		if self.elements[self.pointer].character != openChar {
-			return nil
-		}
-		
-		if isEscaped() {
-			return nil
-		}
-		
-		let range : ClosedRange<Int>
-		if tag.count > 1 {
-			guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else {
-				return nil
-			}
-			// If it's already a tag, then it should be ignored
-			if elements.filter({ $0.type != .string }).count > 0 {
-				return nil
-			}
-			if elements.map( { String($0.character) }).joined() != tag {
-				return nil
-			}
-			let endIdx = (self.pointer + tag.count - 1)
-			for i in self.pointer...endIdx {
-				self.elements[i].type = .tag
-			}
-			range = self.pointer...endIdx
-			self.pointer += tag.count
-		} else {
-			// If it's already a tag, then it should be ignored
-			if self.elements[self.pointer].type != .string {
-				return nil
-			}
-			self.elements[self.pointer].type = .tag
-			range = self.pointer...self.pointer
-			self.pointer += 1
-		}
-		return range
-	}
-	
-	
-	func resetTagGroup( withID id : String ) {
-		if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) {
-			for range in self.tagGroups[idx].tagRanges {
-				self.resetTag(in: range)
-			}
-			self.tagGroups.remove(at: idx)
-		}
-		self.isMetadataOpen = false
-	}
-	
-	func resetTag( in range : ClosedRange<Int>) {
-		for idx in range {
-			self.elements[idx].type = .string
-		}
-	}
-	
-	func resetLastTag( for range : inout [ClosedRange<Int>]) {
-		guard let last = range.last else {
-			return
-		}
-		for idx in last {
-			self.elements[idx].type = .string
-		}
-	}
-	
-	func closeTag( _ tag : String, withGroupID id : String ) {
-
-		guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else {
-			return
-		}
-
-		var metadataString = ""
-		if self.isMetadataOpen {
-			let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast()
-			let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast()
-			
-			if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) {
-				if self.enableLog {
-					os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
-				}
-			} else {
-				for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) {
-					self.elements[idx].type = .metadata
-					if self.rule.definesBoundary {
-						self.elements[idx].boundaryCount += 1
-					}
-				}
-				
-				
-				let key = self.elements[metadataOpenRange.upperBound + 1..<metadataCloseRange.lowerBound].map( { String( $0.character )}).joined()
-				if self.rule.metadataLookup {
-					metadataString = self.metadata[key] ?? ""
-				} else {
-					metadataString = key
-				}
-			}
-		}
-		
-		let closeRange = self.tagGroups[tagIdx].tagRanges.removeLast()
-		let openRange = self.tagGroups[tagIdx].tagRanges.removeLast()
-
-		if self.rule.balancedTags && closeRange.count != openRange.count {
-			self.tagGroups[tagIdx].tagRanges.append(openRange)
-			self.tagGroups[tagIdx].tagRanges.append(closeRange)
-			return
-		}
-
-		var shouldRemove = true
-		var styles : [CharacterStyling] = []
-		if openRange.upperBound + 1 == (closeRange.lowerBound) {
-			if self.enableLog {
-				os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
-			}
-		} else {
-			var remainingTags = min(openRange.upperBound - openRange.lowerBound, closeRange.upperBound - closeRange.lowerBound) + 1
-			while remainingTags > 0 {
-				if remainingTags >= self.rule.maxTags {
-					remainingTags -= self.rule.maxTags
-					if let style = self.rule.styles[ self.rule.maxTags ] {
-						if !styles.contains(where: { $0.isEqualTo(style)}) {
-							styles.append(style)
-						}
-					}
-				}
-				if let style = self.rule.styles[remainingTags] {
-					remainingTags -= remainingTags
-					if !styles.contains(where: { $0.isEqualTo(style)}) {
-						styles.append(style)
-					}
-				}
-			}
-			
-			for idx in (openRange.upperBound)...(closeRange.lowerBound) {
-				self.elements[idx].styles.append(contentsOf: styles)
-				self.elements[idx].metadata.append(metadataString)
-				if self.rule.definesBoundary {
-					self.elements[idx].boundaryCount += 1
-				}
-				if self.rule.shouldCancelRemainingRules {
-					self.elements[idx].boundaryCount = 1000
-				}
-			}
-			
-			if self.rule.isRepeatingTag {
-				let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound)
-				switch difference {
-				case 1...:
-					shouldRemove = false
-					self.tagGroups[tagIdx].count = difference
-					self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound )
-				case ...(-1):
-					for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound {
-						self.elements[idx].type = .string
-					}
-				default:
-					break
-				}
-			}
-			
-		}
-		if shouldRemove {
-			self.tagGroups.removeAll(where: { $0.groupID == id })
-		}
-		self.isMetadataOpen = false
-	}
-	
-	func emptyRanges( _ ranges : inout [ClosedRange<Int>] ) {
-		while !ranges.isEmpty {
-			self.resetLastTag(for: &ranges)
-			ranges.removeLast()
-		}
-	}
-	
-	func scanNonRepeatingTags() {
-		var groupID = ""
-		let closeTag = self.rule.tag(for: .close)?.tag
-		let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag
-		let metadataClose = self.rule.tag(for: .metadataClose)?.tag
-		
-		while self.pointer < self.elements.count {
-			if self.enableLog {
-				os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character))
-			}
-			
-			if let range = self.range(for: metadataClose) {
-				if self.isMetadataOpen {
-					guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else {
-						self.pointer += 1
-						continue
-					}
-					
-					guard !self.tagGroups.isEmpty else {
-						self.resetTagGroup(withID: groupID)
-						continue
-					}
-				
-					guard self.isMetadataOpen else {
-						
-						self.resetTagGroup(withID: groupID)
-						continue
-					}
-					if self.enableLog {
-						os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
-					}
-					self.tagGroups[groupIdx].tagRanges.append(range)
-					self.closeTag(closeTag!, withGroupID: groupID)
-					self.isMetadataOpen = false
-					continue
-				} else {
-					self.resetTag(in: range)
-					self.pointer -= metadataClose!.count
-				}
-
-			}
-			
-			if let openRange = self.range(for: self.rule.primaryTag.tag) {
-				if self.isMetadataOpen {
-					self.resetTagGroup(withID: groupID)
-				}
-				
-				let tagGroup = TagGroup(tagRanges: [openRange])
-				groupID = tagGroup.groupID
-				if self.enableLog {
-					os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
-				}
-				if self.rule.isRepeatingTag {
-					
-				}
-				
-				self.tagGroups.append(tagGroup)
-				continue
-			}
-	
-			if let range = self.range(for: closeTag) {
-				guard !self.tagGroups.isEmpty else {
-					if self.enableLog {
-						os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScannerScanner, type:.info)
-					}
-					self.resetTag(in: range)
-					continue
-				}
-				self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
-				groupID = self.tagGroups[self.tagGroups.count - 1].groupID
-				if self.enableLog {
-					os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
-				}
-				guard metadataOpen != nil else {
-					if self.enableLog {
-						os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
-					}
-					self.closeTag(closeTag!, withGroupID: groupID)
-					continue
-				}
-				
-				guard self.pointer != self.elements.count else {
-					continue
-				}
-				
-				guard let range = self.range(for: metadataOpen) else {
-					if self.enableLog {
-						os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
-					}
-					self.resetTagGroup(withID: groupID)
-					continue
-				}
-				self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
-				self.isMetadataOpen = true
-				continue
-			}
-			
-
-			if let range = self.range(for: metadataOpen) {
-				if self.enableLog {
-					os_log("Multiple open metadata tags found!", log: OSLog.swiftyScannerScanner, type:.info , groupID)
-				}
-				self.resetTag(in: range)
-				self.resetTagGroup(withID: groupID)
-				self.isMetadataOpen = false
-				continue
-			}
-			self.pointer += 1
-		}
-	}
-	
-	func scanRepeatingTags() {
-				
-		var groupID = ""
-		let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined()
-		let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters))
-		while self.pointer < self.elements.count {
-			if self.enableLog {
-				os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character))
-			}
-			
-			if var openRange = self.range(for: self.rule.primaryTag.tag) {
-				
-				if self.elements[openRange].first?.boundaryCount == 1000 {
-					self.resetTag(in: openRange)
-					continue
-				}
-				
-				var count = 1
-				var tagType : RepeatingTagType = .open
-				if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1))  {
-					if !unionSet.containsUnicodeScalars(of: prevElement.character) {
-						tagType = .either
-					}
-				} else {
-					tagType = .open
-				}
-				
-				while let nextRange = self.range(for: self.rule.primaryTag.tag)  {
-					count += 1
-					openRange = openRange.lowerBound...nextRange.upperBound
-				}
-				
-				if self.rule.minTags > 1 {
-					if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags {
-						self.resetTag(in: openRange)
-						os_log("Tag does not meet minimum length", log: .swiftyScannerScanner, type: .info)
-						continue
-					}
-				}
-				
-				var validTagGroup = true
-				if let nextElement = self.element(for: .forward(0)) {
-					if unionSet.containsUnicodeScalars(of: nextElement.character) {
-						if tagType == .either {
-							tagType = .close
-						} else {
-							validTagGroup = tagType != .open
-						}
-					}
-				} else {
-					if tagType == .either {
-						tagType = .close
-					} else {
-						validTagGroup = tagType != .open
-					}
-				}
-				
-				if !validTagGroup {
-					if self.enableLog {
-						os_log("Tag has whitespace on both sides", log: .swiftyScannerScanner, type: .info)
-					}
-					self.resetTag(in: openRange)
-					continue
-				}
-				
-				if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) {
-					if tagType == .either {
-						if tagGroups[idx].count == count {
-							self.tagGroups[idx].tagRanges.append(openRange)
-							self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
-							
-							if let last = self.tagGroups.last {
-								groupID = last.groupID
-							}
-							
-							continue
-						}
-					} else {
-						if let prevRange = tagGroups[idx].tagRanges.first {
-							if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount {
-								self.tagGroups[idx].tagRanges.append(openRange)
-								self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
-							}
-						}
-						continue
-					}
-				}
-				var tagGroup = TagGroup(tagRanges: [openRange])
-				groupID = tagGroup.groupID
-				tagGroup.tagType = tagType
-				tagGroup.count = count
-				
-				if self.enableLog {
-					os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner,  type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID)
-				}
-				
-				self.tagGroups.append(tagGroup)
-				continue
-			}
-	
-			self.pointer += 1
-		}
-	}
-	
-	
-	func scan() -> [Element] {
-		
-		guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else {
-			return self.elements
-		}
-		
-		self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)")
-		
-		if self.enableLog {
-			os_log("RULE: %@", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
-		}
-		
-		if self.rule.isRepeatingTag {
-			self.scanRepeatingTags()
-		} else {
-			self.scanNonRepeatingTags()
-		}
-		
-		for tagGroup in self.tagGroups {
-			self.resetTagGroup(withID: tagGroup.groupID)
-		}
-		
-		if self.enableLog {
-			for element in self.elements {
-				print(element)
-			}
-		}
-		return self.elements
-	}
-}

Sources/SwiftyMarkdown/SwiftyTokeniser.swift

@@ -23,7 +23,6 @@ public class SwiftyTokeniser {
 	let totalPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Total Run Time", log: OSLog.performance)
 	let currentPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Current", log: OSLog.performance)
 		
-	var scanner : SwiftyScanning!
 	public var metadataLookup : [String : String] = [:]
 	
 	let newlines = CharacterSet.newlines
@@ -53,7 +52,7 @@ public class SwiftyTokeniser {
 	///
 	/// - Parameter inputString: A string to have the CharacterRules in `self.rules` applied to
 	public func process( _ inputString : String ) -> [Token] {
-		var currentTokens = [Token(type: .string, inputString: inputString)]
+		let currentTokens = [Token(type: .string, inputString: inputString)]
 		guard rules.count > 0 else {
 			return currentTokens
 		}
@@ -83,19 +82,13 @@ public class SwiftyTokeniser {
 		
 		while !mutableRules.isEmpty {
 			let nextRule = mutableRules.removeFirst()
-			
-			if nextRule.isRepeatingTag {
-				self.scanner = SwiftyScanner()
-				self.scanner.metadataLookup = self.metadataLookup
-			}
-
 			if enableLog {
 				os_log("------------------------------", log: .tokenising, type: .info)
 				os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
 			}
 			self.currentPerfomanceLog.tag(with: "(start rule %@)")
 			
-			let scanner = SwiftyScannerNonRepeating(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup)
+			let scanner = SwiftyScanner(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup)
 			elementArray = scanner.scan()
 		}
 		
@@ -113,7 +106,6 @@ public class SwiftyTokeniser {
 			tokens.append(token)
 		}
 		
-		
 		var lastElement = elementArray.first!
 		var accumulatedString = ""
 		for element in elementArray {
@@ -141,372 +133,6 @@ public class SwiftyTokeniser {
 		}
 		return output
 	}
-	
-	
-//
-//
-//	/// In order to reinsert the original replacements into the new string token, the replacements
-//	/// need to be searched for in the incoming string one by one.
-//	///
-//	/// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles
-//	/// are passed over into the newly created tokens.
-//	///
-//	/// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with
-//	/// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens:
-//	///
-//	/// - `string`: "This string "
-//	/// - `replacement`: "AAAAA-BBBBB-CCCCC"
-//	/// - `string`: " replacements"
-//	///
-//	/// Each of these need to have a character style of `bold`.
-//	///
-//	/// - Parameters:
-//	///   - replacements: An array of `replacement` tokens
-//	///   - token: The new `string` token that may contain replacement IDs contained in the `replacements` array
-//	func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] {
-//		guard !stringToken.outputString.isEmpty && !replacements.isEmpty else {
-//			return [stringToken]
-//		}
-//		var outputTokens : [Token] = []
-//		let scanner = Scanner(string: stringToken.outputString)
-//		scanner.charactersToBeSkipped = nil
-//
-//		// Remove any replacements that don't appear in the incoming string
-//		var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) })
-//
-//		var testString = "\n"
-//		while !scanner.isAtEnd {
-//			var outputString : String = ""
-//			if repTokens.count > 0 {
-//				testString = repTokens.removeFirst().inputString
-//			}
-//
-//			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-//				if let nextString = scanner.scanUpToString(testString) {
-//					outputString = nextString
-//					outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
-//					if let outputToken = scanner.scanString(testString) {
-//						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-//					}
-//				} else if let outputToken = scanner.scanString(testString) {
-//					outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-//				}
-//			} else {
-//				var oldString : NSString? = nil
-//				var tokenString : NSString? = nil
-//				scanner.scanUpTo(testString, into: &oldString)
-//				if let nextString = oldString {
-//					outputString = nextString as String
-//					outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
-//					scanner.scanString(testString, into: &tokenString)
-//					if let outputToken = tokenString as String? {
-//						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-//					}
-//				} else {
-//					scanner.scanString(testString, into: &tokenString)
-//					if let outputToken = tokenString as String? {
-//						outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
-//					}
-//				}
-//			}
-//		}
-//		return outputTokens
-//	}
-//
-//
-//	/// This function is necessary because a previously tokenised string might have
-//	///
-//	/// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance.
-//	///
-//	/// The incoming tokens will look like this:
-//	///
-//	/// - `string`: "A \*\*Bold"
-//	/// - `replacement` : "AAAAA-BBBBB-CCCCC"
-//	/// - `string`: " with a trailing string**"
-//	///
-//	///	However, because the scanner can only tokenise individual strings, passing in the string values
-//	///	of these tokens individually and applying the styles will not correctly detect the starting and
-//	///	ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC",
-//	///	and finally " with a trailing string\*\*")
-//	///
-//	///	The strings need to be combined, so that they form a single string:
-//	///	A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*.
-//	///	This string is then parsed and tokenised so that it looks like this:
-//	///
-//	/// - `string`: "A "
-//	///	- `repeatingTag`: "\*\*"
-//	///	- `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string"
-//	///	- `repeatingTag`: "\*\*"
-//	///
-//	///	Finally, the replacements from the original incoming token array are searched for and pulled out
-//	///	of this new string, so the final result looks like this:
-//	///
-//	/// - `string`: "A "
-//	///	- `repeatingTag`: "\*\*"
-//	///	- `string`: "Bold "
-//	///	- `replacement`: "AAAAA-BBBBB-CCCCC"
-//	///	- `string`: " with a trailing string"
-//	///	- `repeatingTag`: "\*\*"
-//	///
-//	/// - Parameters:
-//	///   - tokens: The tokens to be combined, scanned, re-tokenised, and merged
-//	///   - rule: The character rule currently being applied
-//	func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
-//		guard tokens.count > 0 else {
-//			return []
-//		}
-//
-//		let combinedString = tokens.map({ $0.outputString }).joined()
-//
-//		let nextTokens = self.scanner.scan(combinedString, with: rule)
-//		var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
-//
-//		/// It's necessary here to check to see if the first token (which will always represent the styles
-//		/// to be applied from previous scans) has any existing metadata or character styles and apply them
-//		/// to *all* the string and replacement tokens found by the new scan.
-//		for idx in 0..<replacedTokens.count {
-//			guard replacedTokens[idx].type == .string || replacedTokens[idx].type == .replacement else {
-//				continue
-//			}
-//			if tokens.first!.metadataString != nil && replacedTokens[idx].metadataString == nil {
-//				replacedTokens[idx].metadataString = tokens.first!.metadataString
-//			}
-//			replacedTokens[idx].characterStyles.append(contentsOf: tokens.first!.characterStyles)
-//		}
-//
-//		// Swap the original replacement tokens back in
-//		let replacements = tokens.filter({ $0.type == .replacement })
-//		var outputTokens : [Token] = []
-//		for token in replacedTokens {
-//			guard token.type == .string else {
-//				outputTokens.append(token)
-//				continue
-//			}
-//			outputTokens.append(contentsOf: self.reinsertReplacements(replacements, from: token))
-//		}
-//
-//		return outputTokens
-//	}
-//
-//
-//
-//	/// This function ensures that only concurrent `string` and `replacement` tokens are processed together.
-//	///
-//	/// i.e. If there is an existing `repeatingTag` token between two strings, then those strings will be
-//	/// processed individually. This prevents incorrect parsing of strings like "\*\*\_Should only be bold\*\*\_"
-//	///
-//	/// - Parameters:
-//	///   - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
-//	///   - rule: The current rule being processed
-//	func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
-//
-//		// Only combine string and replacements that are next to each other.
-//		var newTokenSet : [Token] = []
-//		var currentTokenSet : [Token] = []
-//		for i in 0..<incomingTokens.count {
-//			guard incomingTokens[i].type == .string || incomingTokens[i].type == .replacement else {
-//				newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-//				newTokenSet.append(incomingTokens[i])
-//				currentTokenSet.removeAll()
-//				continue
-//			}
-//			guard !incomingTokens[i].isProcessed && !incomingTokens[i].isMetadata && !incomingTokens[i].shouldSkip else {
-//				newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-//				newTokenSet.append(incomingTokens[i])
-//				currentTokenSet.removeAll()
-//				continue
-//			}
-//			currentTokenSet.append(incomingTokens[i])
-//		}
-//		newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-//
-//		return newTokenSet
-//	}
-//
-//
-//	func handleClosingTagFromOpenTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule ) {
-//
-//		guard rule.closeTag != nil else {
-//			return
-//		}
-//		guard let closeTokenIdx = tokens.firstIndex(where: { $0.type == .closeTag && !$0.isProcessed }) else {
-//			return
-//		}
-//
-//		var metadataIndex = index
-//		// If there's an intermediate tag, get the index of that
-//		if rule.intermediateTag != nil {
-//			guard let nextTokenIdx = tokens.firstIndex(where: { $0.type == .intermediateTag  && !$0.isProcessed }) else {
-//				return
-//			}
-//			metadataIndex = nextTokenIdx
-//			let styles : [CharacterStyling] = rule.styles[1] ?? []
-//			for i in index..<nextTokenIdx {
-//				for style in styles {
-//					if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
-//						tokens[i].characterStyles.append(style)
-//					}
-//				}
-//			}
-//		}
-//
-//		var metadataString : String = ""
-//		for i in metadataIndex..<closeTokenIdx {
-//			if tokens[i].type == .string {
-//				metadataString.append(tokens[i].outputString)
-//				tokens[i].isMetadata = true
-//			}
-//		}
-//
-//		for i in index..<metadataIndex {
-//			if tokens[i].type == .string {
-//				tokens[i].metadataString = metadataString
-//			}
-//		}
-//
-//		tokens[closeTokenIdx].isProcessed = true
-//		tokens[metadataIndex].isProcessed = true
-//		tokens[index].isProcessed = true
-//	}
-//
-//
-//	/// This is here to manage how opening tags are matched with closing tags when they're all the same
-//	/// character.
-//	///
-//	/// Of course, because Markdown is about as loose as a spec can be while still being considered any
-//	/// kind of spec, the number of times this character repeats causes different effects. Then there
-//	/// is the ill-defined way it should work if the number of opening and closing tags are different.
-//	///
-//	/// - Parameters:
-//	///   - index: The index of the current token in the loop
-//	///   - tokens: An inout variable of the loop tokens of interest
-//	///   - rule: The character rule being applied
-//	func handleClosingTagFromRepeatingTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule) {
-//		let theToken = tokens[index]
-//
-//		if enableLog {
-//			os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
-//		}
-//
-//		guard theToken.count > 0 else {
-//			return
-//		}
-//
-//		let startIdx = index
-//		var endIdx : Int? = nil
-//
-//		let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
-//		// Try to find exact match first
-//		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
-//			endIdx = nextTokenIdx
-//		}
-//
-//		if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id  && !$0.isProcessed }) {
-//			endIdx = nextTokenIdx
-//		}
-//		guard let existentEnd = endIdx else {
-//			return
-//		}
-//
-//
-//		let styles : [CharacterStyling] = rule.styles[maxCount] ?? []
-//		for i in startIdx..<existentEnd {
-//			for style in styles {
-//				if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
-//					tokens[i].characterStyles.append(style)
-//				}
-//			}
-//			if rule.cancels == .allRemaining {
-//				tokens[i].shouldSkip = true
-//			}
-//		}
-//
-//		let maxEnd = (tokens[existentEnd].count > rule.maxTags) ? rule.maxTags : tokens[existentEnd].count
-//		tokens[index].count = theToken.count - maxEnd
-//		tokens[existentEnd].count = tokens[existentEnd].count - maxEnd
-//		if maxEnd < rule.maxTags {
-//			self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule)
-//		} else {
-//			tokens[existentEnd].isProcessed = true
-//			tokens[index].isProcessed = true
-//		}
-//
-//
-//	}
-//
-//	func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] {
-//		var mutableTokens : [Token] = tokens
-//
-//		if enableLog {
-//			os_log("Applying styles to tokens: %@", log: .tokenising, type: .info,  tokens.oslogDisplay )
-//		}
-//		for idx in 0..<mutableTokens.count {
-//			let token = mutableTokens[idx]
-//			switch token.type {
-//			case .escape:
-//				if enableLog {
-//					os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
-//				}
-//			case .repeatingTag:
-//				let theToken = mutableTokens[idx]
-//				self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
-//				if enableLog {
-//					os_log("Found repeating tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
-//				}
-//			case .openTag:
-//				let theToken = mutableTokens[idx]
-//				if enableLog {
-//					os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
-//				}
-//
-//				guard rule.closingTag != nil else {
-//
-//					// If there's an intermediate tag, get the index of that
-//
-//					// Get the index of the closing tag
-//
-//					continue
-//				}
-//				self.handleClosingTagFromOpenTag(withIndex: idx, in: &mutableTokens, following: rule)
-//
-//
-//			case .intermediateTag:
-//				let theToken = mutableTokens[idx]
-//				if enableLog {
-//					os_log("Found intermediate tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
-//				}
-//
-//			case .closeTag:
-//				let theToken = mutableTokens[idx]
-//				if enableLog {
-//					os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
-//				}
-//
-//			case .string:
-//				let theToken = mutableTokens[idx]
-//				if enableLog {
-//					if theToken.isMetadata {
-//						os_log("Found Metadata: %@", log: .tokenising, type: .info, theToken.inputString )
-//					} else {
-//						os_log("Found String: %@", log: .tokenising, type: .info, theToken.inputString )
-//					}
-//					if let hasMetadata = theToken.metadataString {
-//						os_log("...with metadata: %@", log: .tokenising, type: .info, hasMetadata )
-//					}
-//				}
-//
-//			case .replacement:
-//				if enableLog {
-//					os_log("Found replacement with ID: %@", log: .tokenising, type: .info, mutableTokens[idx].inputString )
-//				}
-//			}
-//		}
-//		return mutableTokens
-//	}
-//
-//
-//
-	
 }