update docs

docs/USING_PRO.md

@@ -4,7 +4,7 @@ To champion the single-responsibility and open/closed principles, we have tried
 
 <h2 id="renderer">The renderer</h2>
 
-The renderer is...
+The renderer defines the output of the parser.
 
 **Example:** Overriding default heading token by adding an embedded anchor tag like on GitHub.
 
@@ -91,29 +91,36 @@ slugger.slug('foo-1') // foo-1-2
 
 <h2 id="lexer">The lexer</h2>
 
-The lexer is...
+The lexer turns a markdown string into block level tokens.
 
+<h2 id="inlinelexer">The inline lexer</h2>
+
+The inline lexer adds inline tokens to the block level tokens.
 
 <h2 id="parser">The parser</h2>
 
-The parser is...
+The parser takes tokens as input and calls the renderer functions that are accosiated with those tokens.
 
 ***
 
 <h2 id="extend">Access to lexer and parser</h2>
 
-You also have direct access to the lexer and parser if you so desire.
+You also have direct access to the lexer, inline lexer, and parser if you so desire.
 
 ``` js
-const tokens = marked.lexer(text, options);
+const blocks = marked.lexer(markdown, options);
+const tokens = marked.inlineLexer(blocks, options);
 console.log(marked.parser(tokens, options));
 ```
 
 ``` js
 const lexer = new marked.Lexer(options);
-const tokens = lexer.lex(text);
+const inlineLexer = new marked.InlineLexer(options);
+const blocks = lexer.lex(markdown);
+const tokens = inlineLexer.lex(blocks);
 console.log(tokens);
-console.log(lexer.rules);
+console.log(lexer.rules); // block level rules
+console.log(inlineLexer.rules); // inline level rules
 ```
 
 ``` bash
@@ -126,8 +133,8 @@ $ node
   links: {} ]
 ```
 
-The Lexers build an array of tokens, which will be passed to their respective
-Parsers. The Parsers process each token in the token arrays,
+The Lexer and InlineLexer build an array of tokens, which will be passed to the Parser.
+The Parser processes each token in the token array,
 which are removed from the array of tokens:
 
 ``` js
@@ -141,7 +148,8 @@ const md = `
   [1]: #heading "heading"
 `;
 
-const tokens = marked.lexer(md);
+const blocks = marked.lexer(md);
+const tokens = marked.inlineLexer(blocks);
 console.log(tokens);
 
 const html = marked.parser(tokens);

docs/demo/demo.js

@@ -183,7 +183,7 @@ function handleIframeLoad() {
 
 function handleInput() {
   inputDirty = true;
-};
+}
 
 function handleVersionChange() {
   if ($markedVerElem.value === 'commit' || $markedVerElem.value === 'pr') {
@@ -256,7 +256,7 @@ function handleChange(panes, visiblePane) {
     }
   }
   return active;
-};
+}
 
 function addCommitVersion(value, text, commit) {
   if (markedVersions[value]) {
@@ -331,13 +331,13 @@ function jsonString(input) {
     .replace(/[\\"']/g, '\\$&')
     .replace(/\u0000/g, '\\0');
   return '"' + output + '"';
-};
+}
 
 function getScrollSize() {
   var e = $activeOutputElem;
 
   return e.scrollHeight - e.clientHeight;
-};
+}
 
 function getScrollPercent() {
   var size = getScrollSize();
@@ -347,11 +347,11 @@ function getScrollPercent() {
   }
 
   return $activeOutputElem.scrollTop / size;
-};
+}
 
 function setScrollPercent(percent) {
   $activeOutputElem.scrollTop = percent * getScrollSize();
-};
+}
 
 function updateLink() {
   var outputType = '';
@@ -446,7 +446,7 @@ function checkForChanges() {
     }
   }
   checkChangeTimeout = window.setTimeout(checkForChanges, delayTime);
-};
+}
 
 function setResponseTime(ms) {
   var amount = ms;

docs/demo/worker.js

@@ -1,4 +1,5 @@
 /* globals marked, unfetch, ES6Promise, Promise */ // eslint-disable-line no-redeclare
+
 if (!self.Promise) {
   self.importScripts('https://cdn.jsdelivr.net/npm/es6-promise/dist/es6-promise.js');
   self.Promise = ES6Promise;
@@ -48,28 +49,44 @@ function parse(e) {
     case 'parse':
       var startTime = new Date();
       var lexed = marked.lexer(e.data.markdown, e.data.options);
-      var lexedList = [];
-      for (var i = 0; i < lexed.length; i++) {
-        var lexedLine = [];
-        for (var j in lexed[i]) {
-          lexedLine.push(j + ':' + jsonString(lexed[i][j]));
-        }
-        lexedList.push('{' + lexedLine.join(', ') + '}');
-      }
+      var lexedList = getLexedList(lexed);
       var parsed = marked.parser(lexed, e.data.options);
       var endTime = new Date();
-      // setTimeout(function () {
       postMessage({
         task: e.data.task,
-        lexed: lexedList.join('\n'),
+        lexed: lexedList,
         parsed: parsed,
         time: endTime - startTime
       });
-      // }, 10000);
       break;
   }
 }
 
+function getLexedList(lexed, level) {
+  level = level || 0;
+  var lexedList = [];
+  for (var i = 0; i < lexed.length; i++) {
+    var lexedLine = [];
+    for (var j in lexed[i]) {
+      if (j === 'tokens') {
+        lexedLine.push(j + ': [\n' + getLexedList(lexed[i][j], level + 1) + '\n]');
+      } else {
+        lexedLine.push(j + ':' + jsonString(lexed[i][j]));
+      }
+    }
+    lexedList.push(stringRepeat(' ', 2 * level) + '{' + lexedLine.join(', ') + '}');
+  }
+  return lexedList.join('\n');
+}
+
+function stringRepeat(char, times) {
+  var s = '';
+  for (var i = 0; i < times; i++) {
+    s += char;
+  }
+  return s;
+}
+
 function jsonString(input) {
   var output = (input + '')
     .replace(/\n/g, '\\n')
@@ -79,7 +96,7 @@ function jsonString(input) {
     .replace(/[\\"']/g, '\\$&')
     .replace(/\u0000/g, '\\0');
   return '"' + output + '"';
-};
+}
 
 function loadVersion(ver) {
   var promise;