[django mode] Rewrite and improve

 - Add string recognition
 - Include (most probably) all built-in template tags
 - Return null as fallback
 - Operator highlighting
 - Change demo theme to `mdn-like` which sports richer highlighting.
 - Highlight comments
 - Define `blockCommentStart` and `blockCommentEnd`

mode/django/django.js

@@ -14,46 +14,329 @@
   "use strict";
 
   CodeMirror.defineMode("django:inner", function() {
-    var keywords = ["block", "endblock", "for", "endfor", "in", "true", "false",
-                    "loop", "none", "self", "super", "if", "endif", "as", "not", "and",
+    var keywords = ["block", "endblock", "for", "endfor", "true", "false",
+                    "loop", "none", "self", "super", "if", "endif", "as",
                     "else", "import", "with", "endwith", "without", "context", "ifequal", "endifequal",
-                    "ifnotequal", "endifnotequal", "extends", "include", "load", "length", "comment",
-                    "endcomment", "empty"];
-    keywords = new RegExp("^((" + keywords.join(")|(") + "))\\b");
+                    "ifnotequal", "endifnotequal", "extends", "include", "load", "comment",
+                    "endcomment", "empty", "url", "static", "trans", "blocktrans", "now", "regroup",
+                    "lorem", "ifchanged", "endifchanged", "firstof", "debug", "cycle", "csrf_token",
+                    "autoescape", "endautoescape", "spaceless", "ssi", "templatetag",
+                    "verbatim", "endverbatim", "widthratio"],
+        filters = ["add", "addslashes", "capfirst", "center", "cut", "date",
+                   "default", "default_if_none", "dictsort",
+                   "dictsortreversed", "divisibleby", "escape", "escapejs",
+                   "filesizeformat", "first", "floatformat", "force_escape",
+                   "get_digit", "iriencode", "join", "last", "length",
+                   "length_is", "linebreaks", "linebreaksbr", "linenumbers",
+                   "ljust", "lower", "make_list", "phone2numeric", "pluralize",
+                   "pprint", "random", "removetags", "rjust", "safe",
+                   "safeseq", "slice", "slugify", "stringformat", "striptags",
+                   "time", "timesince", "timeuntil", "title", "truncatechars",
+                   "truncatechars_html", "truncatewords", "truncatewords_html",
+                   "unordered_list", "upper", "urlencode", "urlize",
+                   "urlizetrunc", "wordcount", "wordwrap", "yesno"],
+        operators = ["==", "!=", "<", ">", "<=", ">=", "in", "not", "or", "and"];
 
+    keywords = new RegExp("^\\b(" + keywords.join("|") + ")\\b");
+    filters = new RegExp("^\\b(" + filters.join("|") + ")\\b");
+    operators = new RegExp("^\\b(" + operators.join("|") + ")\\b");
+
+    // We have to return "null" instead of null, in order to avoid string
+    // styling as the default, when using Django templates inside HTML
+    // element attributes
     function tokenBase (stream, state) {
-      stream.eatWhile(/[^\{]/);
-      var ch = stream.next();
-      if (ch == "{") {
-        if (ch = stream.eat(/\{|%|#/)) {
-          state.tokenize = inTag(ch);
-          return "tag";
+      // Attempt to identify a variable, template or comment tag respectively
+      if (stream.match("{{")) {
+        state.tokenize = inVariable;
+        return "tag";
+      } else if (stream.match("{%")) {
+        state.tokenize = inTag;
+        return "tag";
+      } else if (stream.match("{#")) {
+        state.tokenize = inComment;
+        return "comment";
+      }
+
+      // Ignore completely any stream series that do not match the
+      // Django template opening tags.
+      while (stream.next() != null && !stream.match("{{", false) && !stream.match("{%", false)) {}
+      return null;
+    }
+
+    // A string can be included in either single or double quotes (this is
+    // the delimeter). Mark everything as a string until the start delimeter
+    // occurs again.
+    function inString (delimeter, previousTokenizer) {
+      return function (stream, state) {
+        if (!state.escapeNext && stream.eat(delimeter)) {
+          state.tokenize = previousTokenizer;
+        } else {
+          if (state.escapeNext) {
+            state.escapeNext = false;
+          }
+
+          var ch = stream.next();
+
+          // Take into account the backslash for escaping characters, such as
+          // the string delimeter.
+          if (ch == "\\") {
+            state.escapeNext = true;
+          }
+        }
+
+        return "string";
+      };
+    }
+
+    // Apply Django template variable syntax highlighting
+    function inVariable (stream, state) {
+      // Attempt to match a dot that precedes a property
+      if (state.waitDot) {
+        state.waitDot = false;
+
+        if (stream.peek() != ".") {
+          return "null";
+        }
+
+        // Dot folowed by a non-word character should be considered an error.
+        if (stream.match(/\.\W+/)) {
+          return "error";
+        } else if (stream.eat(".")) {
+          state.waitProperty = true;
+          return "null";
+        } else {
+          throw Error ("Unexpected error while waiting for property.");
+        }
+      }
+
+      // Attempt to match a pipe that precedes a filter
+      if (state.waitPipe) {
+        state.waitPipe = false;
+
+        if (stream.peek() != "|") {
+          return "null";
+        }
+
+        // Pipe folowed by a non-word character should be considered an error.
+        if (stream.match(/\.\W+/)) {
+          return "error";
+        } else if (stream.eat("|")) {
+          state.waitFilter = true;
+          return "null";
+        } else {
+          throw Error ("Unexpected error while waiting for filter.");
+        }
+      }
+
+      // Highlight properties
+      if (state.waitProperty) {
+        state.waitProperty = false;
+        if (stream.match(/\b(\w+)\b/)) {
+          state.waitDot = true;  // A property can be followed by another property
+          state.waitPipe = true;  // A property can be followed by a filter
+          return "property";
         }
       }
+
+      // Highlight filters
+      if (state.waitFilter) {
+          state.waitFilter = false;
+        if (stream.match(filters)) {
+          return "variable-2";
+        }
+      }
+
+      // Ignore all white spaces
+      if (stream.eatSpace()) {
+        state.waitProperty = false;
+        return "null";
+      }
+
+      // Identify numbers
+      if (stream.match(/\b\d+(\.\d+)?\b/)) {
+        return "number";
+      }
+
+      // Identify strings
+      if (stream.match("'")) {
+        state.tokenize = inString("'", state.tokenize);
+        return "string";
+      } else if (stream.match('"')) {
+        state.tokenize = inString('"', state.tokenize);
+        return "string";
+      }
+
+      // Attempt to find the variable
+      if (stream.match(/\b(\w+)\b/) && !state.foundVariable) {
+        state.waitDot = true;
+        state.waitPipe = true;  // A property can be followed by a filter
+        return "variable";
+      }
+
+      // If found closing tag reset
+      if (stream.match("}}")) {
+        state.waitProperty = null;
+        state.waitFilter = null;
+        state.waitDot = null;
+        state.waitPipe = null;
+        state.tokenize = tokenBase;
+        return "tag";
+      }
+
+      // If nothing was found, advance to the next character
+      stream.next();
+      return "null";
     }
-    function inTag (close) {
-      if (close == "{") {
-        close = "}";
+
+    function inTag (stream, state) {
+      // Attempt to match a dot that precedes a property
+      if (state.waitDot) {
+        state.waitDot = false;
+
+        if (stream.peek() != ".") {
+          return "null";
+        }
+
+        // Dot folowed by a non-word character should be considered an error.
+        if (stream.match(/\.\W+/)) {
+          return "error";
+        } else if (stream.eat(".")) {
+          state.waitProperty = true;
+          return "null";
+        } else {
+          throw Error ("Unexpected error while waiting for property.");
+        }
       }
-      return function (stream, state) {
-        var ch = stream.next();
-        if ((ch == close) && stream.eat("}")) {
-          state.tokenize = tokenBase;
-          return "tag";
+
+      // Attempt to match a pipe that precedes a filter
+      if (state.waitPipe) {
+        state.waitPipe = false;
+
+        if (stream.peek() != "|") {
+          return "null";
         }
-        if (stream.match(keywords)) {
-          return "keyword";
+
+        // Pipe folowed by a non-word character should be considered an error.
+        if (stream.match(/\.\W+/)) {
+          return "error";
+        } else if (stream.eat("|")) {
+          state.waitFilter = true;
+          return "null";
+        } else {
+          throw Error ("Unexpected error while waiting for filter.");
         }
-        return close == "#" ? "comment" : "string";
-      };
+      }
+
+      // Highlight properties
+      if (state.waitProperty) {
+        state.waitProperty = false;
+        if (stream.match(/\b(\w+)\b/)) {
+          state.waitDot = true;  // A property can be followed by another property
+          state.waitPipe = true;  // A property can be followed by a filter
+          return "property";
+        }
+      }
+
+      // Highlight filters
+      if (state.waitFilter) {
+          state.waitFilter = false;
+        if (stream.match(filters)) {
+          return "variable-2";
+        }
+      }
+
+      // Ignore all white spaces
+      if (stream.eatSpace()) {
+        state.waitProperty = false;
+        return "null";
+      }
+
+      // Identify numbers
+      if (stream.match(/\b\d+(\.\d+)?\b/)) {
+        return "number";
+      }
+
+      // Identify strings
+      if (stream.match("'")) {
+        state.tokenize = inString("'", state.tokenize);
+        return "string";
+      } else if (stream.match('"')) {
+        state.tokenize = inString('"', state.tokenize);
+        return "string";
+      }
+
+      // Attempt to match an operator
+      if (stream.match(operators)) {
+        return "operator";
+      }
+
+      // Attempt to match a keyword
+      var keywordMatch = stream.match(keywords);
+      if (keywordMatch) {
+        if (keywordMatch[0] == "comment") {
+          state.blockCommentTag = true;
+        }
+        return "keyword";
+      }
+
+      // Attempt to match a variable
+      if (stream.match(/\b(\w+)\b/)) {
+        state.waitDot = true;
+        state.waitPipe = true;  // A property can be followed by a filter
+        return "variable";
+      }
+
+      // If found closing tag reset
+      if (stream.match("%}")) {
+        state.waitProperty = null;
+        state.waitFilter = null;
+        state.waitDot = null;
+        state.waitPipe = null;
+        // If the tag that closes is a block comment tag, we want to mark the
+        // following code as comment, until the tag closes.
+        if (state.blockCommentTag) {
+          state.blockCommentTag = false;  // Release the "lock"
+          state.tokenize = inBlockComment;
+        } else {
+          state.tokenize = tokenBase;
+        }
+        return "tag";
+      }
+
+      // If nothing was found, advance to the next character
+      stream.next();
+      return "null";
+    }
+
+    // Mark everything as comment inside the tag and the tag itself.
+    function inComment (stream, state) {
+      if (stream.match("#}")) {
+        state.tokenize = tokenBase;
+      }
+      return "comment";
+    }
+
+    // Mark everything as a comment until the `blockcomment` tag closes.
+    function inBlockComment (stream, state) {
+      if (stream.match(/\{%\s*endcomment\s*%\}/, false)) {
+        state.tokenize = inTag;
+        stream.match("{%");
+        return "tag";
+      } else {
+        stream.next();
+        return "comment";
+      }
     }
+
     return {
       startState: function () {
         return {tokenize: tokenBase};
       },
       token: function (stream, state) {
         return state.tokenize(stream, state);
-      }
+      },
+      blockCommentStart: "{% comment %}",
+      blockCommentEnd: "{% endcomment %}"
     };
   });
 

mode/django/index.html

@@ -5,6 +5,7 @@
 <link rel=stylesheet href="../../doc/docs.css">
 
 <link rel="stylesheet" href="../../lib/codemirror.css">
+<link rel="stylesheet" href="../../theme/mdn-like.css">
 <script src="../../lib/codemirror.js"></script>
 <script src="../../addon/mode/overlay.js"></script>
 <script src="../xml/xml.js"></script>
@@ -30,21 +31,29 @@
 <form><textarea id="code" name="code">
 <!doctype html>
 <html>
-    <head>
-        <title>My Django web application</title>
-    </head>
-    <body>
-        <h1>
-            {{ page.title }}
-        </h1>
-        <ul class="my-list">
-            {% for item in items %}
-                <li>{% item.name %}</li>
-            {% empty %}
-                <li>You have no items in your list.</li>
-            {% endfor %}
-        </ul>
-    </body>
+  <head>
+    <title>My Django web application</title>
+  </head>
+  <body>
+    <h1>
+      {{ page.title|capfirst }}
+    </h1>
+    <ul class="my-list">
+      {# traverse a list of items and produce links to their views. #}
+      {% for item in items %}
+      <li>
+        <a href="{% url 'item_view' item.name|slugify %}">
+          {{ item.name }}
+        </a>
+      </li>
+      {% empty %}
+      <li>You have no items in your list.</li>
+      {% endfor %}
+    </ul>
+    {% comment "this is a forgotten footer" %}
+    <footer></footer>
+    {% endcomment %}
+  </body>
 </html>
 </textarea></form>
 
@@ -52,8 +61,9 @@
       var editor = CodeMirror.fromTextArea(document.getElementById("code"), {
         lineNumbers: true,
         mode: "django",
-        indentUnit: 4,
-        indentWithTabs: true
+        indentUnit: 2,
+        indentWithTabs: true,
+        theme: "mdn-like"
       });
     </script>