- added "preprocessor" argument to Template, TemplateLookup - is a single

  callable or list of callables which will be applied to the template text
  before lexing.  given the text as an argument, returns the new text.
- added mako.ext.preprocessors package, contains one preprocessor so far:
  'convert_comments', which will convert single # comments to the new ##
  format

Author: zzzeek

CHANGES

@@ -9,7 +9,13 @@ either one "#" sign or two for now; two is preferred going forward, i.e.
 - UNDEFINED evaluates to False
 - improvement to scoping of "caller" variable when using <%call> tag
 - added lexer error for unclosed control-line (%) line
-
+- added "preprocessor" argument to Template, TemplateLookup - is a single
+  callable or list of callables which will be applied to the template text
+  before lexing.  given the text as an argument, returns the new text.
+- added mako.ext.preprocessors package, contains one preprocessor so far:
+  'convert_comments', which will convert single # comments to the new ##
+  format
+  
 0.1.2
 - fix to parsing of code/expression blocks to insure that non-ascii
   characters, combined with a template that indicates a non-standard

lib/mako/ext/preprocessors.py

@@ -0,0 +1,20 @@
+"""preprocessing functions, used with the 'preprocessor' argument on Template, TemplateLookup"""
+
+import re
+
+def convert_comments(text):
+    """preprocess old style comments.
+    
+    example:
+    
+    from mako.ext.preprocessors import convert_comments
+    t = Template(..., preprocessor=preprocess_comments)"""
+    return re.sub(r'(?<=\n)\s*#[^#]', "##", text)
+
+# TODO
+def create_tag(callable):
+    """given a callable, extract the *args and **kwargs, and produce a preprocessor
+    that will parse for <%<funcname> <args>> and convert to an appropriate <%call> statement.
+    
+    this allows any custom tag to be created which looks like a pure Mako-style tag."""
+    raise NotImplementedError("Future functionality....")

lib/mako/lexer.py

@@ -11,7 +11,7 @@
 from mako.pygen import adjust_whitespace
 
 class Lexer(object):
-    def __init__(self, text, filename=None, input_encoding=None):
+    def __init__(self, text, filename=None, input_encoding=None, preprocessor=None):
         self.text = text
         self.filename = filename
         self.template = parsetree.TemplateNode(self.filename)
@@ -22,6 +22,12 @@ def __init__(self, text, filename=None, input_encoding=None):
         self.tag = []
         self.control_line = []
         self.encoding = input_encoding
+        if preprocessor is None:
+            self.preprocessor = []
+        elif not hasattr(preprocessor, '__iter__'):
+            self.preprocessor = [preprocessor]
+        else:
+            self.preprocessor = preprocessor
 
     def match(self, regexp, flags=None):
         """match the given regular expression string and flags to the current text position.
@@ -95,6 +101,8 @@ def escape_code(self, text):
             return text
 
     def parse(self):
+        for preproc in self.preprocessor:
+            self.text = preproc(self.text)
         parsed_encoding = self.match_encoding()
         if parsed_encoding:
             self.encoding = parsed_encoding

lib/mako/lookup.py

@@ -37,15 +37,15 @@ def adjust_uri(self, uri, filename):
         return uri
 
 class TemplateLookup(TemplateCollection):
-    def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None, cache_type=None, cache_dir=None, modulename_callable=None, default_filters=['unicode'], imports=None, input_encoding=None):
+    def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None, cache_type=None, cache_dir=None, modulename_callable=None, default_filters=['unicode'], imports=None, input_encoding=None, preprocessor=None):
         if isinstance(directories, basestring):
             directories = [directories]        
         self.directories = [posixpath.normpath(d) for d in directories or []]
         self.module_directory = module_directory
         self.modulename_callable = modulename_callable
         self.filesystem_checks = filesystem_checks
         self.collection_size = collection_size
-        self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'default_filters':default_filters, 'imports':imports}
+        self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'default_filters':default_filters, 'imports':imports, 'preprocessor':preprocessor}
         if collection_size == -1:
             self.__collection = {}
             self._uri_cache = {}

lib/mako/template.py

@@ -16,7 +16,7 @@
 
 class Template(object):
     """a compiled template"""
-    def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, module_directory=None, cache_type=None, cache_dir=None, module_filename=None, input_encoding=None, default_filters=['unicode'], imports=None):
+    def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, module_directory=None, cache_type=None, cache_dir=None, module_filename=None, input_encoding=None, default_filters=['unicode'], imports=None, preprocessor=None):
         """construct a new Template instance using either literal template text, or a previously loaded template module
         
         text - textual template source, or None if a module is to be provided
@@ -41,6 +41,7 @@ def __init__(self, text=None, filename=None, uri=None, format_exceptions=False,
         self.default_filters = default_filters
         self.input_encoding = input_encoding
         self.imports = imports
+        self.preprocessor = preprocessor
 
         # if plain text, compile code in memory only
         if text is not None:
@@ -171,7 +172,7 @@ def _get_source(self):
 
 def _compile_text(template, text, filename):
     identifier = template.module_id
-    node = Lexer(text, filename, input_encoding=template.input_encoding).parse()
+    node = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor).parse()
     source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, imports=template.imports)
     cid = identifier
     module = imp.new_module(cid)
@@ -182,7 +183,7 @@ def _compile_text(template, text, filename):
 def _compile_module_file(template, text, filename, outputpath):
     identifier = template.module_id
     (dest, name) = tempfile.mkstemp()
-    node = Lexer(text, filename, input_encoding=template.input_encoding).parse()
+    node = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor).parse()
     source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, imports=template.imports)
     os.write(dest, source)
     os.close(dest)

test/lexer.py

@@ -4,6 +4,7 @@
 from mako import exceptions
 from util import flatten_result, result_lines
 from mako.template import Template
+import re
 
 class LexerTest(unittest.TestCase):
     def test_text_and_tag(self):
@@ -376,6 +377,17 @@ def test_comments(self):
 """
         nodes = Lexer(template).parse()
         assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n<style>\n #someselector\n # other non comment stuff\n</style>\n', (1, 1)), Comment(u'a comment', (6, 1)), Text(u'\n# also not a comment\n\n', (7, 1)), Comment(u'this is a comment', (10, 1)), Text(u'   \nthis is ## not a comment\n\n', (11, 1)), Comment(u' multiline\ncomment\n', (14, 1)), Text(u'\n\nhi\n', (16, 8))])"""
+
+    def test_preprocess(self):
+        def preproc(text):
+            return re.sub(r'(?<=\n)\s*#[^#]', "##", text)
+        template = """
+    hi
+    # old style comment
+# another comment
+"""
+        nodes = Lexer(template, preprocessor=preproc).parse()
+        assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n    hi\n', (1, 1)), Comment(u'old style comment', (3, 1)), Comment(u'another comment', (4, 1))])"""
 
 if __name__ == '__main__':
     unittest.main()

test/template.py

@@ -2,6 +2,7 @@
 
 from mako.template import Template
 from mako.lookup import TemplateLookup
+from mako.ext.preprocessors import convert_comments
 import unittest, re, os
 from util import flatten_result, result_lines
 
@@ -196,7 +197,20 @@ def get_modname(filename, uri):
         t2 = lookup.get_template('/subdir/modtest.html')
         assert t.module.__file__ == 'test_htdocs/foo/modtest.html.py'
         assert t2.module.__file__ == 'test_htdocs/subdir/foo/modtest.html.py'
-        
+
+class PreprocessTest(unittest.TestCase):
+    def test_old_comments(self):
+        t = Template("""
+        im a template
+# old style comment
+    # more old style comment
+    
+    ## new style comment
+    - # not a comment
+    - ## not a comment
+""", preprocessor=convert_comments)
+
+        assert flatten_result(t.render()) == "im a template - # not a comment - ## not a comment"
 
 if __name__ == '__main__':
     unittest.main()