Match pygments input-line-counts with output-line-counts and test that this works

assets/sentry-sdk-issues/README.md

@@ -0,0 +1 @@
+Issues with the sentry-sdk that I've not reported (yet)

assets/sentry-sdk-issues/django-templates.md

@@ -0,0 +1,12 @@
+When running with `DEBUG=True`, the code context for TemplateSyntaxError is mangled in 2 ways:
+
+* There are trailing \n in the lines
+* The code is doubly escaped
+
+This is because:
+
+* In Django, `./django/template/base.py`, in `get_exception_info()`, the lines are escaped.
+    This is possibly an error? I've removed that code and it renders just fine?
+
+* In the Sentry SDK, these values are simply copied:
+    See `./sentry_sdk/integrations/django/templates.py`,  `if hasattr(exc_value, "template_debug")`

theme/templatetags/issues.py

@@ -19,16 +19,44 @@ def _split(joined, lengths):
     return result
 
 
+def _core_pygments(code):
+    # PythonLexer(stripnl=False) does not actually work; we work around it by inserting a space in the empty lines
+    # before calling this function.
+    result = highlight(code, PythonLexer(), HtmlFormatter(nowrap=True))
+
+    # I can't actually get the assertion below to work stably on the level of _core_pygments(code), so it is commented
+    # out. This is because at the present level we have to deal with both pygments' funnyness, and the fact that "what
+    # a line is" is not properly defined. (i.e.: is the thing after the final newline a line or not, both for the input
+    # and the output?). At the level of _pygmentize_lines the idea of a line is properly defined, so we only have to
+    # deal with pygments' funnyness.
+    # assert len(code.split("\n")) == result.count("\n"), "%s != %s" % (len(code.split("\n")), result.count("\n"))
+
+    return result
+
+
+def _pygmentize_lines(lines):
+    if lines == []:
+        # special case; sending the empty string to pygments will result in one newline too many
+        return []
+
+    # newlines should by definition not be part of the code given the fact that it is presented to us as a list of
+    # lines. However, we have seen cases where newlines are present in the code, e.g. in the case of the sentry_sdk's
+    # integration w/ Django giving a TemplateSyntaxError (see assets/sentry-sdk-issues/django-templates.md).
+    # we also add a space to the empty lines to make sure that they are not removed by the pygments formatter
+    lines = [" " if line == "" else line.replace("\n", "") for line in lines]
+    code = "\n".join(lines)
+    result = _core_pygments(code).split('\n')[:-1]  # remove the last empty line, which is a result of split()
+    assert len(lines) == len(result), "%s != %s" % (len(lines), len(result))
+    return result
+
+
 @register.filter
 def pygmentize(value):
-    # first, get the actual code from the frame
     lengths = [len(value['pre_context']), 1, len(value['post_context'])]
 
-    code = "\n".join(value['pre_context'] + [value['context_line']] + value['post_context'])
-    pygments_result = highlight(code, PythonLexer(stripnl=False), HtmlFormatter(nowrap=True))
-    lines = pygments_result.split('\n')[:-1]  # remove the last empty line, which is a result of split()
+    code_as_list = value['pre_context'] + [value['context_line']] + value['post_context']
 
-    assert len(lines) == sum(lengths), "%d != %d" % (len(lines), sum(lengths))
+    lines = _pygmentize_lines(code_as_list)
 
     pre_context, context_lines, post_context = _split(lines, lengths)
 

theme/tests.py

@@ -0,0 +1,51 @@
+from unittest import TestCase
+
+from .templatetags.issues import _pygmentize_lines
+
+
+class TestIssuesTemplateTags(TestCase):
+    # These tests depend on the assert inside the function, simply calling the function and the assert not blowing up is
+    # what we're proving here.
+
+    def test_pygmentize_lines_empty(self):
+        _pygmentize_lines([])
+
+    def test_pygmentize_lines_single_empty_line(self):
+        _pygmentize_lines([""])
+
+    def test_pygmentize_lines_single_space(self):
+        _pygmentize_lines([" "])
+
+    def test_pygmentize_lines_single_line(self):
+        _pygmentize_lines(["print('hello world')"])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_0_1(self):
+        _pygmentize_lines(["print('hello world')", ""])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_0_2(self):
+        _pygmentize_lines(["print('hello world')", "", ""])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_2_0(self):
+        _pygmentize_lines(["", "", "print('hello world')"])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_1_1(self):
+        _pygmentize_lines(["", "print('hello world')", ""])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_2_1(self):
+        _pygmentize_lines(["", "", "print('hello world')", ""])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_1_2(self):
+        _pygmentize_lines(["", "print('hello world')", "", ""])
+
+    def test_pygmentize_lines_leading_and_trailing_emptyness_2_2(self):
+        _pygmentize_lines(["", "", "print('hello world')", "", ""])
+
+    def test_pygmentize_lines_newlines_in_the_middle(self):
+        _pygmentize_lines(["print('hello world')", "", "", "print('goodbye')"])
+
+    def test_pygmentize_lines_non_python(self):
+        # not actually python
+        _pygmentize_lines(["<?= 'hello world' ?>"])
+
+    def test_pygmentize_lines_newline_in_code(self):
+        _pygmentize_lines(["print('hello world')\n"])