Update for 0.9.2 release

SQL queries containing non-ascii byte strings would cause errors, both with and
without Pygments highlighting.

This updates the non-Pygments case to handle a simple decoding to ensure the
value is ascii-safe. It also removes passing an explicit "utf-8" encoding to
Pygments, since this causes errors when the bytes are not utf-8. When the
encoding is omitted, Pygments will default to "guess" the encoding by trying
utf-8 and falling back to latin-1.

Fixes #55

CHANGES.rst

@@ -1,6 +1,15 @@
 Changes
 =======
 
+0.9.2 (2014-12-05)
+------------------
+
+Fixes:
+
+- HTML escape SQL queries when syntax highlighting is not available
+- Use case-insensitive comparison to normalize filenames on Windows
+- Fix exception when SQL query contained non-ASCII characters
+
 0.9.1 (2014-11-24)
 ------------------
 

docs/conf.py

@@ -50,7 +50,7 @@ copyright = u'2012, Matt Good'
 # The short X.Y version.
 version = '0.9'
 # The full version, including alpha/beta/rc tags.
-release = '0.9.1'
+release = '0.9.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

flask_debugtoolbar/__init__.py

@@ -1,15 +1,14 @@
 import os
 
-from flask import current_app, request, g
+from flask import Blueprint, current_app, request, g, send_from_directory
 from flask.globals import _request_ctx_stack
-from flask import send_from_directory
 from jinja2 import Environment, PackageLoader
 from werkzeug.exceptions import HTTPException
 from werkzeug.urls import url_quote_plus
 
-from flask_debugtoolbar.toolbar import DebugToolbar
 from flask_debugtoolbar.compat import iteritems
-from flask import Blueprint
+from flask_debugtoolbar.toolbar import DebugToolbar
+from flask_debugtoolbar.utils import decode_text
 
 
 module = Blueprint('debugtoolbar', __name__)
@@ -30,10 +29,7 @@ def replace_insensitive(string, target, replacement):
 
 def _printable(value):
     try:
-        value = repr(value)
-        if isinstance(value, bytes):
-            value = value.decode('ascii', 'replace')
-        return value
+        return decode_text(repr(value))
     except Exception as e:
         return '<repr(%s) raised %s: %s>' % (
                object.__repr__(value), type(e).__name__, e)

flask_debugtoolbar/panels/sqlalchemy.py

@@ -110,33 +110,22 @@ class SQLAlchemyDebugPanel(DebugPanel):
 
 
 @module.route('/sqlalchemy/sql_select', methods=['GET', 'POST'])
-def sql_select():
+@module.route('/sqlalchemy/sql_explain', methods=['GET', 'POST'],
+              defaults=dict(explain=True))
+def sql_select(explain=False):
     statement, params = load_query(request.args['query'])
     engine = SQLAlchemy().get_engine(current_app)
 
+    if explain:
+        if engine.driver == 'pysqlite':
+            statement = 'EXPLAIN QUERY PLAN\n%s' % statement
+        else:
+            statement = 'EXPLAIN\n%s' % statement
+
     result = engine.execute(statement, params)
     return g.debug_toolbar.render('panels/sqlalchemy_select.html', {
         'result': result.fetchall(),
         'headers': result.keys(),
         'sql': format_sql(statement, params),
         'duration': float(request.args['duration']),
-    })
-
-
-@module.route('/sqlalchemy/sql_explain', methods=['GET', 'POST'])
-def sql_explain():
-    statement, params = load_query(request.args['query'])
-    engine = SQLAlchemy().get_engine(current_app)
-
-    if engine.driver == 'pysqlite':
-        query = 'EXPLAIN QUERY PLAN %s' % statement
-    else:
-        query = 'EXPLAIN %s' % statement
-
-    result = engine.execute(query, params)
-    return g.debug_toolbar.render('panels/sqlalchemy_explain.html', {
-        'result': result.fetchall(),
-        'headers': result.keys(),
-        'sql': format_sql(statement, params),
-        'duration': float(request.args['duration']),
-    })
+    })

flask_debugtoolbar/static/css/toolbar.css

@@ -322,10 +322,6 @@
   text-align: left;
 }
 
-#flDebug .flSqlExplain td {
-  white-space: pre;
-}
-
 #flDebug span.flDebugLineChart {
   background-color:#777;
   height:3px;

flask_debugtoolbar/templates/panels/sqlalchemy.html

@@ -22,39 +22,7 @@
         </td>
         <td class="syntax">
           <div class="flDebugSqlWrap">
-            <div class="flDebugSql">{{ query.sql|safe }}</div>
-            {#
-            {% if query.stacktrace %}
-            <div class="djSQLHideStacktraceDiv" style="display:none;">
-              <table>
-                <tr>
-                  <th>{% trans "Line" %}</th>
-                  <th>{% trans "Method" %}</th>
-                  <th>{% trans "File" %}</th>
-                </tr>
-                {% for file, line, method in query.stacktrace %}
-                  <tr>
-                    <td>{{ line }}</td>
-                    <td><code>{{ method|escape }}</code></td>
-                    <td><code>{{ file|escape }}</code></td>
-                  </tr>
-                {% endfor %}
-              </table>
-              {% if query.template_info %}
-              <table>
-                {% for line in query.template_info.context %}
-                <tr>
-                  <td>{{ line.num }}</td>
-                  <td><code style="font-family: monospace;{% if line.highlight %}background-color: lightgrey{% endif %}">{{ line.content }}</code></td>
-                </tr>
-                {% endfor %}
-              </table>
-              <p><strong>{{ query.template_info.name|default:"(unknown)" }}</strong></p>
-              {% endif %}
-            </div>
-            {% endif %}
-            <span class="djDebugLineChart{% if query.is_slow %} djDebugLineChartWarning{% endif %}" style="width:{{ query.width_ratio }}%; left:{{ query.start_offset }}%;"></span>
-            #}
+            <div class="flDebugSql">{{ query.sql }}</div>
           </div>
         </td>
       </tr>

flask_debugtoolbar/templates/panels/sqlalchemy_explain.html

@@ -1,33 +0,0 @@
-<div class="flDebugPanelTitle">
-  <a class="flDebugClose flDebugBack" href="">Back</a>
-  <h3>SQL Explained</h3>
-</div>
-<div class="flDebugPanelContent">
-  <div class="scroll">
-    <dl>
-      <dt>Executed SQL</dt>
-      <dd>{{ sql|safe }}</dd>
-      <dt>Time</dt>
-      <dd>{{ '%.4f'|format(duration) }} ms</dd>
-    </dl>
-    <table class="flSqlExplain">
-      <thead>
-        <tr>
-          {% for h in headers %}
-            <th>{{ h|upper }}</th>
-          {% endfor %}
-        </tr>
-      </thead>
-      <tbody>
-        {% for row in result %}
-            <tr class="{{ loop.cycle('fjDebugOdd', 'fjDebugEven') }}">
-            {% for column in row %}
-              <td>{{ column }}</td>
-            {% endfor %}
-          </tr>
-        {% endfor %}
-      </tbody>
-    </table>
-  </div>
-</div>
-

flask_debugtoolbar/templates/panels/sqlalchemy_select.html

@@ -1,14 +1,14 @@
 <div class="flDebugPanelTitle">
   <a class="flDebugClose flDebugBack" href="">Back</a>
-  <h3>SQL Explained</h3>
+  <h3>SQL Details</h3>
 </div>
 <div class="flDebugPanelContent">
   <div class="scroll">
     <dl>
       <dt>Executed SQL</dt>
-      <dd>{{ sql|safe }}</dd>
-      <dt>Time</dt>
-      <dd>{{ '%.4f'|format(duration) }} ms</dd>
+      <dd>{{ sql }}</dd>
+      <dt>Original query duration</dt>
+      <dd>{{ '%.4f'|format(duration * 1000) }} ms</dd>
     </dl>
     {% if result %}
     <table class="flSqlSelect">

flask_debugtoolbar/utils.py

@@ -1,3 +1,4 @@
+import itertools
 import os.path
 import sys
 
@@ -12,46 +13,65 @@ except ImportError:
     HAVE_PYGMENTS = False
 
 
-from flask import current_app
+from flask import current_app, Markup
 
 
 def format_fname(value):
-    # If the value is not an absolute path, the it is a builtin or
-    # a relative file (thus a project file).
+    # If the value has a builtin prefix, return it unchanged
+    if value.startswith(('{', '<')):
+        return value
+
+    value = os.path.normpath(value)
+
+    # If the file is absolute, try normalizing it relative to the project root
+    # to handle it as a project file
+    if os.path.isabs(value):
+        value = _shortest_relative_path(
+            value, [current_app.root_path], os.path)
+
+    # If the value is a relative path, it is a project file
     if not os.path.isabs(value):
-        if value.startswith(('{', '<')):
-            return value
-        if value.startswith('.' + os.path.sep):
-            return value
-        return '.' + os.path.sep + value
+        return os.path.join('.', value)
 
-    # If the file is absolute and within the project root handle it as
-    # a project file
-    if value.startswith(current_app.root_path):
-        return "." + value[len(current_app.root_path):]
+    # Otherwise, normalize other paths relative to sys.path
+    return '<%s>' % _shortest_relative_path(value, sys.path, os.path)
 
-    # Loop through sys.path to find the longest match and return
-    # the relative path from there.
-    paths = sys.path
-    prefix = None
-    prefix_len = 0
-    for path in sys.path:
-        new_prefix = os.path.commonprefix([path, value])
-        if len(new_prefix) > prefix_len:
-            prefix = new_prefix
-            prefix_len = len(prefix)
 
-    if not prefix.endswith(os.path.sep):
-        prefix_len -= 1
-    path = value[prefix_len:]
-    return '<%s>' % path
+def _shortest_relative_path(value, paths, path_module):
+    relpaths = _relative_paths(value, paths, path_module)
+    return min(itertools.chain(relpaths, [value]), key=len)
+
+
+def _relative_paths(value, paths, path_module):
+    for path in paths:
+        try:
+            relval = path_module.relpath(value, path)
+        except ValueError:
+            # on Windows, relpath throws a ValueError for
+            # paths with different drives
+            continue
+        if not relval.startswith(path_module.pardir):
+            yield relval
+
+
+def decode_text(value):
+    """
+        Decode a text-like value for display.
+
+        Unicode values are returned unchanged. Byte strings will be decoded
+        with a text-safe replacement for unrecognized characters.
+    """
+    if isinstance(value, bytes):
+        return value.decode('ascii', 'replace')
+    else:
+        return value
 
 
 def format_sql(query, args):
     if not HAVE_PYGMENTS:
-        return query
+        return decode_text(query)
 
-    return highlight(
+    return Markup(highlight(
         query,
-        SqlLexer(encoding='utf-8'),
-        HtmlFormatter(encoding='utf-8', noclasses=True, style=PYGMENT_STYLE))
+        SqlLexer(),
+        HtmlFormatter(noclasses=True, style=PYGMENT_STYLE)))

setup.py

@@ -14,7 +14,7 @@ except:
 
 setup(
     name='Flask-DebugToolbar',
-    version='0.9.1',
+    version='0.9.2',
     url='http://flask-debugtoolbar.rtfd.org/',
     license='BSD',
     author='Michael van Tellingen',

test/test_utils.py

@@ -0,0 +1,120 @@
+import pytest
+
+import posixpath
+import ntpath
+
+from flask import Markup
+
+from flask_debugtoolbar.utils import (_relative_paths, _shortest_relative_path,
+   format_sql, decode_text, HAVE_PYGMENTS)
+
+
+@pytest.mark.parametrize('value,paths,expected,path_module', [
+    # should yield relative path to the parent directory
+    ('/foo/bar', ['/foo'], ['bar'], posixpath),
+    ('c:\\foo\\bar', ['c:\\foo'], ['bar'], ntpath),
+
+    # should not yield result if no path is a parent directory
+    ('/foo/bar', ['/baz'], [], posixpath),
+    ('c:\\foo\\bar', ['c:\\baz'], [], ntpath),
+
+    # should only yield relative paths for parent directories
+    ('/foo/bar', ['/foo', '/baz'], ['bar'], posixpath),
+    ('c:\\foo\\bar', ['c:\\foo', 'c:\\baz'], ['bar'], ntpath),
+
+    # should yield all results when multiple parents match
+    ('/foo/bar/baz', ['/foo', '/foo/bar'], ['bar/baz', 'baz'], posixpath),
+    ('c:\\foo\\bar\\baz', ['c:\\foo', 'c:\\foo\\bar'], ['bar\\baz', 'baz'], ntpath),
+
+    # should ignore case differences on windows
+    ('c:\\Foo\\bar', ['c:\\foo'], ['bar'], ntpath),
+
+    # should preserve original case
+    ('/Foo/Bar', ['/Foo'], ['Bar'], posixpath),
+    ('c:\\Foo\\Bar', ['c:\\foo'], ['Bar'], ntpath),
+])
+def test_relative_paths(value, paths, expected, path_module):
+    assert list(_relative_paths(value, paths, path_module)) == expected
+
+
+@pytest.mark.parametrize('value,paths,expected,path_module', [
+    # should yield relative path to the parent directory
+    ('/foo/bar', ['/foo'], 'bar', posixpath),
+    ('c:\\foo\\bar', ['c:\\foo'], 'bar', ntpath),
+    
+    # should return the original value if no path is a parent directory
+    ('/foo/bar', ['/baz'], '/foo/bar', posixpath),
+    ('c:\\foo\\bar', ['c:\\baz'], 'c:\\foo\\bar', ntpath),
+
+    # should yield shortest result when multiple parents match
+    ('/foo/bar/baz', ['/foo', '/foo/bar'], 'baz', posixpath),
+    ('c:\\foo\\bar\\baz', ['c:\\foo', 'c:\\foo\\bar'], 'baz', ntpath),
+])
+def test_shortest_relative_path(value, paths, expected, path_module):
+    assert _shortest_relative_path(value, paths, path_module) == expected
+
+
+def test_decode_text_unicode():
+    value = u'\uffff'
+    decoded = decode_text(value)
+    assert decoded == value
+
+
+def test_decode_text_ascii():
+    value = 'abc'
+    assert decode_text(value.encode('ascii')) == value
+
+
+def test_decode_text_non_ascii():
+    value = b'abc \xff xyz'
+    assert isinstance(value, bytes)
+
+    decoded = decode_text(value)
+    assert not isinstance(decoded, bytes)
+
+    assert decoded.startswith('abc')
+    assert decoded.endswith('xyz')
+
+
+@pytest.fixture()
+def no_pygments(monkeypatch):
+    monkeypatch.setattr('flask_debugtoolbar.utils.HAVE_PYGMENTS', False)
+
+
+def test_format_sql_no_pygments(no_pygments):
+    sql = 'select 1'
+    assert format_sql(sql, {}) == sql
+
+
+def test_format_sql_no_pygments_non_ascii(no_pygments):
+    sql = b"select '\xff'"
+    formatted = format_sql(sql, {})
+    assert formatted.startswith(u"select '")
+
+
+def test_format_sql_no_pygments_escape_html(no_pygments):
+    sql = 'select x < 1'
+    formatted = format_sql(sql, {})
+    assert not isinstance(formatted, Markup)
+    assert Markup('%s') % formatted == 'select x &lt; 1'
+
+
+@pytest.mark.skipif(not HAVE_PYGMENTS, reason='test requires the "Pygments" library')
+def test_format_sql_pygments():
+    sql = 'select 1'
+    html = format_sql(sql, {})
+    assert isinstance(html, Markup)
+    assert html.startswith('<div')
+    assert 'select' in html
+    assert '1' in html
+
+
+@pytest.mark.skipif(not HAVE_PYGMENTS, reason='test requires the "Pygments" library')
+def test_format_sql_pygments_non_ascii():
+    sql = b"select 'abc \xff xyz'"
+    html = format_sql(sql, {})
+    assert isinstance(html, Markup)
+    assert html.startswith('<div')
+    assert 'select' in html
+    assert 'abc' in html
+    assert 'xyz' in html

tox.ini

@@ -1,9 +1,10 @@
 [tox]
-envlist = py26,py27,py33
+envlist = py26,py27,py34
 
 [testenv]
 deps =
     pytest
     Flask-SQLAlchemy
+    Pygments
 commands =
     py.test