diff --git a/reviewboard/diffviewer/chunk_generator.py b/reviewboard/diffviewer/chunk_generator.py
index 638b6c307a110559f0433ff344e498b7375a1403..8f2d4aebfbeb694f176ac49b7dd22059a2263928 100644
--- a/reviewboard/diffviewer/chunk_generator.py
+++ b/reviewboard/diffviewer/chunk_generator.py
@@ -5,6 +5,7 @@ import functools
import hashlib
import re
+import pygments.util
from django.utils import six
from django.utils.html import escape
from django.utils.safestring import mark_safe
@@ -14,8 +15,8 @@ from djblets.log import log_timed
from djblets.cache.backend import cache_memoize
from djblets.siteconfig.models import SiteConfiguration
from pygments import highlight
-from pygments.lexers import guess_lexer_for_filename
from pygments.formatters import HtmlFormatter
+from pygments.lexers import guess_lexer_for_filename
from reviewboard.diffviewer.differ import DiffCompatVersion, get_differ
from reviewboard.diffviewer.diffutils import (get_line_changed_regions,
@@ -209,21 +210,14 @@ class RawDiffChunkGenerator(object):
markup_b = None
if self._get_enable_syntax_highlighting(old, new, a, b):
- source_file = \
- self.normalize_path_for_display(self.orig_filename)
- dest_file = \
- self.normalize_path_for_display(self.modified_filename)
-
- try:
- # TODO: Try to figure out the right lexer for these files
- # once instead of twice.
- if not source_file.endswith(self.STYLED_EXT_BLACKLIST):
- markup_a = self._apply_pygments(old or '', source_file)
-
- if not dest_file.endswith(self.STYLED_EXT_BLACKLIST):
- markup_b = self._apply_pygments(new or '', dest_file)
- except:
- pass
+ # TODO: Try to figure out the right lexer for these files
+ # once instead of twice.
+ markup_a = self._apply_pygments(
+ old or '',
+ self.normalize_path_for_display(self.orig_filename))
+ markup_b = self._apply_pygments(
+ new or '',
+ self.normalize_path_for_display(self.modified_filename))
if not markup_a:
markup_a = self.NEWLINES_RE.split(escape(old))
@@ -716,14 +710,35 @@ class RawDiffChunkGenerator(object):
self._last_header_index[0] = last_index
def _apply_pygments(self, data, filename):
- """Applies Pygments syntax-highlighting to a file's contents.
+ """Apply Pygments syntax-highlighting to a file's contents.
+
+ This will only apply syntax highlighting if a lexer is available and
+ the file extension is not blacklisted.
+
+ Args:
+ data (unicode):
+ The data to syntax highlight.
+
+ filename (unicode):
+ The name of the file. This is used to help determine a
+ suitable lexer.
- The resulting HTML will be returned as a list of lines.
+ Returns:
+ list of unicode:
+ A list of lines, all syntax-highlighted, if a lexer is found.
+ If no lexer is available, this will return ``None``.
"""
- lexer = guess_lexer_for_filename(filename,
- data,
- stripnl=False,
- encoding='utf-8')
+ if filename.endswith(self.STYLED_EXT_BLACKLIST):
+ return None
+
+ try:
+ lexer = guess_lexer_for_filename(filename,
+ data,
+ stripnl=False,
+ encoding='utf-8')
+ except pygments.util.ClassNotFound:
+ return None
+
lexer.add_filter('codetagify')
return split_line_endings(
diff --git a/reviewboard/diffviewer/diffutils.py b/reviewboard/diffviewer/diffutils.py
index 5d2be62641a19bddd61e671a5803c81618f07477..54b4a886049ccf3549579128b49f06c302f40664 100644
--- a/reviewboard/diffviewer/diffutils.py
+++ b/reviewboard/diffviewer/diffutils.py
@@ -31,7 +31,8 @@ CHUNK_RANGE_RE = re.compile(
NEWLINE_CONVERSION_BYTES_RE = re.compile(br'\r(\r?\n)?')
NEWLINE_CONVERSION_UNICODE_RE = re.compile(r'\r(\r?\n)?')
-NEWLINE_RE = re.compile(br'(?:\n|\r(?:\r?\n)?)')
+NEWLINE_BYTES_RE = re.compile(br'(?:\n|\r(?:\r?\n)?)')
+NEWLINE_UNICODE_RE = re.compile(r'(?:\n|\r(?:\r?\n)?)')
_PATCH_GARBAGE_INPUT = 'patch: **** Only garbage was found in the patch input.'
@@ -142,18 +143,33 @@ def convert_line_endings(data):
def split_line_endings(data):
- """Splits a string into lines while preserving all non-CRLF characters.
+ """Split a string into lines while preserving all non-CRLF characters.
- Unlike the string's splitlines(), this will only split on the following
- character sequences: \\n, \\r, \\r\\n, and \\r\\r\\n.
+ Unlike :py:meth:`str.splitlines`, this will only split on the following
+ character sequences: ``\n``, ``\r``, ``\r\n``, and ``\r\r\n``.
This is needed to prevent the sort of issues encountered with
- Unicode strings when calling splitlines(), which is that form feed
- characters would be split. patch and diff accept form feed characters
- as valid characters in diffs, and doesn't treat them as newlines, but
- splitlines() will treat it as a newline anyway.
+ Unicode strings when calling :py:meth:`str.splitlines``, which is that form
+ feed characters would be split. :program:`patch` and :program:`diff` accept
+ form feed characters as valid characters in diffs, and doesn't treat them
+ as newlines, but :py:meth:`str.splitlines` will treat it as a newline
+ anyway.
+
+ Args:
+ data (bytes or unicode):
+ The data to split into lines.
+
+ Returns:
+ list of bytes or unicode:
+ The list of lines.
"""
- lines = NEWLINE_RE.split(data)
+ if isinstance(data, bytes):
+ lines = NEWLINE_BYTES_RE.split(data)
+ elif isinstance(data, six.text_type):
+ lines = NEWLINE_UNICODE_RE.split(data)
+ else:
+ raise TypeError('data must be a bytes or unicode string, not %s'
+ % type(data))
# splitlines() would chop off the last entry, if the string ends with
# a newline. split() doesn't do this. We need to retain that same
diff --git a/reviewboard/diffviewer/tests/test_diffutils.py b/reviewboard/diffviewer/tests/test_diffutils.py
index af694ba0addd93caf1e2e1afc36f9d22cf36f50a..b5556dbbdd99659392a845f42e31d1358db604aa 100644
--- a/reviewboard/diffviewer/tests/test_diffutils.py
+++ b/reviewboard/diffviewer/tests/test_diffutils.py
@@ -2,6 +2,7 @@ from __future__ import print_function, unicode_literals
from django.contrib.auth.models import AnonymousUser
from django.test.client import RequestFactory
+from django.utils import six
from django.utils.six.moves import zip_longest
from djblets.siteconfig.models import SiteConfiguration
from djblets.testing.decorators import add_fixtures
@@ -21,6 +22,7 @@ from reviewboard.diffviewer.diffutils import (
get_revision_str,
get_sorted_filediffs,
patch,
+ split_line_endings,
_PATCH_GARBAGE_INPUT,
_get_last_header_in_chunks_before_line)
from reviewboard.diffviewer.errors import PatchError
@@ -3371,3 +3373,53 @@ class GetOriginalFileTests(BaseFileDiffAncestorTests):
encoding_list=['ascii'])
self.assertEqual(orig, b'')
+
+
+class SplitLineEndingsTests(TestCase):
+ """Unit tests for reviewboard.diffviewer.diffutils.split_line_endings."""
+
+ def test_with_byte_string(self):
+ """Testing split_line_endings with byte string"""
+ lines = split_line_endings(
+ b'This is line 1\n'
+ b'This is line 2\r\n'
+ b'This is line 3\r'
+ b'This is line 4\r\r\n'
+ b'This is line 5'
+ )
+
+ for line in lines:
+ self.assertIsInstance(line, bytes)
+
+ self.assertEqual(
+ lines,
+ [
+ b'This is line 1',
+ b'This is line 2',
+ b'This is line 3',
+ b'This is line 4',
+ b'This is line 5',
+ ])
+
+ def test_with_unicode_string(self):
+ """Testing split_line_endings with unicode string"""
+ lines = split_line_endings(
+ 'This is line 1\n'
+ 'This is line 2\r\n'
+ 'This is line 3\r'
+ 'This is line 4\r\r\n'
+ 'This is line 5'
+ )
+
+ for line in lines:
+ self.assertIsInstance(line, six.text_type)
+
+ self.assertEqual(
+ lines,
+ [
+ 'This is line 1',
+ 'This is line 2',
+ 'This is line 3',
+ 'This is line 4',
+ 'This is line 5',
+ ])
diff --git a/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py b/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py
index 9865ee0f9e0879212082cf362c1586344c5306f8..9fc0dc4f97cf82819b34e4bb2334f59cb25a9025 100644
--- a/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py
+++ b/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py
@@ -43,6 +43,177 @@ class RawDiffChunkGeneratorTests(TestCase):
self.assertEqual(chunks[2]['change'], 'equal')
self.assertEqual(chunks[3]['change'], 'replace')
+ def test_get_chunks_with_enable_syntax_highlighting_true(self):
+ """Testing RawDiffChunkGenerator.get_chunks with
+ enable_syntax_highlighting=True and syntax highlighting
+ available for file
+ """
+ old = b'This is **bold**'
+ new = b'This is *italic*'
+
+ generator = RawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='file1.md',
+ modified_filename='file2.md')
+ chunks = list(generator.get_chunks())
+
+ self.assertEqual(len(chunks), 1)
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ [
+ 1,
+ 1,
+ 'This is **bold**',
+ [(9, 16)],
+ 1,
+ 'This is *italic*',
+ [(9, 16)],
+ False,
+ ],
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ }
+ )
+
+ def test_get_chunks_with_enable_syntax_highlighting_false(self):
+ """Testing RawDiffChunkGenerator.get_chunks with
+ enable_syntax_highlighting=False
+ """
+ old = b'This is **bold**'
+ new = b'This is *italic*'
+
+ generator = RawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='file1.md',
+ modified_filename='file2.md',
+ enable_syntax_highlighting=False)
+ chunks = list(generator.get_chunks())
+
+ self.assertEqual(len(chunks), 1)
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ [
+ 1,
+ 1,
+ 'This is **bold**',
+ [(9, 16)],
+ 1,
+ 'This is *italic*',
+ [(9, 16)],
+ False,
+ ],
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ }
+ )
+
+ def test_get_chunks_with_syntax_highlighting_blacklisted(self):
+ """Testing RawDiffChunkGenerator.get_chunks with syntax highlighting
+ blacklisted for file
+ """
+ class MyRawDiffChunkGenerator(RawDiffChunkGenerator):
+ STYLED_EXT_BLACKLIST = (
+ '.md',
+ )
+
+ old = b'This is **bold**'
+ new = b'This is *italic*'
+
+ generator = MyRawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='file1.md',
+ modified_filename='file2.md')
+ chunks = list(generator.get_chunks())
+
+ self.assertEqual(len(chunks), 1)
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ [
+ 1,
+ 1,
+ 'This is **bold**',
+ [(9, 16)],
+ 1,
+ 'This is *italic*',
+ [(9, 16)],
+ False,
+ ],
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ }
+ )
+
+ def test_apply_pygments_with_lexer(self):
+ """Testing RawDiffChunkGenerator._apply_pygments with valid lexer"""
+ chunk_generator = RawDiffChunkGenerator(old=[],
+ new=[],
+ orig_filename='file1',
+ modified_filename='file2')
+ self.assertEqual(
+ chunk_generator._apply_pygments(data='This is **bold**\n',
+ filename='test.md'),
+ ['This is **bold**'])
+
+ def test_apply_pygments_without_lexer(self):
+ """Testing RawDiffChunkGenerator._apply_pygments without valid lexer"""
+ chunk_generator = RawDiffChunkGenerator(old=[],
+ new=[],
+ orig_filename='file1',
+ modified_filename='file2')
+ self.assertIsNone(
+ chunk_generator._apply_pygments(data='This is **bold**',
+ filename='test'))
+
+ def test_apply_pygments_with_blacklisted_file(self):
+ """Testing RawDiffChunkGenerator._apply_pygments with blacklisted
+ file extension
+ """
+ class MyRawDiffChunkGenerator(RawDiffChunkGenerator):
+ STYLED_EXT_BLACKLIST = (
+ '.md',
+ )
+
+ chunk_generator = MyRawDiffChunkGenerator(old=[],
+ new=[],
+ orig_filename='file1',
+ modified_filename='file2')
+ self.assertIsNone(
+ chunk_generator._apply_pygments(data='This is **bold**',
+ filename='test.md'))
+
def test_get_move_info_with_new_range_no_preceding(self):
"""Testing RawDiffChunkGenerator._get_move_info with new move range and
no adjacent preceding move range