diff --git a/reviewboard/diffviewer/chunk_generator.py b/reviewboard/diffviewer/chunk_generator.py
index 61beaf1b97962d7f907a7149cb1bd101d22e7ce1..07fd8ae548ffe026ec5329ba5dba9723867a2651 100644
--- a/reviewboard/diffviewer/chunk_generator.py
+++ b/reviewboard/diffviewer/chunk_generator.py
@@ -21,6 +21,7 @@ from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import (find_lexer_class,
guess_lexer_for_filename)
+from tree_sitter_language_pack import get_parser
from reviewboard.codesafety import code_safety_checker_registry
from reviewboard.deprecation import (
@@ -40,16 +41,20 @@ from reviewboard.diffviewer.diffutils import (
)
from reviewboard.diffviewer.opcode_generator import get_diff_opcode_generator
from reviewboard.diffviewer.settings import DiffSettings
+from reviewboard.treesitter.highlight import highlight as ts_highlight
+from reviewboard.treesitter.language import get_language_name_for_file
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Sequence
+ import tree_sitter
from django.http import HttpRequest
from typing_extensions import TypeAlias
from reviewboard.diffviewer.differ import Differ, DiffOpcodeTag
from reviewboard.diffviewer.models.filediff import FileDiff
from reviewboard.diffviewer.opcode_generator import DiffOpcodeGenerator
+ from reviewboard.treesitter.language import SupportedLanguage
_HtmlFormatter = HtmlFormatter[str]
else:
@@ -254,6 +259,15 @@ class RawDiffChunkGenerator:
#: The old version of the file.
old: bytes | Sequence[bytes]
+ #: The tree-sitter language for the old file.
+ old_language_name: SupportedLanguage | None
+
+ #: The MIME type for the old file.
+ old_mimetype: str | None
+
+ #: The old file as a tree-sitter tree.
+ old_tree: tree_sitter.Tree | None
+
#: The filename for the old version of the file.
orig_filename: str
@@ -263,6 +277,15 @@ class RawDiffChunkGenerator:
#: The new version of the file.
new: bytes | Sequence[bytes]
+ #: The tree-sitter language for the new file.
+ new_language_name: SupportedLanguage | None
+
+ #: The MIME type for the new file.
+ new_mimetype: str | None
+
+ #: The new file as a tree-sitter tree.
+ new_tree: tree_sitter.Tree | None
+
#: The current chunk being processed.
_chunk_index: int
@@ -371,6 +394,13 @@ class RawDiffChunkGenerator:
self.diff_compat = diff_compat
self.differ = None
+ self.old_language_name = None
+ self.old_mimetype = None
+ self.old_tree = None
+ self.new_language_name = None
+ self.new_mimetype = None
+ self.new_tree = None
+
self.all_code_safety_results = {}
# Chunk processing state.
@@ -469,6 +499,9 @@ class RawDiffChunkGenerator:
is_lists = isinstance(old, list)
assert is_lists == isinstance(new, list)
+ old_filename = self.normalize_path_for_display(self.orig_filename)
+ new_filename = self.normalize_path_for_display(self.modified_filename)
+
if old_encoding_list is None:
old_encoding_list = self.encoding_list
@@ -496,14 +529,37 @@ class RawDiffChunkGenerator:
old_markup = None
new_markup = None
+ old_language_name = get_language_name_for_file(old_filename)
+ new_language_name = get_language_name_for_file(new_filename)
+
+ self.old_language_name = old_language_name
+ self.new_language_name = new_language_name
+
+ if old_language_name:
+ parser = get_parser(old_language_name)
+ self.old_tree = parser.parse(old_str.encode())
+
+ if new_language_name:
+ parser = get_parser(new_language_name)
+ self.new_tree = parser.parse(new_str.encode())
+
if self._get_enable_syntax_highlighting(
old, new, old_lines, new_lines):
- old_markup = self._apply_pygments(
- old_str,
- self.normalize_path_for_display(self.orig_filename))
- new_markup = self._apply_pygments(
- new_str,
- self.normalize_path_for_display(self.modified_filename))
+
+ old_markup = self._highlight(
+ data_str=old_str,
+ data_lines=old_lines,
+ filename=old_filename,
+ tree=self.old_tree,
+ language=old_language_name,
+ mimetype=self.old_mimetype)
+ new_markup = self._highlight(
+ data_str=new_str,
+ data_lines=new_lines,
+ filename=new_filename,
+ tree=self.new_tree,
+ language=new_language_name,
+ mimetype=self.new_mimetype)
if not old_markup:
old_markup = self.NEWLINES_RE.split(escape(old_str))
@@ -522,7 +578,7 @@ class RawDiffChunkGenerator:
self.differ = get_differ(old_lines, new_lines,
ignore_space=ignore_space,
compat_version=self.diff_compat)
- self.differ.add_interesting_lines_for_headers(self.orig_filename)
+ self.differ.add_interesting_lines_for_headers(old_filename)
context_num_lines = self.diff_settings.context_num_lines
collapse_threshold = 2 * context_num_lines + 3
@@ -1493,12 +1549,65 @@ class RawDiffChunkGenerator:
else:
self._last_header_index[0] = last_index
+ def _highlight(
+ self,
+ *,
+ data_str: str,
+ data_lines: Sequence[str],
+ filename: str,
+ tree: tree_sitter.Tree | None,
+ language: SupportedLanguage | None,
+ mimetype: str | None,
+ ) -> Sequence[str] | None:
+ """Apply syntax highlighting to a file's context.
+
+ Args:
+ data_str (str):
+ The file's content, as a string.
+
+ data_lines (list of str):
+ The file's content, split into lines.
+
+ filename (str):
+ The name of the file.
+
+ tree (tree_sitter.Tree):
+ The parsed tree-sitter tree for the file, if available.
+
+ language (reviewboard.treesitter.language.SupportedLanguage):
+ The name of the language used for tree-sitter parsing, if
+ available.
+
+ mimetype (str):
+ The MIME type of the file, if available,
+
+ Returns:
+ list of str:
+ A list of lines, all syntax-highlighted, if highlighting is
+ possible. If neither tree-sitter or Pygments can highlight the
+ file, this will return ``None``.
+ """
+ if tree and language:
+ try:
+ highlighted = ts_highlight(
+ data_str.encode(),
+ data_lines,
+ tree,
+ language)
+
+ if highlighted:
+ return highlighted
+ except Exception as e:
+ logger.exception('Tree sitter highlighting failed: %s', e)
+
+ return self._apply_pygments(data_str, filename)
+
def _apply_pygments(
self,
data: str,
filename: str,
) -> Sequence[str] | None:
- """Apply Pygments syntax-highlighting to a file's contents.
+ """Use Pygments to syntax highlight a file's contents.
This will only apply syntax highlighting if a lexer is available and
the file extension is not blacklisted.
@@ -1763,6 +1872,9 @@ class DiffChunkGenerator(RawDiffChunkGenerator):
old_encoding_list = get_filediff_encodings(filediff)
new_encoding_list = old_encoding_list
+ self.old_mimetype = filediff.extra_data.get('source_mimetype')
+ self.new_mimetype = filediff.extra_data.get('dest_mimetype')
+
if base_filediff is not None:
# The diff is against a commit that:
#
diff --git a/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py b/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py
index b0b5c00d729675fb786afbfd52ba8071ef938c92..8b6b6b7e7f1616fc7e245c06b43dee2c72680a4b 100644
--- a/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py
+++ b/reviewboard/diffviewer/tests/test_raw_diff_chunk_generator.py
@@ -1,13 +1,22 @@
+"""Unit tests for RawDiffChunkGenerator."""
+
+from __future__ import annotations
+
+import kgb
+from tree_sitter import QueryError
+
from reviewboard.diffviewer.chunk_generator import RawDiffChunkGenerator
from reviewboard.diffviewer.settings import DiffSettings
from reviewboard.testing import TestCase
+from reviewboard.treesitter.highlight import highlight
+from reviewboard.treesitter.language import get_language_name_for_file
-class RawDiffChunkGeneratorTests(TestCase):
+class RawDiffChunkGeneratorTests(kgb.SpyAgency, TestCase):
"""Unit tests for RawDiffChunkGenerator."""
@property
- def generator(self):
+ def generator(self) -> RawDiffChunkGenerator:
"""Create a dummy generator for tests that need it.
This generator will be void of any content. It's intended for
@@ -19,7 +28,7 @@ class RawDiffChunkGeneratorTests(TestCase):
modified_filename='',
diff_settings=DiffSettings.create())
- def test_get_chunks(self):
+ def test_get_chunks(self) -> None:
"""Testing RawDiffChunkGenerator.get_chunks"""
old = (
b'This is line 1\n'
@@ -148,7 +157,7 @@ class RawDiffChunkGeneratorTests(TestCase):
'numlines': 1,
})
- def test_get_chunks_with_settings_syntax_highlighting_true(self):
+ def test_get_chunks_with_settings_syntax_highlighting_true(self) -> None:
"""Testing RawDiffChunkGenerator.get_chunks with
DiffSettings.syntax_highlighting=True and syntax highlighting
available for file
@@ -197,7 +206,7 @@ class RawDiffChunkGeneratorTests(TestCase):
}
)
- def test_get_chunks_with_settings_syntax_highlighting_false(self):
+ def test_get_chunks_with_settings_syntax_highlighting_false(self) -> None:
"""Testing RawDiffChunkGenerator.get_chunks with
legacy enable_syntax_highlighting=False
"""
@@ -245,8 +254,7 @@ class RawDiffChunkGeneratorTests(TestCase):
}
)
-
- def test_get_chunks_with_syntax_highlighting_blacklisted(self):
+ def test_get_chunks_with_syntax_highlighting_blacklisted(self) -> None:
"""Testing RawDiffChunkGenerator.get_chunks with syntax highlighting
blacklisted for file
"""
@@ -297,7 +305,7 @@ class RawDiffChunkGeneratorTests(TestCase):
}
)
- def test_generate_chunks_with_encodings(self):
+ def test_generate_chunks_with_encodings(self) -> None:
"""Testing RawDiffChunkGenerator.generate_chunks with explicit
encodings for old and new
"""
@@ -436,7 +444,7 @@ class RawDiffChunkGeneratorTests(TestCase):
'numlines': 1,
})
- def test_get_chunks_with_code_safety_results(self):
+ def test_get_chunks_with_code_safety_results(self) -> None:
"""Testing RawDiffChunkGenerator.get_chunks with code safety results"""
old = '/* Say hello; newline\u2067 /*/ return 0 ;'.encode('utf-8')
new = 'def is\u200BAdmin():'.encode('utf-8')
@@ -460,20 +468,20 @@ class RawDiffChunkGeneratorTests(TestCase):
(
1,
1,
- ('/* Say hello; newline\u2067 /*/'
- ' '
- 'return'
- ' '
- '0'
- ' '
- ';'),
+ (
+ ' return 0 ;'
+ ),
None,
1,
- ('def '
- 'is'
- '\u200b'
- 'Admin'
- '():'),
+ (
+ 'def '
+ 'is\u200b'
+ 'Admin():'
+ ),
None,
False,
{
@@ -907,3 +915,473 @@ class RawDiffChunkGeneratorTests(TestCase):
'warnings': {'bidi', 'zws'},
}),
])
+
+ def test_get_chunks_syntax_highlighting_treesitter(self) -> None:
+ """Testing RawDiffChunkGenerator.get_chunks with syntax highlighting
+ from Tree Sitter
+ """
+ old = b'int main() {\n return 0;\n}'
+ new = b'int main() {\n return 1;\n}'
+
+ diff_settings = DiffSettings.create(syntax_highlighting=True)
+
+ generator = RawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='main.c',
+ modified_filename='main.c',
+ diff_settings=diff_settings)
+ self.assertTrue(generator.enable_syntax_highlighting)
+
+ chunks = list(generator.get_chunks())
+
+ self.assertEqual(len(chunks), 3)
+
+ # First chunk: equal line "int main() {"
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ (
+ 1,
+ 1,
+ 'int main'
+ '() {',
+ None,
+ 1,
+ 'int main'
+ '() {',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [(1, 'int main() {')],
+ 'right_headers': [(1, 'int main() {')],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Second chunk: replaced line with return statement
+ self.assertEqual(
+ chunks[1],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 1,
+ 'lines': [
+ (
+ 2,
+ 2,
+ ' return 0;',
+ [(11, 12)],
+ 2,
+ ' return 1;',
+ [(11, 12)],
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Third chunk: equal line "}"
+ self.assertEqual(
+ chunks[2],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 2,
+ 'lines': [
+ (
+ 3,
+ 3,
+ '}',
+ None,
+ 3,
+ '}',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ def test_get_chunks_syntax_highlighting_no_ts_language(self) -> None:
+ """Testing RawDiffChunkGenerator.get_chunks with syntax highlighting
+ with no Tree Sitter language detected (fall back to Pygemnts)
+ """
+ old = b'int main() {\n return 0;\n}'
+ new = b'int main() {\n return 1;\n}'
+
+ diff_settings = DiffSettings.create(syntax_highlighting=True)
+
+ self.spy_on(get_language_name_for_file,
+ op=kgb.SpyOpReturn(None))
+
+ generator = RawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='main.c',
+ modified_filename='main.c',
+ diff_settings=diff_settings)
+ self.assertTrue(generator.enable_syntax_highlighting)
+
+ chunks = list(generator.get_chunks())
+
+ self.assertSpyCalled(get_language_name_for_file)
+ self.assertEqual(len(chunks), 3)
+
+ # First chunk: equal line "int main() {"
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ (
+ 1,
+ 1,
+ 'int '
+ 'main()'
+ ' {',
+ None,
+ 1,
+ 'int '
+ 'main()'
+ ' {',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [(1, 'int main() {')],
+ 'right_headers': [(1, 'int main() {')],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Second chunk: replaced line with return statement
+ self.assertEqual(
+ chunks[1],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 1,
+ 'lines': [
+ (
+ 2,
+ 2,
+ ' return'
+ ' 0'
+ ';',
+ [(11, 12)],
+ 2,
+ ' return'
+ ' 1'
+ ';',
+ [(11, 12)],
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Third chunk: equal line "}"
+ self.assertEqual(
+ chunks[2],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 2,
+ 'lines': [
+ (
+ 3,
+ 3,
+ '}',
+ None,
+ 3,
+ '}',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ def test_get_chunks_syntax_highlighting_ts_fails(self) -> None:
+ """Testing RawDiffChunkGenerator.get_chunks with syntax highlighting
+ when Tree Sitter highlighting returns None (fall back to Pygemnts)
+ """
+ old = b'int main() {\n return 0;\n}'
+ new = b'int main() {\n return 1;\n}'
+
+ diff_settings = DiffSettings.create(syntax_highlighting=True)
+
+ self.spy_on(highlight,
+ op=kgb.SpyOpReturn(None))
+
+ generator = RawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='main.c',
+ modified_filename='main.c',
+ diff_settings=diff_settings)
+ self.assertTrue(generator.enable_syntax_highlighting)
+
+ chunks = list(generator.get_chunks())
+
+ self.assertSpyCalled(highlight)
+ self.assertEqual(len(chunks), 3)
+
+ # First chunk: equal line "int main() {"
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ (
+ 1,
+ 1,
+ 'int '
+ 'main()'
+ ' {',
+ None,
+ 1,
+ 'int '
+ 'main()'
+ ' {',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [(1, 'int main() {')],
+ 'right_headers': [(1, 'int main() {')],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Second chunk: replaced line with return statement
+ self.assertEqual(
+ chunks[1],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 1,
+ 'lines': [
+ (
+ 2,
+ 2,
+ ' return'
+ ' 0'
+ ';',
+ [(11, 12)],
+ 2,
+ ' return'
+ ' 1'
+ ';',
+ [(11, 12)],
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Third chunk: equal line "}"
+ self.assertEqual(
+ chunks[2],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 2,
+ 'lines': [
+ (
+ 3,
+ 3,
+ '}',
+ None,
+ 3,
+ '}',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ def test_get_chunks_syntax_highlighting_ts_exception(self) -> None:
+ """Testing RawDiffChunkGenerator.get_chunks with syntax highlighting
+ when Tree Sitter highlighting has an exception (fall back to Pygemnts)
+ """
+ old = b'int main() {\n return 0;\n}'
+ new = b'int main() {\n return 1;\n}'
+
+ diff_settings = DiffSettings.create(syntax_highlighting=True)
+
+ self.spy_on(highlight,
+ op=kgb.SpyOpRaise(QueryError('query failed')))
+
+ generator = RawDiffChunkGenerator(old=old,
+ new=new,
+ orig_filename='main.c',
+ modified_filename='main.c',
+ diff_settings=diff_settings)
+ self.assertTrue(generator.enable_syntax_highlighting)
+
+ chunks = list(generator.get_chunks())
+
+ self.assertSpyCalled(highlight)
+ self.assertEqual(len(chunks), 3)
+
+ # First chunk: equal line "int main() {"
+ self.assertEqual(
+ chunks[0],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 0,
+ 'lines': [
+ (
+ 1,
+ 1,
+ 'int '
+ 'main()'
+ ' {',
+ None,
+ 1,
+ 'int '
+ 'main()'
+ ' {',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [(1, 'int main() {')],
+ 'right_headers': [(1, 'int main() {')],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Second chunk: replaced line with return statement
+ self.assertEqual(
+ chunks[1],
+ {
+ 'change': 'replace',
+ 'collapsable': False,
+ 'index': 1,
+ 'lines': [
+ (
+ 2,
+ 2,
+ ' return'
+ ' 0'
+ ';',
+ [(11, 12)],
+ 2,
+ ' return'
+ ' 1'
+ ';',
+ [(11, 12)],
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
+
+ # Third chunk: equal line "}"
+ self.assertEqual(
+ chunks[2],
+ {
+ 'change': 'equal',
+ 'collapsable': False,
+ 'index': 2,
+ 'lines': [
+ (
+ 3,
+ 3,
+ '}',
+ None,
+ 3,
+ '}',
+ None,
+ False,
+ None,
+ ),
+ ],
+ 'meta': {
+ 'left_headers': [],
+ 'right_headers': [],
+ 'whitespace_chunk': False,
+ 'whitespace_lines': [],
+ },
+ 'numlines': 1,
+ })
diff --git a/reviewboard/webapi/tests/test_filediff.py b/reviewboard/webapi/tests/test_filediff.py
index 898323d12c9cdbf03347938342f29771def72457..35e6ade0aab0ab4b15f03e51291e63ea4b353692 100644
--- a/reviewboard/webapi/tests/test_filediff.py
+++ b/reviewboard/webapi/tests/test_filediff.py
@@ -498,11 +498,11 @@ class ResourceItemTests(ExtraDataItemMixin, ReviewRequestChildItemMixin,
'',
None,
1,
- 'print'
- '('
- ''hello, '
- 'world!''
- ')',
+ 'print'
+ '(''
+ 'hello, world!')',
None,
False,
],