diff --git a/reviewboard/hostingsvcs/gitlab.py b/reviewboard/hostingsvcs/gitlab.py index 4a9210e6394d6b93b2dc52875e0ba66970e90910..45fc511fa10ee32e4d548f05c540bc508cf61211 100644 --- a/reviewboard/hostingsvcs/gitlab.py +++ b/reviewboard/hostingsvcs/gitlab.py @@ -4,6 +4,7 @@ import json import re from django import forms +from django.core.cache import cache from django.core.exceptions import ValidationError from django.core.validators import validate_email from django.utils import six @@ -20,6 +21,7 @@ from reviewboard.hostingsvcs.service import HostingService from reviewboard.scmtools.crypto_utils import (decrypt_password, encrypt_password) from reviewboard.scmtools.errors import FileNotFoundError +from reviewboard.scmtools.core import Branch, Commit class GitLabPersonalForm(HostingServiceForm): @@ -52,9 +54,13 @@ class GitLab(HostingService): """ name = 'GitLab' + # The maximum number of commits returned from each call to get_commits() + COMMITS_PER_PAGE = 20 + self_hosted = True needs_authorization = True supports_bug_trackers = True + supports_post_commit = True supports_repositories = True supported_scmtools = ['Git'] @@ -181,6 +187,150 @@ class GitLab(HostingService): except (HTTPError, URLError): return False + def get_branches(self, repository): + """Get a list of branches. + + This will perform an API request to fetch a list of branches. + """ + repo_api_url = ('%s/repository/branches?private_token=%s' + % (self._get_repo_api_url(repository), + self._get_private_token())) + refs = self._api_get(repo_api_url)[0] + + results = [] + + for ref in refs: + if 'name' in ref: + name = ref['name'] + results.append(Branch(id=name, + commit=ref['commit']['id'], + default=(name == 'master'))) + + return results + + def get_commits(self, repository, branch=None, start=None): + """Get a list of commits + + This will perform an API request to fetch a list of commits. + The start parameter is a 40-character commit id. + """ + + # Ask GitLab for 21 commits per page. GitLab's API doesn't + # include the parent IDs, so we use subsequent commits to fill + # them in (allowing us to return 20 commits with parent IDs). + page_size = self.COMMITS_PER_PAGE + 1 + + repo_api_url = ('%s/repository/commits?private_token=%s&per_page=%s' + % (self._get_repo_api_url(repository), + self._get_private_token(), + page_size)) + + if start: + # If start parameter is given, use it as the latest commit to log + # from, so that we fetch a page of commits, and the first commit id + # on the page is the start parameter. + repo_api_url += '&ref_name=%s' % start + elif branch: + # The branch is optional. If it is not given, use the default + # branch. The default branch is set to 'master' in get_branches() + repo_api_url += '&ref_name=%s' % branch + + # The GitLab API will return a tuple consists of two elements. + # The first one is a list of commits, and the other one is an instance + # type object containing all kinds of headers, which is not required. + commits = self._api_get(repo_api_url)[0] + + results = [] + + for idx, item in enumerate(commits): + commit = Commit( + author_name=item['author_name'], + id=item['id'], + date=item['created_at'], + message=item['message'], + parent='') + + if idx > 0: + # Note that GitLab API documents do not show any returned + # 'parent_id' from the query for a list of commits. So we use + # the current commit id as the previous commit's parent id, and + # remove the last commit from results. + results[idx - 1].parent = commit.id + + results.append(commit) + + # Strip off the last commit since we don't know its parent id yet. + if len(commits) == page_size: + results.pop() + + return results + + def get_change(self, repository, revision): + """Get the diff of one commit with given commit ID. + + Revision is a commit ID, which is a long SHA consisting of 40 + characters. + """ + repo_api_url = self._get_repo_api_url(repository) + private_token = self._get_private_token() + + # Step 1: Fetch the commit itself that we want to review, to get + # the parent SHA and the commit message. Hopefully this information + # is still in cache so we don't have to fetch it again. However, the + # parent SHA is probably empty. + commit = cache.get(repository.get_commit_cache_key(revision)) + + if commit: + author_name = commit.author_name + date = commit.date + parent_revision = commit.parent + message = commit.message + else: + commit_api_url = ('%s/repository/commits/%s?private_token=%s' + % (repo_api_url, revision, private_token)) + + # This response from GitLab consists of one dict type commit and + # on instance type header object. Only the first element is needed. + commit = self._api_get(commit_api_url)[0] + + author_name = commit['author_name'] + date = commit['created_at'] + parent_revision = commit['parent_ids'][0] + message = commit['message'] + + # Step 2: Get the diff. The revision is the commit header in here. + # Firstly, a diff url should be built up, which has the format of + # https://gitlab.com///commit/.diff, + # then append the private_token to the end of the url and get the diff. + + # Get the project path with the namespace. + path_api_url = ('%s?private_token=%s' + % (repo_api_url, private_token)) + project = self._api_get(path_api_url)[0] + path_with_namespace = project['path_with_namespace'] + + # Build up diff url and get diff. + diff_url = ('https://gitlab.com/%s/commit/%s.diff?private_token=%s' + % (path_with_namespace, revision, private_token)) + diff, headers = self.client.http_get( + diff_url, + headers={'Accept': 'text/plain'}) + + # Remove the last two lines. The last line is 'libgit ', + # and the second last line is '--', ending with '\n'. To avoid the + # error from parsing the empty file (size is 0), split the string into + # two parts using the delimiter '--\nlibgit'. If only use '\n' or '--' + # delimiter, more characters might be stripped out from file + # modification commit diff. + diff = diff.rsplit('--\nlibgit', 2)[0] + + # Make sure there's a trailing newline. + if not diff.endswith('\n'): + diff += '\n' + + return Commit(author_name, revision, date, message, parent_revision, + diff=diff) + def _find_repository_id(self, plan, owner, repo_name): """Finds the ID of a repository matching the given name and owner. diff --git a/reviewboard/hostingsvcs/tests.py b/reviewboard/hostingsvcs/tests.py index 930eac56b6fe30ba71b29fe0302595eccc42f18a..b61482c270025634f5e6f7428c5ae210f87747a8 100644 --- a/reviewboard/hostingsvcs/tests.py +++ b/reviewboard/hostingsvcs/tests.py @@ -2264,6 +2264,190 @@ class GitLabTests(ServiceTests): self.assertEqual(fields['login'], 'myuser') self.assertEqual(fields['password'], 'mypass') + def test_get_branches(self): + """Testing GitLab get_branches implementation""" + branches_api_response = json.dumps([ + { + 'name': 'master', + 'commit': { + 'id': 'ed899a2f4b50b4370feeea94676502b42383c746' + } + }, + { + 'name': 'branch1', + 'commit': { + 'id': '6104942438c14ec7bd21c6cd5bd995272b3faff6' + } + }, + { + 'name': 'branch2', + 'commit': { + 'id': '21b3bcabcff2ab3dc3c9caa172f783aad602c0b0' + } + }, + { + 'branch-name': 'branch3', + 'commit': { + 'id': 'd5a3ff139356ce33e37e73add446f16869741b50' + } + } + ]) + + def _http_get(self, *args, **kwargs): + return branches_api_response, None + + account = self._get_hosting_account(use_url=True) + account.data['private_token'] = encrypt_password('abc123') + + service = account.service + + repository = Repository(hosting_account=account) + repository.extra_data = {'gitlab_project_id': 123456} + + self.spy_on(service.client.http_get, call_fake=_http_get) + + branches = service.get_branches(repository) + + self.assertTrue(service.client.http_get.called) + self.assertEqual(len(branches), 3) + self.assertEqual( + branches, + [ + Branch(id='master', + commit='ed899a2f4b50b4370feeea94676502b42383c746', + default=True), + Branch(id='branch1', + commit='6104942438c14ec7bd21c6cd5bd995272b3faff6', + default=False), + Branch(id='branch2', + commit='21b3bcabcff2ab3dc3c9caa172f783aad602c0b0', + default=False) + ]) + + def test_get_commits(self): + """Testing GitLab get_commits implementation""" + commits_api_response = json.dumps([ + { + 'id': 'ed899a2f4b50b4370feeea94676502b42383c746', + 'author_name': 'Chester Li', + 'created_at': '2015-03-10T11:50:22+03:00', + 'message': 'Replace sanitize with escape once' + }, + { + 'id': '6104942438c14ec7bd21c6cd5bd995272b3faff6', + 'author_name': 'Chester Li', + 'created_at': '2015-03-10T09:06:12+03:00', + 'message': 'Sanitize for network graph' + }, + { + 'id': '21b3bcabcff2ab3dc3c9caa172f783aad602c0b0', + 'author_name': 'East Coast', + 'created_at': '2015-03-04T15:31:18.000-04:00', + 'message': 'Add a timer to test file' + } + ]) + + def _http_get(self, *args, **kargs): + return commits_api_response, None + + account = self._get_hosting_account(use_url=True) + account.data['private_token'] = encrypt_password('abc123') + + service = account.service + + repository = Repository(hosting_account=account) + repository.extra_data = {'gitlab_project_id': 123456} + + self.spy_on(service.client.http_get, call_fake=_http_get) + + commits = service.get_commits( + repository, start='ed899a2f4b50b4370feeea94676502b42383c746') + + self.assertTrue(service.client.http_get.called) + self.assertEqual(len(commits), 3) + self.assertEqual(commits[0].id, + 'ed899a2f4b50b4370feeea94676502b42383c746') + self.assertNotEqual(commits[0].author_name, 'East Coast') + self.assertEqual(commits[1].date, '2015-03-10T09:06:12+03:00') + self.assertNotEqual(commits[1].message, + 'Replace sanitize with escape once') + self.assertEqual(commits[2].author_name, 'East Coast') + + def test_get_change(self): + """Testing GitLab get_change implementation""" + commit_id = 'ed899a2f4b50b4370feeea94676502b42383c746' + + commit_api_response = json.dumps( + { + 'author_name': 'Chester Li', + 'id': commit_id, + 'created_at': '2015-03-10T11:50:22+03:00', + 'message': 'Replace sanitize with escape once', + 'parent_ids': ['ae1d9fb46aa2b07ee9836d49862ec4e2c46fbbba'] + } + ) + + path_api_response = json.dumps( + { + 'path_with_namespace': 'username/project_name' + } + ) + + diff = dedent(b'''\ + --- + f1 | 1 + + f2 | 1 + + 2 files changed, 2 insertions(+), 0 deletions(-) + + diff --git a/f1 b/f1 + index 11ac561..3ea0691 100644 + --- a/f1 + +++ b/f1 + @@ -1 +1,2 @@ + this is f1 + +add one line to f1 + diff --git a/f2 b/f2 + index c837441..9302ecd 100644 + --- a/f2 + +++ b/f2 + @@ -1 +1,2 @@ + this is f2 + +add one line to f2 + ''') + + def _http_get(service, url, *args, **kwargs): + parsed = urlparse(url) + if parsed.path.startswith( + '/api/v3/projects/123456/repository/commits'): + # If the url is commit_api_url. + return commit_api_response, None + elif parsed.path == '/api/v3/projects/123456': + # If the url is path_api_url. + return path_api_response, None + elif parsed.path.endswith('.diff'): + # If the url is diff_url. + return diff, None + else: + print(parsed) + self.fail('Got an unexpected GET request') + + account = self._get_hosting_account(use_url=True) + account.data['private_token'] = encrypt_password('abc123') + + service = account.service + + repository = Repository(hosting_account=account) + repository.extra_data = {'gitlab_project_id': 123456} + + self.spy_on(service.client.http_get, call_fake=_http_get) + + commit = service.get_change(repository, commit_id) + + self.assertTrue(service.client.http_get.called) + self.assertEqual(commit.date, '2015-03-10T11:50:22+03:00') + self.assertEqual(commit.diff, diff) + self.assertNotEqual(commit.parent, '') + def _test_check_repository(self, expected_user='myuser', **kwargs): def _http_get(service, url, *args, **kwargs): if url == 'https://example.com/api/v3/projects?per_page=100':