diff --git a/bot/reviewbot/tools/rbsecretscanner.py b/bot/reviewbot/tools/rbsecretscanner.py index c5e18549c23c21c4e5bcbae139cf90e7ca65d51b..035aefbde32041ca27ad9647da77426a92c73268 100644 --- a/bot/reviewbot/tools/rbsecretscanner.py +++ b/bot/reviewbot/tools/rbsecretscanner.py @@ -134,6 +134,12 @@ class SecretScannerTool(BaseTool): (pypi[-:][A-Za-z0-9_]{128,}) | + # Review Board 5+ API Token + # + # Rule updated: October 31, 2022 + (?Prbp_[A-Za-z0-9]{251}) + | + # RSA Private Key # # Rule updated: April 7, 2021 @@ -360,3 +366,36 @@ class SecretScannerTool(BaseTool): except Exception: # This isn't a JSON web token. return False + + def _is_reviewboard_api_token_valid(self, token, m): + """Return whether a Review Board API Token is valid. + + Review Board API tokens (which were revamped in October 2022) are 255 + characters long, prefixed with a ``rbp_`` prefix and contain a + Base62-encoded CRC32 checksum of the token data that follows, stored + as the last 6 characters of the token. + + Args: + token (bytes): + The full token to validate. + + m (object, unused): + The regex match data for the token. + + Returns: + bool: + ``True`` if the token is valid. ``False`` if it is not. + """ + checksum = base62_encode(crc32(token[4:-6]) & 0xFFFFFFFF).zfill(6) + checksum = checksum.decode('utf-8') + token = token.decode('utf-8') + token_checksum = token[-6:] + + # Review Board 5.0 generated token checksums using an incorrect + # base62-encoding, which resulted in capital and lowercase letters + # being swapped. We check against checksum.swapcase() to catch those. + return (len(token) == 255 and + token.startswith('rbp') and + re.match(r'^_[0-9A-Za-z]+$', token[3:]) is not None and + (token_checksum == checksum or + token_checksum == checksum.swapcase())) diff --git a/bot/reviewbot/tools/tests/test_rbsecretscanner.py b/bot/reviewbot/tools/tests/test_rbsecretscanner.py index 08590ee92c4d8dffc0a2236f6a793d42b3a7e57b..6c0856e718ae0a3e8265ae505f1420c7549d68bc 100644 --- a/bot/reviewbot/tools/tests/test_rbsecretscanner.py +++ b/bot/reviewbot/tools/tests/test_rbsecretscanner.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +import kgb import six from reviewbot.tools.rbsecretscanner import SecretScannerTool @@ -12,7 +13,7 @@ from reviewbot.utils.process import execute @six.add_metaclass(ToolTestCaseMetaclass) -class SecretScannerToolTests(BaseToolTestCase): +class SecretScannerToolTests(BaseToolTestCase, kgb.SpyAgency): """Unit tests for SecretScannerTool.""" tool_class = SecretScannerTool @@ -210,6 +211,8 @@ class SecretScannerToolTests(BaseToolTestCase): @integration_test() def test_execute_with_github_oauth_token_gho(self): """Testing SecretScannerTool.execute with GitHub token (gho...)""" + self.spy_on(self.tool_class._is_github_modern_token_valid) + # Lower bounds of length. self._run_token_test( 'gho_1234567890ABCDEFGabcdefg1234508vKGb') @@ -234,9 +237,24 @@ class SecretScannerToolTests(BaseToolTestCase): 'gho_1234567890ABCDEFGabcdefg1234508vKGbZ', match=False) + # No match because checksum does not match. + self._run_token_test( + 'gho_1234567890abcdef1234567890abcdef1234567890abcdef1234567890a' + 'bcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' + 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789' + '0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123451X8' + '8Ax', + match=False) + + self.assert_spy_call_count( + self.tool_class._is_github_modern_token_valid, + 5) + @integration_test() def test_execute_with_github_oauth_token_ghp(self): """Testing SecretScannerTool.execute with GitHub token (ghp...)""" + self.spy_on(self.tool_class._is_github_modern_token_valid) + # Lower bounds of length. self._run_token_test( 'ghp_1234567890ABCDEFGabcdefg1234508vKGb') @@ -261,9 +279,24 @@ class SecretScannerToolTests(BaseToolTestCase): 'ghp_1234567890ABCDEFGabcdefg1234508vKGbZ', match=False) + # No match because checksum does not match. + self._run_token_test( + 'ghp_1234567890abcdef1234567890abcdef1234567890abcdef1234567890a' + 'bcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' + 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789' + '0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123451X8' + '8Ax', + match=False) + + self.assert_spy_call_count( + self.tool_class._is_github_modern_token_valid, + 5) + @integration_test() def test_execute_with_github_oauth_token_ghr(self): """Testing SecretScannerTool.execute with GitHub token (ghr...)""" + self.spy_on(self.tool_class._is_github_modern_token_valid) + # Lower bounds of length. self._run_token_test( 'ghr_1234567890ABCDEFGabcdefg1234508vKGb') @@ -288,9 +321,24 @@ class SecretScannerToolTests(BaseToolTestCase): 'ghr_1234567890ABCDEFGabcdefg1234508vKGbZ', match=False) + # No match because checksum does not match. + self._run_token_test( + 'ghr_1234567890abcdef1234567890abcdef1234567890abcdef1234567890a' + 'bcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' + 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789' + '0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123451X8' + '8Ax', + match=False) + + self.assert_spy_call_count( + self.tool_class._is_github_modern_token_valid, + 5) + @integration_test() def test_execute_with_github_oauth_token_ghs(self): """Testing SecretScannerTool.execute with GitHub token (ghs...)""" + self.spy_on(self.tool_class._is_github_modern_token_valid) + # Lower bounds of length. self._run_token_test( 'ghs_1234567890ABCDEFGabcdefg1234508vKGb') @@ -315,9 +363,24 @@ class SecretScannerToolTests(BaseToolTestCase): 'ghs_1234567890ABCDEFGabcdefg1234508vKGbZ', match=False) + # No match because checksum does not match. + self._run_token_test( + 'ghs_1234567890abcdef1234567890abcdef1234567890abcdef1234567890a' + 'bcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' + 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789' + '0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123451X8' + '8Ax', + match=False) + + self.assert_spy_call_count( + self.tool_class._is_github_modern_token_valid, + 5) + @integration_test() def test_execute_with_github_oauth_token_ghu(self): """Testing SecretScannerTool.execute with GitHub token (ghu...)""" + self.spy_on(self.tool_class._is_github_modern_token_valid) + # Lower bounds of length. self._run_token_test( 'ghu_1234567890ABCDEFGabcdefg1234508vKGb') @@ -342,6 +405,19 @@ class SecretScannerToolTests(BaseToolTestCase): 'ghu_1234567890ABCDEFGabcdefg1234508vKGbZ', match=False) + # No match because checksum does not match. + self._run_token_test( + 'ghu_1234567890abcdef1234567890abcdef1234567890abcdef1234567890a' + 'bcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' + 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789' + '0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123451X8' + '8Ax', + match=False) + + self.assert_spy_call_count( + self.tool_class._is_github_modern_token_valid, + 5) + @integration_test() def test_execute_with_google_gcp_api_key(self): """Testing SecretScannerTool.execute with Google GCP API Key""" @@ -383,6 +459,8 @@ class SecretScannerToolTests(BaseToolTestCase): @integration_test() def test_execute_with_json_web_token(self): """Testing SecretScannerTool.execute with JSON Web Token""" + self.spy_on(self.tool_class._is_json_web_token_valid) + self._run_token_test( 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJrZXkiOiJ2YWx1ZSJ9.' '5pps1XMxciBCpOhezqTk9XuGny-4_HZ9aEKp3AqgekA') @@ -398,6 +476,10 @@ class SecretScannerToolTests(BaseToolTestCase): 'foo.bar.baz', match=False) + self.assert_spy_call_count( + self.tool_class._is_json_web_token_valid, + 4) + @integration_test() def test_execute_with_mailchimp_api_key(self): """Testing SecretScannerTool.execute with Mailchimp API key""" @@ -443,6 +525,50 @@ class SecretScannerToolTests(BaseToolTestCase): '0123456789_abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRST' 'UVWXYZ_0123456789_abcdefghijklmnopqrstuvwxyz') + @integration_test() + def test_execute_with_reviewboard_api_token(self): + """Testing SecretScannerTool.execute with Review Board API Token""" + self.spy_on(self.tool_class._is_reviewboard_api_token_valid) + + self._run_token_test( + 'rbp_eVZF8sv33KEWtMuupDJeipmW566fRZHveuAC7MchaktaJNTBQ4eNqhBQJKIv' + '6iKQJL64s8yMWfzp1Buc8p9m9ItCnkvNKTVuMJM299H7zffcrUJ0PQaSVSG0aUhg' + 'zzwIZ0MtK9spxYTLtu8bBVoQEiAsbDSfYICdKx23PfphGGqHyTwZRvYaebe1gGbv' + 'KgZ3PT8HqNQRAnAbUsDjwm4oiXXlmNOSM84a1uACZ2DCm9frU9z32pzHs4Ss9cI') + + # A Review Board 5.0.0 token whose checksum has the incorrect + # base62-encoding. + self._run_token_test( + 'rbp_eVZF8sv33KEWtMuupDJeipmW566fRZHveuAC7MchaktaJNTBQ4eNqhBQJKIv' + '6iKQJL64s8yMWfzp1Buc8p9m9ItCnkvNKTVuMJM299H7zffcrUJ0PQaSVSG0aUhg' + 'zzwIZ0MtK9spxYTLtu8bBVoQEiAsbDSfYICdKx23PfphGGqHyTwZRvYaebe1gGbv' + 'KgZ3PT8HqNQRAnAbUsDjwm4oiXXlmNOSM84a1uACZ2DCm9frU9z32pzHs4sS9Ci') + + self._run_token_test( + '!rbp_eVZF8sv33KEWtMuupDJeipmW566fRZHveuAC7MchaktaJNTBQ4eNqhBQJKI' + '6iKQJL64s8yMWfzp1Buc8p9m9ItCnkvNKTVuMJM299H7zffcrUJ0PQaSVSG0aUhg' + 'zzwIZ0MtK9spxYTLtu8bBVoQEiAsbDSfYICdKx23PfphGGqHyTwZRvYaebe1gGbv' + 'KgZ3PT8HqNQRAnAbUsDjwm4oiXXlmNOSM84a1uACZ2DCm9frU9z32pzHs4sS9Ci', + match=False) + self._run_token_test( + 'rbp_eVZF8sv33KEWtMuupDJeipmW566fRZHveuAC7MchaktaJNTBQ4eNqhBQJKIv' + '6iKQJL64s8yMWfzp1Buc8p9m9ItCnkvNKTVuMJM299H7zffcrUJ0PQaSVSG0aUhg' + 'zzwIZ0MtK9spxYTLtu8bBVoQEiAsbDSfYICdKx23PfphGGqHyTwZRvYaebe1gGbv' + 'KgZ3PT8HqNQRAnAbUsDjwm4oiXXlmNOSM84a1uACZ2DCm9frU9z32pzHs4sS9C!', + match=False) + + # No match because checksum does not match. + self._run_token_test( + 'rbp_eVZF8sv33KEWtMuupDJeipmW566fRZHveuAC7MchaktaJNTBQ4eNqhBQJKIv' + '6iKQJL64s8yMWfzp1Buc8p9m9ItCnkvNKTVuMJM299H7zffcrUJ0PQaSVSG0aUhg' + 'zzwIZ0MtK9spxYTLtu8bBVoQEiAsbDSfYICdKx23PfphGGqHyTwZRvYaebe1gGbv' + 'KgZ3PT8HqNQRAnAbUsDjwm4oiXXlmNOSM84a1uACZ2DCm9frU9z32pzHs4sS9Cb', + match=False) + + self.assert_spy_call_count( + self.tool_class._is_reviewboard_api_token_valid, + 3) + @integration_test() def test_execute_with_rsa_private_key(self): """Testing SecretScannerTool.execute with RSA Private Key""" diff --git a/docs/reviewbot/tools/rbsecretscanner.rst b/docs/reviewbot/tools/rbsecretscanner.rst index 126335bcada977728a195c5e547e23379319cadb..c22f53cc81ef4e6158091afa8fdb61b94fff6491 100644 --- a/docs/reviewbot/tools/rbsecretscanner.rst +++ b/docs/reviewbot/tools/rbsecretscanner.rst @@ -49,6 +49,7 @@ The following types of secrets are checked. * NPM Access Tokens * PGP Private Keys * PyPI API Tokens +* Review Board 5+ API Tokens * RSA Private Keys * SSH (DSA, EC, and OPENSSH) Private Keys * SSL Certificates