diff --git a/reviewboard/admin/forms.py b/reviewboard/admin/forms.py
index 684c9029ad95f07b2840b8f62745667b370d68d3..0505c49fd4fa718bc2217b09113f64e567488743 100644
--- a/reviewboard/admin/forms.py
+++ b/reviewboard/admin/forms.py
@@ -31,6 +31,7 @@ import os
 import re
 
 from django import forms
+from django.contrib import messages
 from django.contrib.sites.models import Site
 from django.conf import settings
 from django.core.cache import get_cache
@@ -635,13 +636,18 @@ class EMailSettingsForm(SiteSettingsForm):
             else:
                 to_user = siteconfig.get('site_admin_email')
 
-            send_mail(ugettext('E-mail settings test'),
-                      ugettext('This is a test of the e-mail settings '
-                               'for the Review Board server at %s.')
-                      % site_url,
-                      siteconfig.get('mail_default_from'),
-                      [to_user],
-                      fail_silently=True)
+            try:
+                send_mail(ugettext('E-mail settings test'),
+                          ugettext('This is a test of the e-mail settings '
+                                   'for the Review Board server at %s.')
+                          % site_url,
+                          siteconfig.get('mail_default_from'),
+                          [to_user],
+                          fail_silently=False)
+            except:
+                messages.error(self.request,
+                               ugettext('Failed to send test e-mail.'))
+                logging.exception('Failed to send test e-mail.')
 
     class Meta:
         title = _("E-Mail Settings")