diff --git a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberDisableController.php b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberDisableController.php
--- a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberDisableController.php
+++ b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberDisableController.php
@@ -7,16 +7,19 @@
     $viewer = $request->getViewer();
     $id = $request->getURIData('id');
 
-    $number = id(new PhabricatorAuthContactNumberQuery())
-      ->setViewer($viewer)
-      ->withIDs(array($id))
-      ->requireCapabilities(
-        array(
-          PhabricatorPolicyCapability::CAN_VIEW,
-          PhabricatorPolicyCapability::CAN_EDIT,
-        ))
-      ->executeOne();
-    if (!$number) {
+    $sms_auth_factor = new PhabricatorSMSAuthFactor();
+    if ($sms_auth_factor->isSMSMailerConfigured()) {
+      $number = id(new PhabricatorAuthContactNumberQuery())
+        ->setViewer($viewer)
+        ->withIDs(array($id))
+        ->requireCapabilities(
+          array(
+            PhabricatorPolicyCapability::CAN_VIEW,
+            PhabricatorPolicyCapability::CAN_EDIT,
+          ))
+        ->executeOne();
+    }
+    if (!isset($number) || !$number) {
       return new Aphront404Response();
     }
 
diff --git a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberEditController.php b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberEditController.php
--- a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberEditController.php
+++ b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberEditController.php
@@ -4,9 +4,14 @@
   extends PhabricatorAuthContactNumberController {
 
   public function handleRequest(AphrontRequest $request) {
-    return id(new PhabricatorAuthContactNumberEditEngine())
-      ->setController($this)
-      ->buildResponse();
+    $sms_auth_factor = new PhabricatorSMSAuthFactor();
+    if ($sms_auth_factor->isSMSMailerConfigured()) {
+      return id(new PhabricatorAuthContactNumberEditEngine())
+        ->setController($this)
+        ->buildResponse();
+    } else {
+      return new Aphront404Response();
+    }
   }
 
 }
diff --git a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberPrimaryController.php b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberPrimaryController.php
--- a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberPrimaryController.php
+++ b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberPrimaryController.php
@@ -7,16 +7,19 @@
     $viewer = $request->getViewer();
     $id = $request->getURIData('id');
 
-    $number = id(new PhabricatorAuthContactNumberQuery())
-      ->setViewer($viewer)
-      ->withIDs(array($id))
-      ->requireCapabilities(
-        array(
-          PhabricatorPolicyCapability::CAN_VIEW,
-          PhabricatorPolicyCapability::CAN_EDIT,
-        ))
-      ->executeOne();
-    if (!$number) {
+    $sms_auth_factor = new PhabricatorSMSAuthFactor();
+    if ($sms_auth_factor->isSMSMailerConfigured()) {
+      $number = id(new PhabricatorAuthContactNumberQuery())
+        ->setViewer($viewer)
+        ->withIDs(array($id))
+        ->requireCapabilities(
+          array(
+            PhabricatorPolicyCapability::CAN_VIEW,
+            PhabricatorPolicyCapability::CAN_EDIT,
+          ))
+        ->executeOne();
+    }
+    if (!isset($number) || !$number) {
       return new Aphront404Response();
     }
 
diff --git a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberTestController.php b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberTestController.php
--- a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberTestController.php
+++ b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberTestController.php
@@ -7,16 +7,19 @@
     $viewer = $request->getViewer();
     $id = $request->getURIData('id');
 
-    $number = id(new PhabricatorAuthContactNumberQuery())
-      ->setViewer($viewer)
-      ->withIDs(array($id))
-      ->requireCapabilities(
-        array(
-          PhabricatorPolicyCapability::CAN_VIEW,
-          PhabricatorPolicyCapability::CAN_EDIT,
-        ))
-      ->executeOne();
-    if (!$number) {
+    $sms_auth_factor = new PhabricatorSMSAuthFactor();
+    if ($sms_auth_factor->isSMSMailerConfigured()) {
+      $number = id(new PhabricatorAuthContactNumberQuery())
+        ->setViewer($viewer)
+        ->withIDs(array($id))
+        ->requireCapabilities(
+          array(
+            PhabricatorPolicyCapability::CAN_VIEW,
+            PhabricatorPolicyCapability::CAN_EDIT,
+          ))
+        ->executeOne();
+    }
+    if (!isset($number) || !$number) {
       return new Aphront404Response();
     }
 
diff --git a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberViewController.php b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberViewController.php
--- a/src/applications/auth/controller/contact/PhabricatorAuthContactNumberViewController.php
+++ b/src/applications/auth/controller/contact/PhabricatorAuthContactNumberViewController.php
@@ -6,11 +6,14 @@
   public function handleRequest(AphrontRequest $request) {
     $viewer = $this->getViewer();
 
-    $number = id(new PhabricatorAuthContactNumberQuery())
-      ->setViewer($viewer)
-      ->withIDs(array($request->getURIData('id')))
-      ->executeOne();
-    if (!$number) {
+    $sms_auth_factor = new PhabricatorSMSAuthFactor();
+    if ($sms_auth_factor->isSMSMailerConfigured()) {
+      $number = id(new PhabricatorAuthContactNumberQuery())
+        ->setViewer($viewer)
+        ->withIDs(array($request->getURIData('id')))
+        ->executeOne();
+    }
+    if (!isset($number) || !$number) {
       return new Aphront404Response();
     }
 
diff --git a/src/applications/auth/factor/PhabricatorSMSAuthFactor.php b/src/applications/auth/factor/PhabricatorSMSAuthFactor.php
--- a/src/applications/auth/factor/PhabricatorSMSAuthFactor.php
+++ b/src/applications/auth/factor/PhabricatorSMSAuthFactor.php
@@ -334,7 +334,7 @@
     return $value;
   }
 
-  private function isSMSMailerConfigured() {
+  public function isSMSMailerConfigured() {
     $mailers = PhabricatorMetaMTAMail::newMailers(
       array(
         'outbound' => true,
diff --git a/src/applications/settings/panel/PhabricatorContactNumbersSettingsPanel.php b/src/applications/settings/panel/PhabricatorContactNumbersSettingsPanel.php
--- a/src/applications/settings/panel/PhabricatorContactNumbersSettingsPanel.php
+++ b/src/applications/settings/panel/PhabricatorContactNumbersSettingsPanel.php
@@ -19,6 +19,18 @@
     return PhabricatorSettingsAuthenticationPanelGroup::PANELGROUPKEY;
   }
 
+  /**
+   * Whether to display "Contact Numbers" panel in users' Personal
+   * Settings by checking if global SMS support is configured
+   */
+  public function isUserPanel() {
+    $sms_auth_factor = new PhabricatorSMSAuthFactor();
+    if ($sms_auth_factor->isSMSMailerConfigured()) {
+      return true;
+    }
+    return false;
+  }
+
   public function isMultiFactorEnrollmentPanel() {
     return true;
   }