No OneTemporary
Actions

Size

70 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/scripts/sql/manage_storage.php b/scripts/sql/manage_storage.php
	index 2ba1e9920c..521e269386 100755
	--- a/scripts/sql/manage_storage.php
	+++ b/scripts/sql/manage_storage.php
	@@ -1,211 +1,213 @@
	#!/usr/bin/env php
	<?php

	$root = dirname(dirname(dirname(__FILE__)));
	require_once $root.'/scripts/init/init-setup.php';

	$args = new PhutilArgumentParser($argv);
	$args->setTagline(pht('manage Phabricator storage and schemata'));
	$args->setSynopsis(<<<EOHELP
	storage __workflow__ [__options__]
	Manage Phabricator database storage and schema versioning.

	storage upgrade
	Initialize or upgrade Phabricator storage.

	storage upgrade --user __root__ --password __hunter2__
	Use administrative credentials for schema changes.
	EOHELP
	);
	$args->parseStandardArguments();

	$default_namespace = PhabricatorLiskDAO::getDefaultStorageNamespace();

	try {
	$args->parsePartial(
	array(
	array(
	'name' => 'force',
	'short' => 'f',
	'help' => pht(
	'Do not prompt before performing dangerous operations.'),
	),
	array(
	'name' => 'host',
	'param' => 'hostname',
	'help' => pht(
	'Connect to __host__ instead of the default host.'),
	),
	array(
	'name' => 'user',
	'short' => 'u',
	'param' => 'username',
	'help' => pht(
	'Connect with __username__ instead of the configured default.'),
	),
	array(
	'name' => 'password',
	'short' => 'p',
	'param' => 'password',
	'help' => pht('Use __password__ instead of the configured default.'),
	),
	array(
	'name' => 'namespace',
	'param' => 'name',
	'default' => $default_namespace,
	'help' => pht(
	"Use namespace __namespace__ instead of the configured ".
	"default ('%s'). This is an advanced feature used by unit tests; ".
	"you should not normally use this flag.",
	$default_namespace),
	),
	array(
	'name' => 'dryrun',
	'help' => pht(
	'Do not actually change anything, just show what would be changed.'),
	),
	array(
	'name' => 'disable-utf8mb4',
	'help' => pht(
	'Disable %s, even if the database supports it. This is an '.
	'advanced feature used for testing changes to Phabricator; you '.
	'should not normally use this flag.',
	'utf8mb4'),
	),
	));
	} catch (PhutilArgumentUsageException $ex) {
	$args->printUsageException($ex);
	exit(77);
	}

	// First, test that the Phabricator configuration is set up correctly. After
	// we know this works we'll test any administrative credentials specifically.

	$host = $args->getArg('host');
	if (strlen($host)) {
	$ref = null;

	$refs = PhabricatorDatabaseRef::getLiveRefs();

	// Include the master in case the user is just specifying a redundant
	// "--host" flag for no reason and does not actually have a database
	// cluster configured.
	- $refs[] = PhabricatorDatabaseRef::getMasterDatabaseRef();
	+ foreach (PhabricatorDatabaseRef::getMasterDatabaseRefs() as $master_ref) {
	+ $refs[] = $master_ref;
	+ }

	foreach ($refs as $possible_ref) {
	if ($possible_ref->getHost() == $host) {
	$ref = $possible_ref;
	break;
	}
	}

	if (!$ref) {
	throw new PhutilArgumentUsageException(
	pht(
	'There is no configured database on host "%s". This command can '.
	'only interact with configured databases.',
	$host));
	}
	} else {
	$ref = PhabricatorDatabaseRef::getMasterDatabaseRef();
	if (!$ref) {
	throw new Exception(
	pht('No database master is configured.'));
	}
	}

	$default_user = $ref->getUser();
	$default_host = $ref->getHost();
	$default_port = $ref->getPort();

	$test_api = id(new PhabricatorStorageManagementAPI())
	->setUser($default_user)
	->setHost($default_host)
	->setPort($default_port)
	->setPassword($ref->getPass())
	->setNamespace($args->getArg('namespace'));

	try {
	queryfx(
	$test_api->getConn(null),
	'SELECT 1');
	} catch (AphrontQueryException $ex) {
	$message = phutil_console_format(
	"%s\n\n%s\n\n%s\n\n%s\n\n%s: %s\n",
	pht('MySQL Credentials Not Configured'),
	pht(
	'Unable to connect to MySQL using the configured credentials. '.
	'You must configure standard credentials before you can upgrade '.
	'storage. Run these commands to set up credentials:'),
	" phabricator/ $ ./bin/config set mysql.host __host__\n".
	" phabricator/ $ ./bin/config set mysql.user __username__\n".
	" phabricator/ $ ./bin/config set mysql.pass __password__",
	pht(
	'These standard credentials are separate from any administrative '.
	'credentials provided to this command with __%s__ or '.
	'__%s__, and must be configured correctly before you can proceed.',
	'--user',
	'--password'),
	pht('Raw MySQL Error'),
	$ex->getMessage());
	echo phutil_console_wrap($message);
	exit(1);
	}

	if ($args->getArg('password') === null) {
	// This is already a PhutilOpaqueEnvelope.
	$password = $ref->getPass();
	} else {
	// Put this in a PhutilOpaqueEnvelope.
	$password = new PhutilOpaqueEnvelope($args->getArg('password'));
	PhabricatorEnv::overrideConfig('mysql.pass', $args->getArg('password'));
	}

	$selected_user = $args->getArg('user');
	if ($selected_user === null) {
	$selected_user = $default_user;
	}

	$api = id(new PhabricatorStorageManagementAPI())
	->setUser($selected_user)
	->setHost($default_host)
	->setPort($default_port)
	->setPassword($password)
	->setNamespace($args->getArg('namespace'))
	->setDisableUTF8MB4($args->getArg('disable-utf8mb4'));
	PhabricatorEnv::overrideConfig('mysql.user', $api->getUser());

	try {
	queryfx(
	$api->getConn(null),
	'SELECT 1');
	} catch (AphrontQueryException $ex) {
	$message = phutil_console_format(
	"%s\n\n%s\n\n%s: %s\n",
	pht('Bad Administrative Credentials'),
	pht(
	'Unable to connect to MySQL using the administrative credentials '.
	'provided with the __%s__ and __%s__ flags. Check that '.
	'you have entered them correctly.',
	'--user',
	'--password'),
	pht('Raw MySQL Error'),
	$ex->getMessage());
	echo phutil_console_wrap($message);
	exit(1);
	}

	$workflows = id(new PhutilClassMapQuery())
	->setAncestorClass('PhabricatorStorageManagementWorkflow')
	->execute();

	$patches = PhabricatorSQLPatchList::buildAllPatches();

	foreach ($workflows as $workflow) {
	$workflow->setAPI($api);
	$workflow->setPatches($patches);
	}

	$workflows[] = new PhutilHelpArgumentWorkflow();

	$args->parseWorkflows($workflows);
	diff --git a/src/applications/config/check/PhabricatorDatabaseSetupCheck.php b/src/applications/config/check/PhabricatorDatabaseSetupCheck.php
	index fd318a5fa1..a0c83d6a73 100644
	--- a/src/applications/config/check/PhabricatorDatabaseSetupCheck.php
	+++ b/src/applications/config/check/PhabricatorDatabaseSetupCheck.php
	@@ -1,130 +1,139 @@
	<?php

	final class PhabricatorDatabaseSetupCheck extends PhabricatorSetupCheck {

	public function getDefaultGroup() {
	return self::GROUP_IMPORTANT;
	}

	public function getExecutionOrder() {
	// This must run after basic PHP checks, but before most other checks.
	return 500;
	}

	protected function executeChecks() {
	- $master = PhabricatorDatabaseRef::getMasterDatabaseRef();
	- if (!$master) {
	+ $host = PhabricatorEnv::getEnvConfig('mysql.host');
	+ $matches = null;
	+ if (preg_match('/^([^:]+):(\d+)$/', $host, $matches)) {
	+ $host = $matches[1];
	+ $port = $matches[2];
	+
	+ $this->newIssue('storage.mysql.hostport')
	+ ->setName(pht('Deprecated mysql.host Format'))
	+ ->setSummary(
	+ pht(
	+ 'Move port information from `%s` to `%s` in your config.',
	+ 'mysql.host',
	+ 'mysql.port'))
	+ ->setMessage(
	+ pht(
	+ 'Your `%s` configuration contains a port number, but this usage '.
	+ 'is deprecated. Instead, put the port number in `%s`.',
	+ 'mysql.host',
	+ 'mysql.port'))
	+ ->addPhabricatorConfig('mysql.host')
	+ ->addPhabricatorConfig('mysql.port')
	+ ->addCommand(
	+ hsprintf(
	+ '<tt>phabricator/ $</tt> ./bin/config set mysql.host %s',
	+ $host))
	+ ->addCommand(
	+ hsprintf(
	+ '<tt>phabricator/ $</tt> ./bin/config set mysql.port %s',
	+ $port));
	+ }
	+
	+ $masters = PhabricatorDatabaseRef::getMasterDatabaseRefs();
	+ if (!$masters) {
	// If we're implicitly in read-only mode during disaster recovery,
	// don't bother with these setup checks.
	return;
	}

	+ foreach ($masters as $master) {
	+ if ($this->checkMasterDatabase($master)) {
	+ break;
	+ }
	+ }
	+ }
	+
	+ private function checkMasterDatabase(PhabricatorDatabaseRef $master) {
	$conn_raw = $master->newManagementConnection();

	try {
	queryfx($conn_raw, 'SELECT 1');
	$database_exception = null;
	} catch (AphrontInvalidCredentialsQueryException $ex) {
	$database_exception = $ex;
	} catch (AphrontConnectionQueryException $ex) {
	$database_exception = $ex;
	}

	if ($database_exception) {
	$issue = PhabricatorSetupIssue::newDatabaseConnectionIssue(
	$database_exception);
	$this->addIssue($issue);
	- return;
	+ return true;
	}

	$engines = queryfx_all($conn_raw, 'SHOW ENGINES');
	$engines = ipull($engines, 'Support', 'Engine');

	$innodb = idx($engines, 'InnoDB');
	if ($innodb != 'YES' && $innodb != 'DEFAULT') {
	$message = pht(
	"The 'InnoDB' engine is not available in MySQL. Enable InnoDB in ".
	"your MySQL configuration.".
	"\n\n".
	"(If you aleady created tables, MySQL incorrectly used some other ".
	"engine to create them. You need to convert them or drop and ".
	"reinitialize them.)");

	$this->newIssue('mysql.innodb')
	->setName(pht('MySQL InnoDB Engine Not Available'))
	->setMessage($message)
	->setIsFatal(true);
	- return;
	+ return true;
	}

	$namespace = PhabricatorEnv::getEnvConfig('storage.default-namespace');

	$databases = queryfx_all($conn_raw, 'SHOW DATABASES');
	$databases = ipull($databases, 'Database', 'Database');

	if (empty($databases[$namespace.'_meta_data'])) {
	$message = pht(
	"Run the storage upgrade script to setup Phabricator's database ".
	"schema.");

	$this->newIssue('storage.upgrade')
	->setName(pht('Setup MySQL Schema'))
	->setMessage($message)
	->setIsFatal(true)
	->addCommand(hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
	- } else {
	- $conn_meta = $master->newApplicationConnection(
	- $namespace.'_meta_data');
	-
	- $applied = queryfx_all($conn_meta, 'SELECT patch FROM patch_status');
	- $applied = ipull($applied, 'patch', 'patch');
	-
	- $all = PhabricatorSQLPatchList::buildAllPatches();
	- $diff = array_diff_key($all, $applied);
	-
	- if ($diff) {
	- $this->newIssue('storage.patch')
	- ->setName(pht('Upgrade MySQL Schema'))
	- ->setMessage(
	- pht(
	- "Run the storage upgrade script to upgrade Phabricator's ".
	- "database schema. Missing patches:<br />%s<br />",
	- phutil_implode_html(phutil_tag('br'), array_keys($diff))))
	- ->addCommand(
	- hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
	- }
	+ return true;
	}

	- $host = PhabricatorEnv::getEnvConfig('mysql.host');
	- $matches = null;
	- if (preg_match('/^([^:]+):(\d+)$/', $host, $matches)) {
	- $host = $matches[1];
	- $port = $matches[2];
	+ $conn_meta = $master->newApplicationConnection(
	+ $namespace.'_meta_data');

	- $this->newIssue('storage.mysql.hostport')
	- ->setName(pht('Deprecated mysql.host Format'))
	- ->setSummary(
	- pht(
	- 'Move port information from `%s` to `%s` in your config.',
	- 'mysql.host',
	- 'mysql.port'))
	+ $applied = queryfx_all($conn_meta, 'SELECT patch FROM patch_status');
	+ $applied = ipull($applied, 'patch', 'patch');
	+
	+ $all = PhabricatorSQLPatchList::buildAllPatches();
	+ $diff = array_diff_key($all, $applied);
	+
	+ if ($diff) {
	+ $this->newIssue('storage.patch')
	+ ->setName(pht('Upgrade MySQL Schema'))
	->setMessage(
	pht(
	- 'Your `%s` configuration contains a port number, but this usage '.
	- 'is deprecated. Instead, put the port number in `%s`.',
	- 'mysql.host',
	- 'mysql.port'))
	- ->addPhabricatorConfig('mysql.host')
	- ->addPhabricatorConfig('mysql.port')
	+ "Run the storage upgrade script to upgrade Phabricator's ".
	+ "database schema. Missing patches:<br />%s<br />",
	+ phutil_implode_html(phutil_tag('br'), array_keys($diff))))
	->addCommand(
	- hsprintf(
	- '<tt>phabricator/ $</tt> ./bin/config set mysql.host %s',
	- $host))
	- ->addCommand(
	- hsprintf(
	- '<tt>phabricator/ $</tt> ./bin/config set mysql.port %s',
	- $port));
	+ hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
	+ return true;
	}
	-
	}
	}
	diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
	index 85df807f3e..003ddb6beb 100644
	--- a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
	+++ b/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
	@@ -1,98 +1,90 @@
	<?php

	final class PhabricatorClusterDatabasesConfigOptionType
	extends PhabricatorConfigJSONOptionType {

	public function validateOption(PhabricatorConfigOption $option, $value) {
	if (!is_array($value)) {
	throw new Exception(
	pht(
	'Database cluster configuration is not valid: value must be a '.
	'list of database hosts.'));
	}

	foreach ($value as $index => $spec) {
	if (!is_array($spec)) {
	throw new Exception(
	pht(
	'Database cluster configuration is not valid: each entry in the '.
	'list must be a dictionary describing a database host, but '.
	'the value with index "%s" is not a dictionary.',
	$index));
	}
	}

	$masters = array();
	$map = array();
	foreach ($value as $index => $spec) {
	try {
	PhutilTypeSpec::checkMap(
	$spec,
	array(
	'host' => 'string',
	'role' => 'string',
	'port' => 'optional int',
	'user' => 'optional string',
	'pass' => 'optional string',
	'disabled' => 'optional bool',
	));
	} catch (Exception $ex) {
	throw new Exception(
	pht(
	'Database cluster configuration has an invalid host '.
	'specification (at index "%s"): %s.',
	$index,
	$ex->getMessage()));
	}

	$role = $spec['role'];
	$host = $spec['host'];
	$port = idx($spec, 'port');

	switch ($role) {
	case 'master':
	case 'replica':
	break;
	default:
	throw new Exception(
	pht(
	'Database cluster configuration describes an invalid '.
	'host ("%s", at index "%s") with an unrecognized role ("%s"). '.
	'Valid roles are "%s" or "%s".',
	$spec['host'],
	$index,
	$spec['role'],
	'master',
	'replica'));
	}

	if ($role === 'master') {
	$masters[] = $host;
	}

	// We can't guarantee that you didn't just give the same host two
	// different names in DNS, but this check can catch silly copy/paste
	// mistakes.
	$key = "{$host}:{$port}";
	if (isset($map[$key])) {
	throw new Exception(
	pht(
	'Database cluster configuration is invalid: it describes the '.
	'same host ("%s") multiple times. Each host should appear only '.
	'once in the list.',
	$host));
	}
	$map[$key] = true;
	}

	- if (count($masters) > 1) {
	- throw new Exception(
	- pht(
	- 'Database cluster configuration is invalid: it describes multiple '.
	- 'masters. No more than one host may be a master. Hosts currently '.
	- 'configured as masters: %s.',
	- implode(', ', $masters)));
	- }
	}

	}
	diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	index c81de56823..2cab28104f 100644
	--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	@@ -1,538 +1,563 @@
	<?php

	final class PhabricatorDatabaseRef
	extends Phobject {

	const STATUS_OKAY = 'okay';
	const STATUS_FAIL = 'fail';
	const STATUS_AUTH = 'auth';
	const STATUS_REPLICATION_CLIENT = 'replication-client';

	const REPLICATION_OKAY = 'okay';
	const REPLICATION_MASTER_REPLICA = 'master-replica';
	const REPLICATION_REPLICA_NONE = 'replica-none';
	const REPLICATION_SLOW = 'replica-slow';
	const REPLICATION_NOT_REPLICATING = 'not-replicating';

	const KEY_REFS = 'cluster.db.refs';
	const KEY_INDIVIDUAL = 'cluster.db.individual';

	private $host;
	private $port;
	private $user;
	private $pass;
	private $disabled;
	private $isMaster;
	private $isIndividual;

	private $connectionLatency;
	private $connectionStatus;
	private $connectionMessage;

	private $replicaStatus;
	private $replicaMessage;
	private $replicaDelay;

	private $healthRecord;
	private $didFailToConnect;

	public function setHost($host) {
	$this->host = $host;
	return $this;
	}

	public function getHost() {
	return $this->host;
	}

	public function setPort($port) {
	$this->port = $port;
	return $this;
	}

	public function getPort() {
	return $this->port;
	}

	public function setUser($user) {
	$this->user = $user;
	return $this;
	}

	public function getUser() {
	return $this->user;
	}

	public function setPass(PhutilOpaqueEnvelope $pass) {
	$this->pass = $pass;
	return $this;
	}

	public function getPass() {
	return $this->pass;
	}

	public function setIsMaster($is_master) {
	$this->isMaster = $is_master;
	return $this;
	}

	public function getIsMaster() {
	return $this->isMaster;
	}

	public function setDisabled($disabled) {
	$this->disabled = $disabled;
	return $this;
	}

	public function getDisabled() {
	return $this->disabled;
	}

	public function setConnectionLatency($connection_latency) {
	$this->connectionLatency = $connection_latency;
	return $this;
	}

	public function getConnectionLatency() {
	return $this->connectionLatency;
	}

	public function setConnectionStatus($connection_status) {
	$this->connectionStatus = $connection_status;
	return $this;
	}

	public function getConnectionStatus() {
	if ($this->connectionStatus === null) {
	throw new PhutilInvalidStateException('queryAll');
	}

	return $this->connectionStatus;
	}

	public function setConnectionMessage($connection_message) {
	$this->connectionMessage = $connection_message;
	return $this;
	}

	public function getConnectionMessage() {
	return $this->connectionMessage;
	}

	public function setReplicaStatus($replica_status) {
	$this->replicaStatus = $replica_status;
	return $this;
	}

	public function getReplicaStatus() {
	return $this->replicaStatus;
	}

	public function setReplicaMessage($replica_message) {
	$this->replicaMessage = $replica_message;
	return $this;
	}

	public function getReplicaMessage() {
	return $this->replicaMessage;
	}

	public function setReplicaDelay($replica_delay) {
	$this->replicaDelay = $replica_delay;
	return $this;
	}

	public function getReplicaDelay() {
	return $this->replicaDelay;
	}

	public function setIsIndividual($is_individual) {
	$this->isIndividual = $is_individual;
	return $this;
	}

	public function getIsIndividual() {
	return $this->isIndividual;
	}

	public static function getConnectionStatusMap() {
	return array(
	self::STATUS_OKAY => array(
	'icon' => 'fa-exchange',
	'color' => 'green',
	'label' => pht('Okay'),
	),
	self::STATUS_FAIL => array(
	'icon' => 'fa-times',
	'color' => 'red',
	'label' => pht('Failed'),
	),
	self::STATUS_AUTH => array(
	'icon' => 'fa-key',
	'color' => 'red',
	'label' => pht('Invalid Credentials'),
	),
	self::STATUS_REPLICATION_CLIENT => array(
	'icon' => 'fa-eye-slash',
	'color' => 'yellow',
	'label' => pht('Missing Permission'),
	),
	);
	}

	public static function getReplicaStatusMap() {
	return array(
	self::REPLICATION_OKAY => array(
	'icon' => 'fa-download',
	'color' => 'green',
	'label' => pht('Okay'),
	),
	self::REPLICATION_MASTER_REPLICA => array(
	'icon' => 'fa-database',
	'color' => 'red',
	'label' => pht('Replicating Master'),
	),
	self::REPLICATION_REPLICA_NONE => array(
	'icon' => 'fa-download',
	'color' => 'red',
	'label' => pht('Not A Replica'),
	),
	self::REPLICATION_SLOW => array(
	'icon' => 'fa-hourglass',
	'color' => 'red',
	'label' => pht('Slow Replication'),
	),
	self::REPLICATION_NOT_REPLICATING => array(
	'icon' => 'fa-exclamation-triangle',
	'color' => 'red',
	'label' => pht('Not Replicating'),
	),
	);
	}

	public static function getLiveRefs() {
	$cache = PhabricatorCaches::getRequestCache();

	$refs = $cache->getKey(self::KEY_REFS);
	if (!$refs) {
	$refs = self::newRefs();
	$cache->setKey(self::KEY_REFS, $refs);
	}

	return $refs;
	}

	public static function getLiveIndividualRef() {
	$cache = PhabricatorCaches::getRequestCache();

	$ref = $cache->getKey(self::KEY_INDIVIDUAL);
	if (!$ref) {
	$ref = self::newIndividualRef();
	$cache->setKey(self::KEY_INDIVIDUAL, $ref);
	}

	return $ref;
	}

	public static function newRefs() {
	$refs = array();

	$default_port = PhabricatorEnv::getEnvConfig('mysql.port');
	$default_port = nonempty($default_port, 3306);

	$default_user = PhabricatorEnv::getEnvConfig('mysql.user');

	$default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
	$default_pass = new PhutilOpaqueEnvelope($default_pass);

	$config = PhabricatorEnv::getEnvConfig('cluster.databases');
	foreach ($config as $server) {
	$host = $server['host'];
	$port = idx($server, 'port', $default_port);
	$user = idx($server, 'user', $default_user);
	$disabled = idx($server, 'disabled', false);

	$pass = idx($server, 'pass');
	if ($pass) {
	$pass = new PhutilOpaqueEnvelope($pass);
	} else {
	$pass = clone $default_pass;
	}

	$role = $server['role'];

	$ref = id(new self())
	->setHost($host)
	->setPort($port)
	->setUser($user)
	->setPass($pass)
	->setDisabled($disabled)
	->setIsMaster(($role == 'master'));

	$refs[] = $ref;
	}

	return $refs;
	}

	public static function queryAll() {
	$refs = self::newRefs();

	foreach ($refs as $ref) {
	if ($ref->getDisabled()) {
	continue;
	}

	$conn = $ref->newManagementConnection();

	$t_start = microtime(true);
	$replica_status = false;
	try {
	$replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
	$ref->setConnectionStatus(self::STATUS_OKAY);
	} catch (AphrontAccessDeniedQueryException $ex) {
	$ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
	$ref->setConnectionMessage(
	pht(
	'No permission to run "SHOW SLAVE STATUS". Grant this user '.
	'"REPLICATION CLIENT" permission to allow Phabricator to '.
	'monitor replica health.'));
	} catch (AphrontInvalidCredentialsQueryException $ex) {
	$ref->setConnectionStatus(self::STATUS_AUTH);
	$ref->setConnectionMessage($ex->getMessage());
	} catch (AphrontQueryException $ex) {
	$ref->setConnectionStatus(self::STATUS_FAIL);

	$class = get_class($ex);
	$message = $ex->getMessage();
	$ref->setConnectionMessage(
	pht(
	'%s: %s',
	get_class($ex),
	$ex->getMessage()));
	}
	$t_end = microtime(true);
	$ref->setConnectionLatency($t_end - $t_start);

	if ($replica_status !== false) {
	$is_replica = (bool)$replica_status;
	if ($ref->getIsMaster() && $is_replica) {
	$ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
	$ref->setReplicaMessage(
	pht(
	'This host has a "master" role, but is replicating data from '.
	'another host ("%s")!',
	idx($replica_status, 'Master_Host')));
	} else if (!$ref->getIsMaster() && !$is_replica) {
	$ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
	$ref->setReplicaMessage(
	pht(
	'This host has a "replica" role, but is not replicating data '.
	'from a master (no output from "SHOW SLAVE STATUS").'));
	} else {
	$ref->setReplicaStatus(self::REPLICATION_OKAY);
	}

	if ($is_replica) {
	$latency = idx($replica_status, 'Seconds_Behind_Master');
	if (!strlen($latency)) {
	$ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING);
	} else {
	$latency = (int)$latency;
	$ref->setReplicaDelay($latency);
	if ($latency > 30) {
	$ref->setReplicaStatus(self::REPLICATION_SLOW);
	$ref->setReplicaMessage(
	pht(
	'This replica is lagging far behind the master. Data is at '.
	'risk!'));
	}
	}
	}
	}
	}

	return $refs;
	}

	public function newManagementConnection() {
	return $this->newConnection(
	array(
	'retries' => 0,
	'timeout' => 2,
	));
	}

	public function newApplicationConnection($database) {
	return $this->newConnection(
	array(
	'database' => $database,
	));
	}

	public function isSevered() {
	// If we only have an individual database, never sever our connection to
	// it, at least for now. It's possible that using the same severing rules
	// might eventually make sense to help alleviate load-related failures,
	// but we should wait for all the cluster stuff to stabilize first.
	if ($this->getIsIndividual()) {
	return false;
	}

	if ($this->didFailToConnect) {
	return true;
	}

	$record = $this->getHealthRecord();
	$is_healthy = $record->getIsHealthy();
	if (!$is_healthy) {
	return true;
	}

	return false;
	}

	public function isReachable(AphrontDatabaseConnection $connection) {
	$record = $this->getHealthRecord();
	$should_check = $record->getShouldCheck();

	if ($this->isSevered() && !$should_check) {
	return false;
	}

	try {
	$connection->openConnection();
	$reachable = true;
	} catch (AphrontSchemaQueryException $ex) {
	// We get one of these if the database we're trying to select does not
	// exist. In this case, just re-throw the exception. This is expected
	// during first-time setup, when databases like "config" will not exist
	// yet.
	throw $ex;
	} catch (Exception $ex) {
	$reachable = false;
	}

	if ($should_check) {
	$record->didHealthCheck($reachable);
	}

	if (!$reachable) {
	$this->didFailToConnect = true;
	}

	return $reachable;
	}

	public function checkHealth() {
	$health = $this->getHealthRecord();

	$should_check = $health->getShouldCheck();
	if ($should_check) {
	// This does an implicit health update.
	$connection = $this->newManagementConnection();
	$this->isReachable($connection);
	}

	return $this;
	}

	public function getHealthRecord() {
	if (!$this->healthRecord) {
	$this->healthRecord = new PhabricatorDatabaseHealthRecord($this);
	}
	return $this->healthRecord;
	}

	- public static function getMasterDatabaseRef() {
	+ public static function getMasterDatabaseRefs() {
	$refs = self::getLiveRefs();

	if (!$refs) {
	- return self::getLiveIndividualRef();
	+ return array(self::getLiveIndividualRef());
	}

	- $master = null;
	+ $masters = array();
	foreach ($refs as $ref) {
	if ($ref->getDisabled()) {
	continue;
	}
	if ($ref->getIsMaster()) {
	- return $ref;
	+ $masters[] = $ref;
	}
	}

	- return null;
	+ return $masters;
	+ }
	+
	+ public static function getMasterDatabaseRef() {
	+ // TODO: Remove this method; it no longer makes sense with application
	+ // partitioning.
	+
	+ return head(self::getMasterDatabaseRefs());
	+ }
	+
	+ public static function getMasterDatabaseRefForDatabase($database) {
	+ $masters = self::getMasterDatabaseRefs();
	+
	+ // TODO: Actually implement this.
	+
	+ return head($masters);
	}

	public static function newIndividualRef() {
	$conf = PhabricatorEnv::newObjectFromConfig(
	'mysql.configuration-provider',
	array(null, 'w', null));

	return id(new self())
	->setHost($conf->getHost())
	->setPort($conf->getPort())
	->setUser($conf->getUser())
	->setPass($conf->getPassword())
	->setIsIndividual(true)
	->setIsMaster(true);
	}

	- public static function getReplicaDatabaseRef() {
	+ public static function getReplicaDatabaseRefs() {
	$refs = self::getLiveRefs();

	if (!$refs) {
	- return null;
	+ return array();
	}

	- // TODO: We may have multiple replicas to choose from, and could make
	- // more of an effort to pick the "best" one here instead of always
	- // picking the first one. Once we've picked one, we should try to use
	- // the same replica for the rest of the request, though.
	-
	+ $replicas = array();
	foreach ($refs as $ref) {
	if ($ref->getDisabled()) {
	continue;
	}
	if ($ref->getIsMaster()) {
	continue;
	}
	- return $ref;
	+
	+ $replicas[] = $ref;
	}

	- return null;
	+ return $replicas;
	+ }
	+
	+ public static function getReplicaDatabaseRefForDatabase($database) {
	+ $replicas = self::getReplicaDatabaseRefs();
	+
	+ // TODO: Actually implement this.
	+
	+ // TODO: We may have multiple replicas to choose from, and could make
	+ // more of an effort to pick the "best" one here instead of always
	+ // picking the first one. Once we've picked one, we should try to use
	+ // the same replica for the rest of the request, though.
	+
	+ return head($replicas);
	}

	private function newConnection(array $options) {
	// If we believe the database is unhealthy, don't spend as much time
	// trying to connect to it, since it's likely to continue to fail and
	// hammering it can only make the problem worse.
	$record = $this->getHealthRecord();
	if ($record->getIsHealthy()) {
	$default_retries = 3;
	$default_timeout = 10;
	} else {
	$default_retries = 0;
	$default_timeout = 2;
	}

	$spec = $options + array(
	'user' => $this->getUser(),
	'pass' => $this->getPass(),
	'host' => $this->getHost(),
	'port' => $this->getPort(),
	'database' => null,
	'retries' => $default_retries,
	'timeout' => $default_timeout,
	);

	return PhabricatorEnv::newObjectFromConfig(
	'mysql.implementation',
	array(
	$spec,
	));
	}

	}
	diff --git a/src/infrastructure/env/PhabricatorEnv.php b/src/infrastructure/env/PhabricatorEnv.php
	index 31fd116688..58d6a7cc3d 100644
	--- a/src/infrastructure/env/PhabricatorEnv.php
	+++ b/src/infrastructure/env/PhabricatorEnv.php
	@@ -1,918 +1,929 @@
	<?php

	/**
	* Manages the execution environment configuration, exposing APIs to read
	* configuration settings and other similar values that are derived directly
	* from configuration settings.
	*
	*
	* = Reading Configuration =
	*
	* The primary role of this class is to provide an API for reading
	* Phabricator configuration, @{method:getEnvConfig}:
	*
	* $value = PhabricatorEnv::getEnvConfig('some.key', $default);
	*
	* The class also handles some URI construction based on configuration, via
	* the methods @{method:getURI}, @{method:getProductionURI},
	* @{method:getCDNURI}, and @{method:getDoclink}.
	*
	* For configuration which allows you to choose a class to be responsible for
	* some functionality (e.g., which mail adapter to use to deliver email),
	* @{method:newObjectFromConfig} provides a simple interface that validates
	* the configured value.
	*
	*
	* = Unit Test Support =
	*
	* In unit tests, you can use @{method:beginScopedEnv} to create a temporary,
	* mutable environment. The method returns a scope guard object which restores
	* the environment when it is destroyed. For example:
	*
	* public function testExample() {
	* $env = PhabricatorEnv::beginScopedEnv();
	* $env->overrideEnv('some.key', 'new-value-for-this-test');
	*
	* // Some test which depends on the value of 'some.key'.
	*
	* }
	*
	* Your changes will persist until the `$env` object leaves scope or is
	* destroyed.
	*
	* You should //not// use this in normal code.
	*
	*
	* @task read Reading Configuration
	* @task uri URI Validation
	* @task test Unit Test Support
	* @task internal Internals
	*/
	final class PhabricatorEnv extends Phobject {

	private static $sourceStack;
	private static $repairSource;
	private static $overrideSource;
	private static $requestBaseURI;
	private static $cache;
	private static $localeCode;
	private static $readOnly;
	private static $readOnlyReason;

	const READONLY_CONFIG = 'config';
	const READONLY_UNREACHABLE = 'unreachable';
	const READONLY_SEVERED = 'severed';
	const READONLY_MASTERLESS = 'masterless';

	/**
	* @phutil-external-symbol class PhabricatorStartup
	*/
	public static function initializeWebEnvironment() {
	self::initializeCommonEnvironment(false);
	}

	public static function initializeScriptEnvironment($config_optional) {
	self::initializeCommonEnvironment($config_optional);

	// NOTE: This is dangerous in general, but we know we're in a script context
	// and are not vulnerable to CSRF.
	AphrontWriteGuard::allowDangerousUnguardedWrites(true);

	// There are several places where we log information (about errors, events,
	// service calls, etc.) for analysis via DarkConsole or similar. These are
	// useful for web requests, but grow unboundedly in long-running scripts and
	// daemons. Discard data as it arrives in these cases.
	PhutilServiceProfiler::getInstance()->enableDiscardMode();
	DarkConsoleErrorLogPluginAPI::enableDiscardMode();
	DarkConsoleEventPluginAPI::enableDiscardMode();
	}


	private static function initializeCommonEnvironment($config_optional) {
	PhutilErrorHandler::initialize();

	self::resetUmask();
	self::buildConfigurationSourceStack($config_optional);

	// Force a valid timezone. If both PHP and Phabricator configuration are
	// invalid, use UTC.
	$tz = self::getEnvConfig('phabricator.timezone');
	if ($tz) {
	@date_default_timezone_set($tz);
	}
	$ok = @date_default_timezone_set(date_default_timezone_get());
	if (!$ok) {
	date_default_timezone_set('UTC');
	}

	// Prepend '/support/bin' and append any paths to $PATH if we need to.
	$env_path = getenv('PATH');
	$phabricator_path = dirname(phutil_get_library_root('phabricator'));
	$support_path = $phabricator_path.'/support/bin';
	$env_path = $support_path.PATH_SEPARATOR.$env_path;
	$append_dirs = self::getEnvConfig('environment.append-paths');
	if (!empty($append_dirs)) {
	$append_path = implode(PATH_SEPARATOR, $append_dirs);
	$env_path = $env_path.PATH_SEPARATOR.$append_path;
	}
	putenv('PATH='.$env_path);

	// Write this back into $_ENV, too, so ExecFuture picks it up when creating
	// subprocess environments.
	$_ENV['PATH'] = $env_path;


	// If an instance identifier is defined, write it into the environment so
	// it's available to subprocesses.
	$instance = self::getEnvConfig('cluster.instance');
	if (strlen($instance)) {
	putenv('PHABRICATOR_INSTANCE='.$instance);
	$_ENV['PHABRICATOR_INSTANCE'] = $instance;
	}

	PhabricatorEventEngine::initialize();

	// TODO: Add a "locale.default" config option once we have some reasonable
	// defaults which aren't silly nonsense.
	self::setLocaleCode('en_US');
	}

	public static function beginScopedLocale($locale_code) {
	return new PhabricatorLocaleScopeGuard($locale_code);
	}

	public static function getLocaleCode() {
	return self::$localeCode;
	}

	public static function setLocaleCode($locale_code) {
	if (!$locale_code) {
	return;
	}

	if ($locale_code == self::$localeCode) {
	return;
	}

	try {
	$locale = PhutilLocale::loadLocale($locale_code);
	$translations = PhutilTranslation::getTranslationMapForLocale(
	$locale_code);

	$override = self::getEnvConfig('translation.override');
	if (!is_array($override)) {
	$override = array();
	}

	PhutilTranslator::getInstance()
	->setLocale($locale)
	->setTranslations($override + $translations);

	self::$localeCode = $locale_code;
	} catch (Exception $ex) {
	// Just ignore this; the user likely has an out-of-date locale code.
	}
	}

	private static function buildConfigurationSourceStack($config_optional) {
	self::dropConfigCache();

	$stack = new PhabricatorConfigStackSource();
	self::$sourceStack = $stack;

	$default_source = id(new PhabricatorConfigDefaultSource())
	->setName(pht('Global Default'));
	$stack->pushSource($default_source);

	$env = self::getSelectedEnvironmentName();
	if ($env) {
	$stack->pushSource(
	id(new PhabricatorConfigFileSource($env))
	->setName(pht("File '%s'", $env)));
	}

	$stack->pushSource(
	id(new PhabricatorConfigLocalSource())
	->setName(pht('Local Config')));

	// If the install overrides the database adapter, we might need to load
	// the database adapter class before we can push on the database config.
	// This config is locked and can't be edited from the web UI anyway.
	foreach (self::getEnvConfig('load-libraries') as $library) {
	phutil_load_library($library);
	}

	// Drop any class map caches, since they will have generated without
	// any classes from libraries. Without this, preflight setup checks can
	// cause generation of a setup check cache that omits checks defined in
	// libraries, for example.
	PhutilClassMapQuery::deleteCaches();

	// If custom libraries specify config options, they won't get default
	// values as the Default source has already been loaded, so we get it to
	// pull in all options from non-phabricator libraries now they are loaded.
	$default_source->loadExternalOptions();

	// If this install has site config sources, load them now.
	$site_sources = id(new PhutilClassMapQuery())
	->setAncestorClass('PhabricatorConfigSiteSource')
	->setSortMethod('getPriority')
	->execute();

	foreach ($site_sources as $site_source) {
	$stack->pushSource($site_source);
	}

	- $master = PhabricatorDatabaseRef::getMasterDatabaseRef();
	- if (!$master) {
	+ $masters = PhabricatorDatabaseRef::getMasterDatabaseRefs();
	+ if (!$masters) {
	self::setReadOnly(true, self::READONLY_MASTERLESS);
	- } else if ($master->isSevered()) {
	- $master->checkHealth();
	- if ($master->isSevered()) {
	- self::setReadOnly(true, self::READONLY_SEVERED);
	+ } else {
	+ // If any master is severed, we drop to readonly mode. In theory we
	+ // could try to continue if we're only missing some applications, but
	+ // this is very complex and we're unlikely to get it right.
	+
	+ foreach ($masters as $master) {
	+ // Give severed masters one last chance to get healthy.
	+ if ($master->isSevered()) {
	+ $master->checkHealth();
	+ }
	+
	+ if ($master->isSevered()) {
	+ self::setReadOnly(true, self::READONLY_SEVERED);
	+ break;
	+ }
	}
	}

	try {
	$stack->pushSource(
	id(new PhabricatorConfigDatabaseSource('default'))
	->setName(pht('Database')));
	} catch (AphrontSchemaQueryException $exception) {
	// If the database is not available, just skip this configuration
	// source. This happens during `bin/storage upgrade`, `bin/conf` before
	// schema setup, etc.
	} catch (AphrontConnectionQueryException $ex) {
	if (!$config_optional) {
	throw $ex;
	}
	} catch (AphrontInvalidCredentialsQueryException $ex) {
	if (!$config_optional) {
	throw $ex;
	}
	}
	}

	public static function repairConfig($key, $value) {
	if (!self::$repairSource) {
	self::$repairSource = id(new PhabricatorConfigDictionarySource(array()))
	->setName(pht('Repaired Config'));
	self::$sourceStack->pushSource(self::$repairSource);
	}
	self::$repairSource->setKeys(array($key => $value));
	self::dropConfigCache();
	}

	public static function overrideConfig($key, $value) {
	if (!self::$overrideSource) {
	self::$overrideSource = id(new PhabricatorConfigDictionarySource(array()))
	->setName(pht('Overridden Config'));
	self::$sourceStack->pushSource(self::$overrideSource);
	}
	self::$overrideSource->setKeys(array($key => $value));
	self::dropConfigCache();
	}

	public static function getUnrepairedEnvConfig($key, $default = null) {
	foreach (self::$sourceStack->getStack() as $source) {
	if ($source === self::$repairSource) {
	continue;
	}
	$result = $source->getKeys(array($key));
	if ($result) {
	return $result[$key];
	}
	}
	return $default;
	}

	public static function getSelectedEnvironmentName() {
	$env_var = 'PHABRICATOR_ENV';

	$env = idx($_SERVER, $env_var);

	if (!$env) {
	$env = getenv($env_var);
	}

	if (!$env) {
	$env = idx($_ENV, $env_var);
	}

	if (!$env) {
	$root = dirname(phutil_get_library_root('phabricator'));
	$path = $root.'/conf/local/ENVIRONMENT';
	if (Filesystem::pathExists($path)) {
	$env = trim(Filesystem::readFile($path));
	}
	}

	return $env;
	}


	/* -( Reading Configuration )---------------------------------------------- */


	/**
	* Get the current configuration setting for a given key.
	*
	* If the key is not found, then throw an Exception.
	*
	* @task read
	*/
	public static function getEnvConfig($key) {
	if (!self::$sourceStack) {
	throw new Exception(
	pht(
	'Trying to read configuration "%s" before configuration has been '.
	'initialized.',
	$key));
	}

	if (isset(self::$cache[$key])) {
	return self::$cache[$key];
	}

	if (array_key_exists($key, self::$cache)) {
	return self::$cache[$key];
	}

	$result = self::$sourceStack->getKeys(array($key));
	if (array_key_exists($key, $result)) {
	self::$cache[$key] = $result[$key];
	return $result[$key];
	} else {
	throw new Exception(
	pht(
	"No config value specified for key '%s'.",
	$key));
	}
	}

	/**
	* Get the current configuration setting for a given key. If the key
	* does not exist, return a default value instead of throwing. This is
	* primarily useful for migrations involving keys which are slated for
	* removal.
	*
	* @task read
	*/
	public static function getEnvConfigIfExists($key, $default = null) {
	try {
	return self::getEnvConfig($key);
	} catch (Exception $ex) {
	return $default;
	}
	}


	/**
	* Get the fully-qualified URI for a path.
	*
	* @task read
	*/
	public static function getURI($path) {
	return rtrim(self::getAnyBaseURI(), '/').$path;
	}


	/**
	* Get the fully-qualified production URI for a path.
	*
	* @task read
	*/
	public static function getProductionURI($path) {
	// If we're passed a URI which already has a domain, simply return it
	// unmodified. In particular, files may have URIs which point to a CDN
	// domain.
	$uri = new PhutilURI($path);
	if ($uri->getDomain()) {
	return $path;
	}

	$production_domain = self::getEnvConfig('phabricator.production-uri');
	if (!$production_domain) {
	$production_domain = self::getAnyBaseURI();
	}
	return rtrim($production_domain, '/').$path;
	}

	public static function getAllowedURIs($path) {
	$uri = new PhutilURI($path);
	if ($uri->getDomain()) {
	return $path;
	}

	$allowed_uris = self::getEnvConfig('phabricator.allowed-uris');
	$return = array();
	foreach ($allowed_uris as $allowed_uri) {
	$return[] = rtrim($allowed_uri, '/').$path;
	}

	return $return;
	}


	/**
	* Get the fully-qualified production URI for a static resource path.
	*
	* @task read
	*/
	public static function getCDNURI($path) {
	$alt = self::getEnvConfig('security.alternate-file-domain');
	if (!$alt) {
	$alt = self::getAnyBaseURI();
	}
	$uri = new PhutilURI($alt);
	$uri->setPath($path);
	return (string)$uri;
	}


	/**
	* Get the fully-qualified production URI for a documentation resource.
	*
	* @task read
	*/
	public static function getDoclink($resource, $type = 'article') {
	$uri = new PhutilURI('https://secure.phabricator.com/diviner/find/');
	$uri->setQueryParam('name', $resource);
	$uri->setQueryParam('type', $type);
	$uri->setQueryParam('jump', true);
	return (string)$uri;
	}


	/**
	* Build a concrete object from a configuration key.
	*
	* @task read
	*/
	public static function newObjectFromConfig($key, $args = array()) {
	$class = self::getEnvConfig($key);
	return newv($class, $args);
	}

	public static function getAnyBaseURI() {
	$base_uri = self::getEnvConfig('phabricator.base-uri');

	if (!$base_uri) {
	$base_uri = self::getRequestBaseURI();
	}

	if (!$base_uri) {
	throw new Exception(
	pht(
	"Define '%s' in your configuration to continue.",
	'phabricator.base-uri'));
	}

	return $base_uri;
	}

	public static function getRequestBaseURI() {
	return self::$requestBaseURI;
	}

	public static function setRequestBaseURI($uri) {
	self::$requestBaseURI = $uri;
	}

	public static function isReadOnly() {
	if (self::$readOnly !== null) {
	return self::$readOnly;
	}
	return self::getEnvConfig('cluster.read-only');
	}

	public static function setReadOnly($read_only, $reason) {
	self::$readOnly = $read_only;
	self::$readOnlyReason = $reason;
	}

	public static function getReadOnlyMessage() {
	$reason = self::getReadOnlyReason();
	switch ($reason) {
	case self::READONLY_MASTERLESS:
	return pht(
	'Phabricator is in read-only mode (no writable database '.
	'is configured).');
	case self::READONLY_UNREACHABLE:
	return pht(
	'Phabricator is in read-only mode (unreachable master).');
	case self::READONLY_SEVERED:
	return pht(
	'Phabricator is in read-only mode (major interruption).');
	}

	return pht('Phabricator is in read-only mode.');
	}

	public static function getReadOnlyURI() {
	return urisprintf(
	'/readonly/%s/',
	self::getReadOnlyReason());
	}

	public static function getReadOnlyReason() {
	if (!self::isReadOnly()) {
	return null;
	}

	if (self::$readOnlyReason !== null) {
	return self::$readOnlyReason;
	}

	return self::READONLY_CONFIG;
	}


	/* -( Unit Test Support )-------------------------------------------------- */


	/**
	* @task test
	*/
	public static function beginScopedEnv() {
	return new PhabricatorScopedEnv(self::pushTestEnvironment());
	}


	/**
	* @task test
	*/
	private static function pushTestEnvironment() {
	self::dropConfigCache();
	$source = new PhabricatorConfigDictionarySource(array());
	self::$sourceStack->pushSource($source);
	return spl_object_hash($source);
	}


	/**
	* @task test
	*/
	public static function popTestEnvironment($key) {
	self::dropConfigCache();
	$source = self::$sourceStack->popSource();
	$stack_key = spl_object_hash($source);
	if ($stack_key !== $key) {
	self::$sourceStack->pushSource($source);
	throw new Exception(
	pht(
	'Scoped environments were destroyed in a different order than they '.
	'were initialized.'));
	}
	}


	/* -( URI Validation )----------------------------------------------------- */


	/**
	* Detect if a URI satisfies either @{method:isValidLocalURIForLink} or
	* @{method:isValidRemoteURIForLink}, i.e. is a page on this server or the
	* URI of some other resource which has a valid protocol. This rejects
	* garbage URIs and URIs with protocols which do not appear in the
	* `uri.allowed-protocols` configuration, notably 'javascript:' URIs.
	*
	* NOTE: This method is generally intended to reject URIs which it may be
	* unsafe to put in an "href" link attribute.
	*
	* @param string URI to test.
	* @return bool True if the URI identifies a web resource.
	* @task uri
	*/
	public static function isValidURIForLink($uri) {
	return self::isValidLocalURIForLink($uri) \|\|
	self::isValidRemoteURIForLink($uri);
	}


	/**
	* Detect if a URI identifies some page on this server.
	*
	* NOTE: This method is generally intended to reject URIs which it may be
	* unsafe to issue a "Location:" redirect to.
	*
	* @param string URI to test.
	* @return bool True if the URI identifies a local page.
	* @task uri
	*/
	public static function isValidLocalURIForLink($uri) {
	$uri = (string)$uri;

	if (!strlen($uri)) {
	return false;
	}

	if (preg_match('/\s/', $uri)) {
	// PHP hasn't been vulnerable to header injection attacks for a bunch of
	// years, but we can safely reject these anyway since they're never valid.
	return false;
	}

	// Chrome (at a minimum) interprets backslashes in Location headers and the
	// URL bar as forward slashes. This is probably intended to reduce user
	// error caused by confusion over which key is "forward slash" vs "back
	// slash".
	//
	// However, it means a URI like "/\evil.com" is interpreted like
	// "//evil.com", which is a protocol relative remote URI.
	//
	// Since we currently never generate URIs with backslashes in them, reject
	// these unconditionally rather than trying to figure out how browsers will
	// interpret them.
	if (preg_match('/\\\\/', $uri)) {
	return false;
	}

	// Valid URIs must begin with '/', followed by the end of the string or some
	// other non-'/' character. This rejects protocol-relative URIs like
	// "//evil.com/evil_stuff/".
	return (bool)preg_match('@^/([^/]\|$)@', $uri);
	}


	/**
	* Detect if a URI identifies some valid linkable remote resource.
	*
	* @param string URI to test.
	* @return bool True if a URI idenfies a remote resource with an allowed
	* protocol.
	* @task uri
	*/
	public static function isValidRemoteURIForLink($uri) {
	try {
	self::requireValidRemoteURIForLink($uri);
	return true;
	} catch (Exception $ex) {
	return false;
	}
	}


	/**
	* Detect if a URI identifies a valid linkable remote resource, throwing a
	* detailed message if it does not.
	*
	* A valid linkable remote resource can be safely linked or redirected to.
	* This is primarily a protocol whitelist check.
	*
	* @param string URI to test.
	* @return void
	* @task uri
	*/
	public static function requireValidRemoteURIForLink($raw_uri) {
	$uri = new PhutilURI($raw_uri);

	$proto = $uri->getProtocol();
	if (!strlen($proto)) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid linkable resource. A valid linkable '.
	'resource URI must specify a protocol.',
	$raw_uri));
	}

	$protocols = self::getEnvConfig('uri.allowed-protocols');
	if (!isset($protocols[$proto])) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid linkable resource. A valid linkable '.
	'resource URI must use one of these protocols: %s.',
	$raw_uri,
	implode(', ', array_keys($protocols))));
	}

	$domain = $uri->getDomain();
	if (!strlen($domain)) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid linkable resource. A valid linkable '.
	'resource URI must specify a domain.',
	$raw_uri));
	}
	}


	/**
	* Detect if a URI identifies a valid fetchable remote resource.
	*
	* @param string URI to test.
	* @param list<string> Allowed protocols.
	* @return bool True if the URI is a valid fetchable remote resource.
	* @task uri
	*/
	public static function isValidRemoteURIForFetch($uri, array $protocols) {
	try {
	self::requireValidRemoteURIForFetch($uri, $protocols);
	return true;
	} catch (Exception $ex) {
	return false;
	}
	}


	/**
	* Detect if a URI identifies a valid fetchable remote resource, throwing
	* a detailed message if it does not.
	*
	* A valid fetchable remote resource can be safely fetched using a request
	* originating on this server. This is a primarily an address check against
	* the outbound address blacklist.
	*
	* @param string URI to test.
	* @param list<string> Allowed protocols.
	* @return pair<string, string> Pre-resolved URI and domain.
	* @task uri
	*/
	public static function requireValidRemoteURIForFetch(
	$uri,
	array $protocols) {

	$uri = new PhutilURI($uri);

	$proto = $uri->getProtocol();
	if (!strlen($proto)) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid fetchable resource. A valid fetchable '.
	'resource URI must specify a protocol.',
	$uri));
	}

	$protocols = array_fuse($protocols);
	if (!isset($protocols[$proto])) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid fetchable resource. A valid fetchable '.
	'resource URI must use one of these protocols: %s.',
	$uri,
	implode(', ', array_keys($protocols))));
	}

	$domain = $uri->getDomain();
	if (!strlen($domain)) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid fetchable resource. A valid fetchable '.
	'resource URI must specify a domain.',
	$uri));
	}

	$addresses = gethostbynamel($domain);
	if (!$addresses) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid fetchable resource. The domain "%s" could '.
	'not be resolved.',
	$uri,
	$domain));
	}

	foreach ($addresses as $address) {
	if (self::isBlacklistedOutboundAddress($address)) {
	throw new Exception(
	pht(
	'URI "%s" is not a valid fetchable resource. The domain "%s" '.
	'resolves to the address "%s", which is blacklisted for '.
	'outbound requests.',
	$uri,
	$domain,
	$address));
	}
	}

	$resolved_uri = clone $uri;
	$resolved_uri->setDomain(head($addresses));

	return array($resolved_uri, $domain);
	}


	/**
	* Determine if an IP address is in the outbound address blacklist.
	*
	* @param string IP address.
	* @return bool True if the address is blacklisted.
	*/
	public static function isBlacklistedOutboundAddress($address) {
	$blacklist = self::getEnvConfig('security.outbound-blacklist');

	return PhutilCIDRList::newList($blacklist)->containsAddress($address);
	}

	public static function isClusterRemoteAddress() {
	$cluster_addresses = self::getEnvConfig('cluster.addresses');
	if (!$cluster_addresses) {
	return false;
	}

	$address = idx($_SERVER, 'REMOTE_ADDR');
	if (!$address) {
	throw new Exception(
	pht(
	'Unable to test remote address against cluster whitelist: '.
	'REMOTE_ADDR is not defined.'));
	}

	return self::isClusterAddress($address);
	}

	public static function isClusterAddress($address) {
	$cluster_addresses = self::getEnvConfig('cluster.addresses');
	if (!$cluster_addresses) {
	throw new Exception(
	pht(
	'Phabricator is not configured to serve cluster requests. '.
	'Set `cluster.addresses` in the configuration to whitelist '.
	'cluster hosts before sending requests that use a cluster '.
	'authentication mechanism.'));
	}

	return PhutilCIDRList::newList($cluster_addresses)
	->containsAddress($address);
	}

	/* -( Internals )---------------------------------------------------------- */


	/**
	* @task internal
	*/
	public static function envConfigExists($key) {
	return array_key_exists($key, self::$sourceStack->getKeys(array($key)));
	}


	/**
	* @task internal
	*/
	public static function getAllConfigKeys() {
	return self::$sourceStack->getAllKeys();
	}

	public static function getConfigSourceStack() {
	return self::$sourceStack;
	}

	/**
	* @task internal
	*/
	public static function overrideTestEnvConfig($stack_key, $key, $value) {
	$tmp = array();

	// If we don't have the right key, we'll throw when popping the last
	// source off the stack.
	do {
	$source = self::$sourceStack->popSource();
	array_unshift($tmp, $source);
	if (spl_object_hash($source) == $stack_key) {
	$source->setKeys(array($key => $value));
	break;
	}
	} while (true);

	foreach ($tmp as $source) {
	self::$sourceStack->pushSource($source);
	}

	self::dropConfigCache();
	}

	private static function dropConfigCache() {
	self::$cache = array();
	}

	private static function resetUmask() {
	// Reset the umask to the common standard umask. The umask controls default
	// permissions when files are created and propagates to subprocesses.

	// "022" is the most common umask, but sometimes it is set to something
	// unusual by the calling environment.

	// Since various things rely on this umask to work properly and we are
	// not aware of any legitimate reasons to adjust it, unconditionally
	// normalize it until such reasons arise. See T7475 for discussion.
	umask(022);
	}


	/**
	* Get the path to an empty directory which is readable by all of the system
	* user accounts that Phabricator acts as.
	*
	* In some cases, a binary needs some valid HOME or CWD to continue, but not
	* all user accounts have valid home directories and even if they do they
	* may not be readable after a `sudo` operation.
	*
	* @return string Path to an empty directory suitable for use as a CWD.
	*/
	public static function getEmptyCWD() {
	$root = dirname(phutil_get_library_root('phabricator'));
	return $root.'/support/empty/';
	}


	}
	diff --git a/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php b/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php
	index d3a287259a..8bcd00c4f2 100644
	--- a/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php
	+++ b/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php
	@@ -1,354 +1,356 @@
	<?php

	/**
	* @task config Configuring Storage
	*/
	abstract class PhabricatorLiskDAO extends LiskDAO {

	private static $namespaceStack = array();

	const ATTACHABLE = '<attachable>';
	const CONFIG_APPLICATION_SERIALIZERS = 'phabricator/serializers';

	/* -( Configuring Storage )------------------------------------------------ */

	/**
	* @task config
	*/
	public static function pushStorageNamespace($namespace) {
	self::$namespaceStack[] = $namespace;
	}

	/**
	* @task config
	*/
	public static function popStorageNamespace() {
	array_pop(self::$namespaceStack);
	}

	/**
	* @task config
	*/
	public static function getDefaultStorageNamespace() {
	return PhabricatorEnv::getEnvConfig('storage.default-namespace');
	}

	/**
	* @task config
	*/
	public static function getStorageNamespace() {
	$namespace = end(self::$namespaceStack);
	if (!strlen($namespace)) {
	$namespace = self::getDefaultStorageNamespace();
	}
	if (!strlen($namespace)) {
	throw new Exception(pht('No storage namespace configured!'));
	}
	return $namespace;
	}

	/**
	* @task config
	*/
	protected function establishLiveConnection($mode) {
	$namespace = self::getStorageNamespace();
	$database = $namespace.'_'.$this->getApplicationName();

	$is_readonly = PhabricatorEnv::isReadOnly();

	if ($is_readonly && ($mode != 'r')) {
	$this->raiseImproperWrite($database);
	}

	$is_cluster = (bool)PhabricatorEnv::getEnvConfig('cluster.databases');
	if ($is_cluster) {
	$connection = $this->newClusterConnection($database, $mode);
	} else {
	$connection = $this->newBasicConnection($database, $mode, $namespace);
	}

	// TODO: This should be testing if the mode is "r", but that would probably
	// break a lot of things. Perform a more narrow test for readonly mode
	// until we have greater certainty that this works correctly most of the
	// time.
	if ($is_readonly) {
	$connection->setReadOnly(true);
	}

	// Unless this is a script running from the CLI:
	// - (T10849) Prevent any query from running for more than 30 seconds.
	// - (T11672) Use persistent connections.
	if (php_sapi_name() != 'cli') {

	// TODO: For now, disable this until after T11044: it's better at high
	// load, but causes us to use slightly more connections at low load and
	// is pushing users over limits like MySQL "max_connections".
	$use_persistent = false;

	$connection
	->setQueryTimeout(30)
	->setPersistent($use_persistent);
	}

	return $connection;
	}

	private function newBasicConnection($database, $mode, $namespace) {
	$conf = PhabricatorEnv::newObjectFromConfig(
	'mysql.configuration-provider',
	array($this, $mode, $namespace));

	return PhabricatorEnv::newObjectFromConfig(
	'mysql.implementation',
	array(
	array(
	'user' => $conf->getUser(),
	'pass' => $conf->getPassword(),
	'host' => $conf->getHost(),
	'port' => $conf->getPort(),
	'database' => $database,
	'retries' => 3,
	'timeout' => 10,
	),
	));
	}

	private function newClusterConnection($database, $mode) {
	- $master = PhabricatorDatabaseRef::getMasterDatabaseRef();
	+ $master = PhabricatorDatabaseRef::getMasterDatabaseRefForDatabase(
	+ $database);

	if ($master && !$master->isSevered()) {
	$connection = $master->newApplicationConnection($database);
	if ($master->isReachable($connection)) {
	return $connection;
	} else {
	if ($mode == 'w') {
	$this->raiseImpossibleWrite($database);
	}
	PhabricatorEnv::setReadOnly(
	true,
	PhabricatorEnv::READONLY_UNREACHABLE);
	}
	}

	- $replica = PhabricatorDatabaseRef::getReplicaDatabaseRef();
	+ $replica = PhabricatorDatabaseRef::getReplicaDatabaseRefForDatabase(
	+ $database);
	if ($replica) {
	$connection = $replica->newApplicationConnection($database);
	$connection->setReadOnly(true);
	if ($replica->isReachable($connection)) {
	return $connection;
	}
	}

	if (!$master && !$replica) {
	$this->raiseUnconfigured($database);
	}

	$this->raiseUnreachable($database);
	}

	private function raiseImproperWrite($database) {
	throw new PhabricatorClusterImproperWriteException(
	pht(
	'Unable to establish a write-mode connection (to application '.
	'database "%s") because Phabricator is in read-only mode. Whatever '.
	'you are trying to do does not function correctly in read-only mode.',
	$database));
	}

	private function raiseImpossibleWrite($database) {
	throw new PhabricatorClusterImpossibleWriteException(
	pht(
	'Unable to connect to master database ("%s"). This is a severe '.
	'failure; your request did not complete.',
	$database));
	}

	private function raiseUnconfigured($database) {
	throw new Exception(
	pht(
	'Unable to establish a connection to any database host '.
	'(while trying "%s"). No masters or replicas are configured.',
	$database));
	}

	private function raiseUnreachable($database) {
	throw new PhabricatorClusterStrandedException(
	pht(
	'Unable to establish a connection to any database host '.
	'(while trying "%s"). All masters and replicas are completely '.
	'unreachable.',
	$database));
	}


	/**
	* @task config
	*/
	public function getTableName() {
	$str = 'phabricator';
	$len = strlen($str);

	$class = strtolower(get_class($this));
	if (!strncmp($class, $str, $len)) {
	$class = substr($class, $len);
	}
	$app = $this->getApplicationName();
	if (!strncmp($class, $app, strlen($app))) {
	$class = substr($class, strlen($app));
	}

	if (strlen($class)) {
	return $app.'_'.$class;
	} else {
	return $app;
	}
	}

	/**
	* @task config
	*/
	abstract public function getApplicationName();

	protected function getConnectionNamespace() {
	return self::getStorageNamespace().'_'.$this->getApplicationName();
	}


	/**
	* Break a list of escaped SQL statement fragments (e.g., VALUES lists for
	* INSERT, previously built with @{function:qsprintf}) into chunks which will
	* fit under the MySQL 'max_allowed_packet' limit.
	*
	* Chunks are glued together with `$glue`, by default ", ".
	*
	* If a statement is too large to fit within the limit, it is broken into
	* its own chunk (but might fail when the query executes).
	*/
	public static function chunkSQL(
	array $fragments,
	$glue = ', ',
	$limit = null) {

	if ($limit === null) {
	// NOTE: Hard-code this at 1MB for now, minus a 10% safety buffer.
	// Eventually we could query MySQL or let the user configure it.
	$limit = (int)((1024 * 1024) * 0.90);
	}

	$result = array();

	$chunk = array();
	$len = 0;
	$glue_len = strlen($glue);
	foreach ($fragments as $fragment) {
	$this_len = strlen($fragment);

	if ($chunk) {
	// Chunks after the first also imply glue.
	$this_len += $glue_len;
	}

	if ($len + $this_len <= $limit) {
	$len += $this_len;
	$chunk[] = $fragment;
	} else {
	if ($chunk) {
	$result[] = $chunk;
	}
	$len = strlen($fragment);
	$chunk = array($fragment);
	}
	}

	if ($chunk) {
	$result[] = $chunk;
	}

	foreach ($result as $key => $fragment_list) {
	$result[$key] = implode($glue, $fragment_list);
	}

	return $result;
	}

	protected function assertAttached($property) {
	if ($property === self::ATTACHABLE) {
	throw new PhabricatorDataNotAttachedException($this);
	}
	return $property;
	}

	protected function assertAttachedKey($value, $key) {
	$this->assertAttached($value);
	if (!array_key_exists($key, $value)) {
	throw new PhabricatorDataNotAttachedException($this);
	}
	return $value[$key];
	}

	protected function detectEncodingForStorage($string) {
	return phutil_is_utf8($string) ? 'utf8' : null;
	}

	protected function getUTF8StringFromStorage($string, $encoding) {
	if ($encoding == 'utf8') {
	return $string;
	}

	if (function_exists('mb_detect_encoding')) {
	if (strlen($encoding)) {
	$try_encodings = array(
	$encoding,
	);
	} else {
	// TODO: This is pretty much a guess, and probably needs to be
	// configurable in the long run.
	$try_encodings = array(
	'JIS',
	'EUC-JP',
	'SJIS',
	'ISO-8859-1',
	);
	}

	$guess = mb_detect_encoding($string, $try_encodings);
	if ($guess) {
	return mb_convert_encoding($string, 'UTF-8', $guess);
	}
	}

	return phutil_utf8ize($string);
	}

	protected function willReadData(array &$data) {
	parent::willReadData($data);

	static $custom;
	if ($custom === null) {
	$custom = $this->getConfigOption(self::CONFIG_APPLICATION_SERIALIZERS);
	}

	if ($custom) {
	foreach ($custom as $key => $serializer) {
	$data[$key] = $serializer->willReadValue($data[$key]);
	}
	}
	}

	protected function willWriteData(array &$data) {
	static $custom;
	if ($custom === null) {
	$custom = $this->getConfigOption(self::CONFIG_APPLICATION_SERIALIZERS);
	}

	if ($custom) {
	foreach ($custom as $key => $serializer) {
	$data[$key] = $serializer->willWriteValue($data[$key]);
	}
	}

	parent::willWriteData($data);
	}


	}

File Metadata

Mime Type: text/x-diff
Expires: Sun, Jan 19, 13:55 (3 w, 2 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1125279
Default Alt Text: (70 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions