No OneTemporary
Actions

Size

20 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/src/applications/diffusion/query/blame/DiffusionBlameQuery.php b/src/applications/diffusion/query/blame/DiffusionBlameQuery.php
	index 6d83020dd2..4d03b4a143 100644
	--- a/src/applications/diffusion/query/blame/DiffusionBlameQuery.php
	+++ b/src/applications/diffusion/query/blame/DiffusionBlameQuery.php
	@@ -1,69 +1,180 @@
	<?php

	abstract class DiffusionBlameQuery extends DiffusionQuery {

	private $timeout;
	private $paths;

	public function setTimeout($timeout) {
	$this->timeout = $timeout;
	return $this;
	}

	public function getTimeout() {
	return $this->timeout;
	}

	public function setPaths(array $paths) {
	$this->paths = $paths;
	return $this;
	}

	public function getPaths() {
	return $this->paths;
	}

	abstract protected function newBlameFuture(DiffusionRequest $request, $path);

	abstract protected function resolveBlameFuture(ExecFuture $future);

	final public static function newFromDiffusionRequest(
	DiffusionRequest $request) {
	return parent::newQueryObject(__CLASS__, $request);
	}

	final protected function executeQuery() {
	$paths = $this->getPaths();
	+
	+ $blame = array();
	+
	+ // Load cache keys: these are the commits at which each path was last
	+ // touched.
	+ $keys = $this->loadCacheKeys($paths);
	+
	+ // Try to read blame data from cache.
	+ $cache = $this->readCacheData($keys);
	+ foreach ($paths as $key => $path) {
	+ if (!isset($cache[$path])) {
	+ continue;
	+ }
	+
	+ $blame[$path] = $cache[$path];
	+ unset($paths[$key]);
	+ }
	+
	+ // If we have no paths left, we filled everything from cache and can
	+ // bail out early.
	+ if (!$paths) {
	+ return $blame;
	+ }
	+
	$request = $this->getRequest();
	$timeout = $this->getTimeout();

	+ // We're still missing at least some data, so we need to run VCS commands
	+ // to pull it.
	$futures = array();
	foreach ($paths as $path) {
	$future = $this->newBlameFuture($request, $path);

	if ($timeout) {
	$future->setTimeout($timeout);
	}

	$futures[$path] = $future;
	}

	+ $futures = id(new FutureIterator($futures))
	+ ->limit(4);

	- $blame = array();
	+ foreach ($futures as $path => $future) {
	+ $path_blame = $this->resolveBlameFuture($future);
	+ if ($path_blame !== null) {
	+ $blame[$path] = $path_blame;
	+ }
	+ }
	+
	+ // Fill the cache with anything we generated.
	+ $this->writeCacheData(
	+ array_select_keys($keys, $paths),
	+ $blame);

	- if ($futures) {
	- $futures = id(new FutureIterator($futures))
	- ->limit(4);
	+ return $blame;
	+ }

	- foreach ($futures as $path => $future) {
	- $path_blame = $this->resolveBlameFuture($future);
	- if ($path_blame !== null) {
	- $blame[$path] = $path_blame;
	- }
	+ private function loadCacheKeys(array $paths) {
	+ $request = $this->getRequest();
	+ $viewer = $request->getUser();
	+
	+ $repository = $request->getRepository();
	+ $repository_id = $repository->getID();
	+
	+ $last_modified = parent::callConduitWithDiffusionRequest(
	+ $viewer,
	+ $request,
	+ 'diffusion.lastmodifiedquery',
	+ array(
	+ 'paths' => array_fill_keys($paths, $request->getCommit()),
	+ ));
	+
	+ $map = array();
	+ foreach ($paths as $path) {
	+ $identifier = idx($last_modified, $path);
	+ if ($identifier === null) {
	+ continue;
	}
	+
	+ $map[$path] = "blame({$repository_id}, {$identifier}, {$path}, raw)";
	}

	- return $blame;
	+ return $map;
	+ }
	+
	+ private function readCacheData(array $keys) {
	+ $cache = PhabricatorCaches::getImmutableCache();
	+ $data = $cache->getKeys($keys);
	+
	+ $results = array();
	+ foreach ($keys as $path => $key) {
	+ if (!isset($data[$key])) {
	+ continue;
	+ }
	+ $results[$path] = $data[$key];
	+ }
	+
	+ // Decode the cache storage format.
	+ foreach ($results as $path => $cache) {
	+ list($head, $body) = explode("\n", $cache, 2);
	+ switch ($head) {
	+ case 'raw':
	+ $body = explode("\n", $body);
	+ break;
	+ default:
	+ $body = null;
	+ break;
	+ }
	+
	+ if ($body === null) {
	+ unset($results[$path]);
	+ } else {
	+ $results[$path] = $body;
	+ }
	+ }
	+
	+ return $results;
	+ }
	+
	+ private function writeCacheData(array $keys, array $blame) {
	+ $writes = array();
	+ foreach ($keys as $path => $key) {
	+ $value = idx($blame, $path);
	+ if ($value === null) {
	+ continue;
	+ }
	+
	+ // For now, just store the entire value with a "raw" header. In the
	+ // future, we could compress this or use IDs instead.
	+ $value = "raw\n".implode("\n", $value);
	+
	+ $writes[$key] = $value;
	+ }
	+
	+ if (!$writes) {
	+ return;
	+ }
	+
	+ $cache = PhabricatorCaches::getImmutableCache();
	+ $data = $cache->setKeys($writes, phutil_units('14 days in seconds'));
	}

	}
	diff --git a/src/applications/repository/query/PhabricatorRepositoryQuery.php b/src/applications/repository/query/PhabricatorRepositoryQuery.php
	index 38fa5bf56c..bd9765130a 100644
	--- a/src/applications/repository/query/PhabricatorRepositoryQuery.php
	+++ b/src/applications/repository/query/PhabricatorRepositoryQuery.php
	@@ -1,596 +1,598 @@
	<?php

	final class PhabricatorRepositoryQuery
	extends PhabricatorCursorPagedPolicyAwareQuery {

	private $ids;
	private $phids;
	private $callsigns;
	private $types;
	private $uuids;
	private $nameContains;
	private $remoteURIs;
	private $datasourceQuery;

	private $numericIdentifiers;
	private $callsignIdentifiers;
	private $phidIdentifiers;
	private $monogramIdentifiers;

	private $identifierMap;

	const STATUS_OPEN = 'status-open';
	const STATUS_CLOSED = 'status-closed';
	const STATUS_ALL = 'status-all';
	private $status = self::STATUS_ALL;

	const HOSTED_PHABRICATOR = 'hosted-phab';
	const HOSTED_REMOTE = 'hosted-remote';
	const HOSTED_ALL = 'hosted-all';
	private $hosted = self::HOSTED_ALL;

	private $needMostRecentCommits;
	private $needCommitCounts;
	private $needProjectPHIDs;

	public function withIDs(array $ids) {
	$this->ids = $ids;
	return $this;
	}

	public function withPHIDs(array $phids) {
	$this->phids = $phids;
	return $this;
	}

	public function withCallsigns(array $callsigns) {
	$this->callsigns = $callsigns;
	return $this;
	}

	public function withIdentifiers(array $identifiers) {
	+ $identifiers = array_fuse($identifiers);
	+
	$ids = array();
	$callsigns = array();
	$phids = array();
	$monograms = array();

	foreach ($identifiers as $identifier) {
	if (ctype_digit((string)$identifier)) {
	$ids[$identifier] = $identifier;
	} else if (preg_match('/^(r[A-Z]+)\|(R[1-9]\d*)\z/', $identifier)) {
	$monograms[$identifier] = $identifier;
	} else {
	$repository_type = PhabricatorRepositoryRepositoryPHIDType::TYPECONST;
	if (phid_get_type($identifier) === $repository_type) {
	$phids[$identifier] = $identifier;
	} else {
	$callsigns[$identifier] = $identifier;
	}
	}
	}

	$this->numericIdentifiers = $ids;
	$this->callsignIdentifiers = $callsigns;
	$this->phidIdentifiers = $phids;
	$this->monogramIdentifiers = $monograms;

	return $this;
	}

	public function withStatus($status) {
	$this->status = $status;
	return $this;
	}

	public function withHosted($hosted) {
	$this->hosted = $hosted;
	return $this;
	}

	public function withTypes(array $types) {
	$this->types = $types;
	return $this;
	}

	public function withUUIDs(array $uuids) {
	$this->uuids = $uuids;
	return $this;
	}

	public function withNameContains($contains) {
	$this->nameContains = $contains;
	return $this;
	}

	public function withRemoteURIs(array $uris) {
	$this->remoteURIs = $uris;
	return $this;
	}

	public function withDatasourceQuery($query) {
	$this->datasourceQuery = $query;
	return $this;
	}

	public function needCommitCounts($need_counts) {
	$this->needCommitCounts = $need_counts;
	return $this;
	}

	public function needMostRecentCommits($need_commits) {
	$this->needMostRecentCommits = $need_commits;
	return $this;
	}

	public function needProjectPHIDs($need_phids) {
	$this->needProjectPHIDs = $need_phids;
	return $this;
	}

	public function getBuiltinOrders() {
	return array(
	'committed' => array(
	'vector' => array('committed', 'id'),
	'name' => pht('Most Recent Commit'),
	),
	'name' => array(
	'vector' => array('name', 'id'),
	'name' => pht('Name'),
	),
	'callsign' => array(
	'vector' => array('callsign'),
	'name' => pht('Callsign'),
	),
	'size' => array(
	'vector' => array('size', 'id'),
	'name' => pht('Size'),
	),
	) + parent::getBuiltinOrders();
	}

	public function getIdentifierMap() {
	if ($this->identifierMap === null) {
	throw new PhutilInvalidStateException('execute');
	}
	return $this->identifierMap;
	}

	protected function willExecute() {
	$this->identifierMap = array();
	}

	public function newResultObject() {
	return new PhabricatorRepository();
	}

	protected function loadPage() {
	$table = $this->newResultObject();
	$data = $this->loadStandardPageRows($table);
	$repositories = $table->loadAllFromArray($data);

	if ($this->needCommitCounts) {
	$sizes = ipull($data, 'size', 'id');
	foreach ($repositories as $id => $repository) {
	$repository->attachCommitCount(nonempty($sizes[$id], 0));
	}
	}

	if ($this->needMostRecentCommits) {
	$commit_ids = ipull($data, 'lastCommitID', 'id');
	$commit_ids = array_filter($commit_ids);
	if ($commit_ids) {
	$commits = id(new DiffusionCommitQuery())
	->setViewer($this->getViewer())
	->withIDs($commit_ids)
	->execute();
	} else {
	$commits = array();
	}
	foreach ($repositories as $id => $repository) {
	$commit = null;
	if (idx($commit_ids, $id)) {
	$commit = idx($commits, $commit_ids[$id]);
	}
	$repository->attachMostRecentCommit($commit);
	}
	}

	return $repositories;
	}

	protected function willFilterPage(array $repositories) {
	assert_instances_of($repositories, 'PhabricatorRepository');

	// TODO: Denormalize repository status into the PhabricatorRepository
	// table so we can do this filtering in the database.
	foreach ($repositories as $key => $repo) {
	$status = $this->status;
	switch ($status) {
	case self::STATUS_OPEN:
	if (!$repo->isTracked()) {
	unset($repositories[$key]);
	}
	break;
	case self::STATUS_CLOSED:
	if ($repo->isTracked()) {
	unset($repositories[$key]);
	}
	break;
	case self::STATUS_ALL:
	break;
	default:
	throw new Exception("Unknown status '{$status}'!");
	}

	// TODO: This should also be denormalized.
	$hosted = $this->hosted;
	switch ($hosted) {
	case self::HOSTED_PHABRICATOR:
	if (!$repo->isHosted()) {
	unset($repositories[$key]);
	}
	break;
	case self::HOSTED_REMOTE:
	if ($repo->isHosted()) {
	unset($repositories[$key]);
	}
	break;
	case self::HOSTED_ALL:
	break;
	default:
	throw new Exception(pht("Unknown hosted failed '%s'!", $hosted));
	}
	}

	// TODO: Denormalize this, too.
	if ($this->remoteURIs) {
	$try_uris = $this->getNormalizedPaths();
	$try_uris = array_fuse($try_uris);
	foreach ($repositories as $key => $repository) {
	if (!isset($try_uris[$repository->getNormalizedPath()])) {
	unset($repositories[$key]);
	}
	}
	}

	// Build the identifierMap
	if ($this->numericIdentifiers) {
	foreach ($this->numericIdentifiers as $id) {
	if (isset($repositories[$id])) {
	$this->identifierMap[$id] = $repositories[$id];
	}
	}
	}

	if ($this->callsignIdentifiers) {
	$repository_callsigns = mpull($repositories, null, 'getCallsign');

	foreach ($this->callsignIdentifiers as $callsign) {
	if (isset($repository_callsigns[$callsign])) {
	$this->identifierMap[$callsign] = $repository_callsigns[$callsign];
	}
	}
	}

	if ($this->phidIdentifiers) {
	$repository_phids = mpull($repositories, null, 'getPHID');

	foreach ($this->phidIdentifiers as $phid) {
	if (isset($repository_phids[$phid])) {
	$this->identifierMap[$phid] = $repository_phids[$phid];
	}
	}
	}

	if ($this->monogramIdentifiers) {
	$monogram_map = array();
	foreach ($repositories as $repository) {
	foreach ($repository->getAllMonograms() as $monogram) {
	$monogram_map[$monogram] = $repository;
	}
	}

	foreach ($this->monogramIdentifiers as $monogram) {
	if (isset($monogram_map[$monogram])) {
	$this->identifierMap[$monogram] = $monogram_map[$monogram];
	}
	}
	}

	return $repositories;
	}

	protected function didFilterPage(array $repositories) {
	if ($this->needProjectPHIDs) {
	$type_project = PhabricatorProjectObjectHasProjectEdgeType::EDGECONST;

	$edge_query = id(new PhabricatorEdgeQuery())
	->withSourcePHIDs(mpull($repositories, 'getPHID'))
	->withEdgeTypes(array($type_project));
	$edge_query->execute();

	foreach ($repositories as $repository) {
	$project_phids = $edge_query->getDestinationPHIDs(
	array(
	$repository->getPHID(),
	));
	$repository->attachProjectPHIDs($project_phids);
	}
	}

	return $repositories;
	}

	protected function getPrimaryTableAlias() {
	return 'r';
	}

	public function getOrderableColumns() {
	return parent::getOrderableColumns() + array(
	'committed' => array(
	'table' => 's',
	'column' => 'epoch',
	'type' => 'int',
	'null' => 'tail',
	),
	'callsign' => array(
	'table' => 'r',
	'column' => 'callsign',
	'type' => 'string',
	'unique' => true,
	'reverse' => true,
	),
	'name' => array(
	'table' => 'r',
	'column' => 'name',
	'type' => 'string',
	'reverse' => true,
	),
	'size' => array(
	'table' => 's',
	'column' => 'size',
	'type' => 'int',
	'null' => 'tail',
	),
	);
	}

	protected function willExecuteCursorQuery(
	PhabricatorCursorPagedPolicyAwareQuery $query) {
	$vector = $this->getOrderVector();

	if ($vector->containsKey('committed')) {
	$query->needMostRecentCommits(true);
	}

	if ($vector->containsKey('size')) {
	$query->needCommitCounts(true);
	}
	}

	protected function getPagingValueMap($cursor, array $keys) {
	$repository = $this->loadCursorObject($cursor);

	$map = array(
	'id' => $repository->getID(),
	'callsign' => $repository->getCallsign(),
	'name' => $repository->getName(),
	);

	foreach ($keys as $key) {
	switch ($key) {
	case 'committed':
	$commit = $repository->getMostRecentCommit();
	if ($commit) {
	$map[$key] = $commit->getEpoch();
	} else {
	$map[$key] = null;
	}
	break;
	case 'size':
	$count = $repository->getCommitCount();
	if ($count) {
	$map[$key] = $count;
	} else {
	$map[$key] = null;
	}
	break;
	}
	}

	return $map;
	}

	protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) {
	$parts = parent::buildSelectClauseParts($conn);

	if ($this->shouldJoinSummaryTable()) {
	$parts[] = 's.*';
	}

	return $parts;
	}

	protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) {
	$joins = parent::buildJoinClauseParts($conn);

	if ($this->shouldJoinSummaryTable()) {
	$joins[] = qsprintf(
	$conn,
	'LEFT JOIN %T s ON r.id = s.repositoryID',
	PhabricatorRepository::TABLE_SUMMARY);
	}

	return $joins;
	}

	private function shouldJoinSummaryTable() {
	if ($this->needCommitCounts) {
	return true;
	}

	if ($this->needMostRecentCommits) {
	return true;
	}

	$vector = $this->getOrderVector();
	if ($vector->containsKey('committed')) {
	return true;
	}

	if ($vector->containsKey('size')) {
	return true;
	}

	return false;
	}

	protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
	$where = parent::buildWhereClauseParts($conn);

	if ($this->ids !== null) {
	$where[] = qsprintf(
	$conn,
	'r.id IN (%Ld)',
	$this->ids);
	}

	if ($this->phids !== null) {
	$where[] = qsprintf(
	$conn,
	'r.phid IN (%Ls)',
	$this->phids);
	}

	if ($this->callsigns !== null) {
	$where[] = qsprintf(
	$conn,
	'r.callsign IN (%Ls)',
	$this->callsigns);
	}

	if ($this->numericIdentifiers \|\|
	$this->callsignIdentifiers \|\|
	$this->phidIdentifiers \|\|
	$this->monogramIdentifiers) {
	$identifier_clause = array();

	if ($this->numericIdentifiers) {
	$identifier_clause[] = qsprintf(
	$conn,
	'r.id IN (%Ld)',
	$this->numericIdentifiers);
	}

	if ($this->callsignIdentifiers) {
	$identifier_clause[] = qsprintf(
	$conn,
	'r.callsign IN (%Ls)',
	$this->callsignIdentifiers);
	}

	if ($this->phidIdentifiers) {
	$identifier_clause[] = qsprintf(
	$conn,
	'r.phid IN (%Ls)',
	$this->phidIdentifiers);
	}

	if ($this->monogramIdentifiers) {
	$monogram_callsigns = array();
	$monogram_ids = array();

	foreach ($this->monogramIdentifiers as $identifier) {
	if ($identifier[0] == 'r') {
	$monogram_callsigns[] = substr($identifier, 1);
	} else {
	$monogram_ids[] = substr($identifier, 1);
	}
	}

	if ($monogram_ids) {
	$identifier_clause[] = qsprintf(
	$conn,
	'r.id IN (%Ld)',
	$monogram_ids);
	}

	if ($monogram_callsigns) {
	$identifier_clause[] = qsprintf(
	$conn,
	'r.callsign IN (%Ls)',
	$monogram_callsigns);
	}
	}

	$where = array('('.implode(' OR ', $identifier_clause).')');
	}

	if ($this->types) {
	$where[] = qsprintf(
	$conn,
	'r.versionControlSystem IN (%Ls)',
	$this->types);
	}

	if ($this->uuids) {
	$where[] = qsprintf(
	$conn,
	'r.uuid IN (%Ls)',
	$this->uuids);
	}

	if (strlen($this->nameContains)) {
	$where[] = qsprintf(
	$conn,
	'name LIKE %~',
	$this->nameContains);
	}

	if (strlen($this->datasourceQuery)) {
	// This handles having "rP" match callsigns starting with "P...".
	$query = trim($this->datasourceQuery);
	if (preg_match('/^r/', $query)) {
	$callsign = substr($query, 1);
	} else {
	$callsign = $query;
	}
	$where[] = qsprintf(
	$conn,
	'r.name LIKE %> OR r.callsign LIKE %>',
	$query,
	$callsign);
	}

	return $where;
	}

	public function getQueryApplicationClass() {
	return 'PhabricatorDiffusionApplication';
	}

	private function getNormalizedPaths() {
	$normalized_uris = array();

	// Since we don't know which type of repository this URI is in the general
	// case, just generate all the normalizations. We could refine this in some
	// cases: if the query specifies VCS types, or the URI is a git-style URI
	// or an `svn+ssh` URI, we could deduce how to normalize it. However, this
	// would be more complicated and it's not clear if it matters in practice.

	foreach ($this->remoteURIs as $uri) {
	$normalized_uris[] = new PhabricatorRepositoryURINormalizer(
	PhabricatorRepositoryURINormalizer::TYPE_GIT,
	$uri);
	$normalized_uris[] = new PhabricatorRepositoryURINormalizer(
	PhabricatorRepositoryURINormalizer::TYPE_SVN,
	$uri);
	$normalized_uris[] = new PhabricatorRepositoryURINormalizer(
	PhabricatorRepositoryURINormalizer::TYPE_MERCURIAL,
	$uri);
	}

	return array_unique(mpull($normalized_uris, 'getNormalizedPath'));
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Sun, Jan 19, 17:54 (1 w, 5 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1127171
Default Alt Text: (20 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions