No OneTemporary
Actions

Size

20 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/src/applications/cache/PhabricatorCaches.php b/src/applications/cache/PhabricatorCaches.php
	index 860abcb130..f4eb7a918c 100644
	--- a/src/applications/cache/PhabricatorCaches.php
	+++ b/src/applications/cache/PhabricatorCaches.php
	@@ -1,286 +1,347 @@
	<?php

	/**
	* @task immutable Immutable Cache
	* @task setup Setup Cache
	+ * @task compress Compression
	*/
	final class PhabricatorCaches {

	public static function getNamespace() {
	return PhabricatorEnv::getEnvConfig('phabricator.cache-namespace');
	}

	private static function newStackFromCaches(array $caches) {
	$caches = self::addNamespaceToCaches($caches);
	$caches = self::addProfilerToCaches($caches);
	return id(new PhutilKeyValueCacheStack())
	->setCaches($caches);
	}


	/* -( Local Cache )-------------------------------------------------------- */


	/**
	* Gets an immutable cache stack.
	*
	* This stack trades mutability away for improved performance. Normally, it is
	* APC + DB.
	*
	* In the general case with multiple web frontends, this stack can not be
	* cleared, so it is only appropriate for use if the value of a given key is
	* permanent and immutable.
	*
	* @return PhutilKeyValueCacheStack Best immutable stack available.
	* @task immutable
	*/
	public static function getImmutableCache() {
	static $cache;
	if (!$cache) {
	$caches = self::buildImmutableCaches();
	$cache = self::newStackFromCaches($caches);
	}
	return $cache;
	}


	/**
	* Build the immutable cache stack.
	*
	* @return list<PhutilKeyValueCache> List of caches.
	* @task immutable
	*/
	private static function buildImmutableCaches() {
	$caches = array();

	$apc = new PhutilKeyValueCacheAPC();
	if ($apc->isAvailable()) {
	$caches[] = $apc;
	}

	$caches[] = new PhabricatorKeyValueDatabaseCache();

	return $caches;
	}


	/* -( Repository Graph Cache )--------------------------------------------- */


	public static function getRepositoryGraphL1Cache() {
	static $cache;
	if (!$cache) {
	$caches = self::buildRepositoryGraphL1Caches();
	$cache = self::newStackFromCaches($caches);
	}
	return $cache;
	}

	private static function buildRepositoryGraphL1Caches() {
	$caches = array();

	$request = new PhutilKeyValueCacheInRequest();
	$request->setLimit(32);
	$caches[] = $request;

	$apc = new PhutilKeyValueCacheAPC();
	if ($apc->isAvailable()) {
	$caches[] = $apc;
	}

	return $caches;
	}

	public static function getRepositoryGraphL2Cache() {
	static $cache;
	if (!$cache) {
	$caches = self::buildRepositoryGraphL2Caches();
	$cache = self::newStackFromCaches($caches);
	}
	return $cache;
	}

	private static function buildRepositoryGraphL2Caches() {
	$caches = array();
	$caches[] = new PhabricatorKeyValueDatabaseCache();
	return $caches;
	}


	/* -( Setup Cache )-------------------------------------------------------- */


	/**
	* Highly specialized cache for performing setup checks. We use this cache
	* to determine if we need to run expensive setup checks when the page
	* loads. Without it, we would need to run these checks every time.
	*
	* Normally, this cache is just APC. In the absence of APC, this cache
	* degrades into a slow, quirky on-disk cache.
	*
	* NOTE: Do not use this cache for anything else! It is not a general-purpose
	* cache!
	*
	* @return PhutilKeyValueCacheStack Most qualified available cache stack.
	* @task setup
	*/
	public static function getSetupCache() {
	static $cache;
	if (!$cache) {
	$caches = self::buildSetupCaches();
	$cache = self::newStackFromCaches($caches);
	}
	return $cache;
	}


	/**
	* @task setup
	*/
	private static function buildSetupCaches() {
	// In most cases, we should have APC. This is an ideal cache for our
	// purposes -- it's fast and empties on server restart.
	$apc = new PhutilKeyValueCacheAPC();
	if ($apc->isAvailable()) {
	return array($apc);
	}

	// If we don't have APC, build a poor approximation on disk. This is still
	// much better than nothing; some setup steps are quite slow.
	$disk_path = self::getSetupCacheDiskCachePath();
	if ($disk_path) {
	$disk = new PhutilKeyValueCacheOnDisk();
	$disk->setCacheFile($disk_path);
	$disk->setWait(0.1);
	if ($disk->isAvailable()) {
	return array($disk);
	}
	}

	return array();
	}


	/**
	* @task setup
	*/
	private static function getSetupCacheDiskCachePath() {
	// The difficulty here is in choosing a path which will change on server
	// restart (we MUST have this property), but as rarely as possible
	// otherwise (we desire this property to give the cache the best hit rate
	// we can).

	// In some setups, the parent PID is more stable and longer-lived that the
	// PID (e.g., under apache, our PID will be a worker while the ppid will
	// be the main httpd process). If we're confident we're running under such
	// a setup, we can try to use the PPID as the basis for our cache instead
	// of our own PID.
	$use_ppid = false;

	switch (php_sapi_name()) {
	case 'cli-server':
	// This is the PHP5.4+ built-in webserver. We should use the pid
	// (the server), not the ppid (probably a shell or something).
	$use_ppid = false;
	break;
	case 'fpm-fcgi':
	// We should be safe to use PPID here.
	$use_ppid = true;
	break;
	case 'apache2handler':
	// We're definitely safe to use the PPID.
	$use_ppid = true;
	break;
	}

	$pid_basis = getmypid();
	if ($use_ppid) {
	if (function_exists('posix_getppid')) {
	$parent_pid = posix_getppid();
	// On most systems, normal processes can never have PIDs lower than 100,
	// so something likely went wrong if we we get one of these.
	if ($parent_pid > 100) {
	$pid_basis = $parent_pid;
	}
	}
	}

	// If possible, we also want to know when the process launched, so we can
	// drop the cache if a process restarts but gets the same PID an earlier
	// process had. "/proc" is not available everywhere (e.g., not on OSX), but
	// check if we have it.
	$epoch_basis = null;
	$stat = @stat("/proc/{$pid_basis}");
	if ($stat !== false) {
	$epoch_basis = $stat['ctime'];
	}

	$tmp_dir = sys_get_temp_dir();

	$tmp_path = $tmp_dir.DIRECTORY_SEPARATOR.'phabricator-setup';
	if (!file_exists($tmp_path)) {
	@mkdir($tmp_path);
	}

	$is_ok = self::testTemporaryDirectory($tmp_path);
	if (!$is_ok) {
	$tmp_path = $tmp_dir;
	$is_ok = self::testTemporaryDirectory($tmp_path);
	if (!$is_ok) {
	// We can't find anywhere to write the cache, so just bail.
	return null;
	}
	}

	$tmp_name = 'setup-'.$pid_basis;
	if ($epoch_basis) {
	$tmp_name .= '.'.$epoch_basis;
	}
	$tmp_name .= '.cache';

	return $tmp_path.DIRECTORY_SEPARATOR.$tmp_name;
	}


	/**
	* @task setup
	*/
	private static function testTemporaryDirectory($dir) {
	if (!@file_exists($dir)) {
	return false;
	}
	if (!@is_dir($dir)) {
	return false;
	}
	if (!@is_writable($dir)) {
	return false;
	}

	return true;
	}

	private static function addProfilerToCaches(array $caches) {
	foreach ($caches as $key => $cache) {
	$pcache = new PhutilKeyValueCacheProfiler($cache);
	$pcache->setProfiler(PhutilServiceProfiler::getInstance());
	$caches[$key] = $pcache;
	}
	return $caches;
	}

	private static function addNamespaceToCaches(array $caches) {
	$namespace = PhabricatorCaches::getNamespace();
	if (!$namespace) {
	return $caches;
	}

	foreach ($caches as $key => $cache) {
	$ncache = new PhutilKeyValueCacheNamespace($cache);
	$ncache->setNamespace($namespace);
	$caches[$key] = $ncache;
	}

	return $caches;
	}

	+
	+ /**
	+ * Deflate a value, if deflation is available and has an impact.
	+ *
	+ * If the value is larger than 1KB, we have `gzdeflate()`, we successfully
	+ * can deflate it, and it benefits from deflation, we deflate it. Otherwise
	+ * we leave it as-is.
	+ *
	+ * Data can later be inflated with @{method:inflateData}.
	+ *
	+ * @param string String to attempt to deflate.
	+ * @return string\|null Deflated string, or null if it was not deflated.
	+ * @task compress
	+ */
	+ public static function maybeDeflateData($value) {
	+ $len = strlen($value);
	+ if ($len <= 1024) {
	+ return null;
	+ }
	+
	+ if (!function_exists('gzdeflate')) {
	+ return null;
	+ }
	+
	+ $deflated = gzdeflate($value);
	+ if ($deflated === false) {
	+ return null;
	+ }
	+
	+ $deflated_len = strlen($deflated);
	+ if ($deflated_len >= ($len / 2)) {
	+ return null;
	+ }
	+
	+ return $deflated;
	+ }
	+
	+
	+ /**
	+ * Inflate data previously deflated by @{method:maybeDeflateData}.
	+ *
	+ * @param string Deflated data, from @{method:maybeDeflateData}.
	+ * @return string Original, uncompressed data.
	+ * @task compress
	+ */
	+ public static function inflateData($value) {
	+ if (!function_exists('gzinflate')) {
	+ throw new Exception(
	+ pht('gzinflate() is not available; unable to read deflated data!'));
	+ }
	+
	+ $value = gzinflate($value);
	+ if ($value === false) {
	+ throw new Exception(pht('Failed to inflate data!'));
	+ }
	+
	+ return $value;
	+ }
	+
	+
	}
	diff --git a/src/applications/cache/PhabricatorKeyValueDatabaseCache.php b/src/applications/cache/PhabricatorKeyValueDatabaseCache.php
	index 822b90ffcf..aa641ab703 100644
	--- a/src/applications/cache/PhabricatorKeyValueDatabaseCache.php
	+++ b/src/applications/cache/PhabricatorKeyValueDatabaseCache.php
	@@ -1,185 +1,170 @@
	<?php

	final class PhabricatorKeyValueDatabaseCache
	extends PhutilKeyValueCache {

	const CACHE_FORMAT_RAW = 'raw';
	const CACHE_FORMAT_DEFLATE = 'deflate';

	public function setKeys(array $keys, $ttl = null) {
	if ($keys) {
	$map = $this->digestKeys(array_keys($keys));
	$conn_w = $this->establishConnection('w');

	$sql = array();
	foreach ($map as $key => $hash) {
	$value = $keys[$key];

	list($format, $storage_value) = $this->willWriteValue($key, $value);

	$sql[] = qsprintf(
	$conn_w,
	'(%s, %s, %s, %B, %d, %nd)',
	$hash,
	$key,
	$format,
	$storage_value,
	time(),
	$ttl ? (time() + $ttl) : null);
	}

	$guard = AphrontWriteGuard::beginScopedUnguardedWrites();
	foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
	queryfx(
	$conn_w,
	'INSERT INTO %T
	(cacheKeyHash, cacheKey, cacheFormat, cacheData,
	cacheCreated, cacheExpires) VALUES %Q
	ON DUPLICATE KEY UPDATE
	cacheKey = VALUES(cacheKey),
	cacheFormat = VALUES(cacheFormat),
	cacheData = VALUES(cacheData),
	cacheCreated = VALUES(cacheCreated),
	cacheExpires = VALUES(cacheExpires)',
	$this->getTableName(),
	$chunk);
	}
	unset($guard);
	}

	return $this;
	}

	public function getKeys(array $keys) {
	$results = array();
	if ($keys) {
	$map = $this->digestKeys($keys);

	$rows = queryfx_all(
	$this->establishConnection('r'),
	'SELECT * FROM %T WHERE cacheKeyHash IN (%Ls)',
	$this->getTableName(),
	$map);
	$rows = ipull($rows, null, 'cacheKey');

	foreach ($keys as $key) {
	if (empty($rows[$key])) {
	continue;
	}

	$row = $rows[$key];

	if ($row['cacheExpires'] && ($row['cacheExpires'] < time())) {
	continue;
	}

	try {
	$results[$key] = $this->didReadValue(
	$row['cacheFormat'],
	$row['cacheData']);
	} catch (Exception $ex) {
	// Treat this as a cache miss.
	phlog($ex);
	}
	}
	}

	return $results;
	}

	public function deleteKeys(array $keys) {
	if ($keys) {
	$map = $this->digestKeys($keys);
	queryfx(
	$this->establishConnection('w'),
	'DELETE FROM %T WHERE cacheKeyHash IN (%Ls)',
	$this->getTableName(),
	$keys);
	}

	return $this;
	}

	public function destroyCache() {
	queryfx(
	$this->establishConnection('w'),
	'DELETE FROM %T',
	$this->getTableName());
	return $this;
	}


	/* -( Raw Cache Access )--------------------------------------------------- */


	public function establishConnection($mode) {
	// TODO: This is the only concrete table we have on the database right
	// now.
	return id(new PhabricatorMarkupCache())->establishConnection($mode);
	}

	public function getTableName() {
	return 'cache_general';
	}


	/* -( Implementation )----------------------------------------------------- */


	private function digestKeys(array $keys) {
	$map = array();
	foreach ($keys as $key) {
	$map[$key] = PhabricatorHash::digestForIndex($key);
	}
	return $map;
	}

	private function willWriteValue($key, $value) {
	if (!is_string($value)) {
	throw new Exception("Only strings may be written to the DB cache!");
	}

	static $can_deflate;
	if ($can_deflate === null) {
	$can_deflate = function_exists('gzdeflate') &&
	PhabricatorEnv::getEnvConfig('cache.enable-deflate');
	}

	- // If the value is larger than 1KB, we have gzdeflate(), we successfully
	- // can deflate it, and it benefits from deflation, store it deflated.
	if ($can_deflate) {
	- $len = strlen($value);
	- if ($len > 1024) {
	- $deflated = gzdeflate($value);
	- if ($deflated !== false) {
	- $deflated_len = strlen($deflated);
	- if ($deflated_len < ($len / 2)) {
	- return array(self::CACHE_FORMAT_DEFLATE, $deflated);
	- }
	- }
	+ $deflated = PhabricatorCaches::maybeDeflateData($value);
	+ if ($deflated !== null) {
	+ return array(self::CACHE_FORMAT_DEFLATE, $deflated);
	}
	}

	return array(self::CACHE_FORMAT_RAW, $value);
	}

	private function didReadValue($format, $value) {
	switch ($format) {
	case self::CACHE_FORMAT_RAW:
	return $value;
	case self::CACHE_FORMAT_DEFLATE:
	- if (!function_exists('gzinflate')) {
	- throw new Exception("No gzinflate() to read deflated cache.");
	- }
	- $value = gzinflate($value);
	- if ($value === false) {
	- throw new Exception("Failed to deflate cache.");
	- }
	- return $value;
	+ return PhabricatorCaches::inflateData($value);
	default:
	throw new Exception("Unknown cache format.");
	}
	}


	}
	diff --git a/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php b/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php
	index 3065658743..2b59f1871d 100644
	--- a/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php
	+++ b/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php
	@@ -1,48 +1,62 @@
	<?php

	final class PhabricatorHunksManagementMigrateWorkflow
	extends PhabricatorHunksManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('migrate')
	->setExamples('migrate')
	->setSynopsis(pht('Migrate hunks to modern storage.'))
	->setArguments(array());
	}

	public function execute(PhutilArgumentParser $args) {
	$saw_any_rows = false;
	$console = PhutilConsole::getConsole();

	$table = new DifferentialHunkLegacy();
	foreach (new LiskMigrationIterator($table) as $hunk) {
	$saw_any_rows = true;

	$id = $hunk->getID();
	$console->writeOut("%s\n", pht('Migrating hunk %d...', $id));

	$new_hunk = id(new DifferentialHunkModern())
	->setChangesetID($hunk->getChangesetID())
	->setOldOffset($hunk->getOldOffset())
	->setOldLen($hunk->getOldLen())
	->setNewOffset($hunk->getNewOffset())
	->setNewLen($hunk->getNewLen())
	->setChanges($hunk->getChanges())
	->setDateCreated($hunk->getDateCreated())
	->setDateModified($hunk->getDateModified());

	$hunk->openTransaction();
	$new_hunk->save();
	$hunk->delete();
	$hunk->saveTransaction();
	+
	+ $old_len = strlen($hunk->getChanges());
	+ $new_len = strlen($new_hunk->getData());
	+ if ($old_len) {
	+ $diff_len = ($old_len - $new_len);
	+ $console->writeOut(
	+ "%s\n",
	+ pht(
	+ 'Saved %s bytes (%s).',
	+ new PhutilNumber($diff_len),
	+ sprintf('%.1f%%', 100 * ($diff_len / $old_len))));
	+ }
	+
	+ break;
	}

	if ($saw_any_rows) {
	$console->writeOut("%s\n", pht('Done.'));
	} else {
	$console->writeOut("%s\n", pht('No rows to migrate.'));
	}
	}

	}
	diff --git a/src/applications/differential/storage/DifferentialHunkModern.php b/src/applications/differential/storage/DifferentialHunkModern.php
	index 50aa48e8c6..39168b3ebe 100644
	--- a/src/applications/differential/storage/DifferentialHunkModern.php
	+++ b/src/applications/differential/storage/DifferentialHunkModern.php
	@@ -1,73 +1,103 @@
	<?php

	final class DifferentialHunkModern extends DifferentialHunk {

	const DATATYPE_TEXT = 'text';
	const DATATYPE_FILE = 'file';

	const DATAFORMAT_RAW = 'byte';
	- const DATAFORMAT_DEFLATE = 'gzde';
	+ const DATAFORMAT_DEFLATED = 'gzde';

	protected $dataType;
	protected $dataEncoding;
	protected $dataFormat;
	protected $data;

	+ private $rawData;
	+
	public function getTableName() {
	return 'differential_hunk_modern';
	}

	public function getConfiguration() {
	return array(
	self::CONFIG_BINARY => array(
	'data' => true,
	),
	) + parent::getConfiguration();
	}

	public function setChanges($text) {
	+ $this->rawData = $text;
	+
	$this->dataEncoding = $this->detectEncodingForStorage($text);
	$this->dataType = self::DATATYPE_TEXT;
	$this->dataFormat = self::DATAFORMAT_RAW;
	$this->data = $text;

	return $this;
	}

	public function getChanges() {
	return $this->getUTF8StringFromStorage(
	$this->getRawData(),
	$this->getDataEncoding());
	}

	- private function getRawData() {
	+ public function save() {
	+
	$type = $this->getDataType();
	- $data = $this->getData();
	-
	- switch ($type) {
	- case self::DATATYPE_TEXT:
	- // In this storage type, the changes are stored on the object.
	- $data = $data;
	- break;
	- case self::DATATYPE_FILE:
	- default:
	- throw new Exception(
	- pht('Hunk has unsupported data type "%s"!', $type));
	+ $format = $this->getDataFormat();
	+
	+ // Before saving the data, attempt to compress it.
	+ if ($type == self::DATATYPE_TEXT) {
	+ if ($format == self::DATAFORMAT_RAW) {
	+ $data = $this->getData();
	+ $deflated = PhabricatorCaches::maybeDeflateData($data);
	+ if ($deflated !== null) {
	+ $this->data = $deflated;
	+ $this->dataFormat = self::DATAFORMAT_DEFLATED;
	+ }
	+ }
	}

	- $format = $this->getDataFormat();
	- switch ($format) {
	- case self::DATAFORMAT_RAW:
	- // In this format, the changes are stored as-is.
	- $data = $data;
	- break;
	- case self::DATAFORMAT_DEFLATE:
	- default:
	- throw new Exception(
	- pht('Hunk has unsupported data encoding "%s"!', $type));
	+ return parent::save();
	+ }
	+
	+ private function getRawData() {
	+ if ($this->rawData === null) {
	+ $type = $this->getDataType();
	+ $data = $this->getData();
	+
	+ switch ($type) {
	+ case self::DATATYPE_TEXT:
	+ // In this storage type, the changes are stored on the object.
	+ $data = $data;
	+ break;
	+ case self::DATATYPE_FILE:
	+ default:
	+ throw new Exception(
	+ pht('Hunk has unsupported data type "%s"!', $type));
	+ }
	+
	+ $format = $this->getDataFormat();
	+ switch ($format) {
	+ case self::DATAFORMAT_RAW:
	+ // In this format, the changes are stored as-is.
	+ $data = $data;
	+ break;
	+ case self::DATAFORMAT_DEFLATED:
	+ $data = PhabricatorCaches::inflateData($data);
	+ break;
	+ default:
	+ throw new Exception(
	+ pht('Hunk has unsupported data encoding "%s"!', $type));
	+ }
	+
	+ $this->rawData = $data;
	}

	- return $data;
	+ return $this->rawData;
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Sun, Jan 19, 13:53 (3 w, 2 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1125258
Default Alt Text: (20 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions