Page MenuHomePhorge
Files F2890641

No OneTemporary
Actions

View Options

diff --git a/src/applications/cache/PhabricatorCaches.php b/src/applications/cache/PhabricatorCaches.php
index 860abcb130..f4eb7a918c 100644
--- a/src/applications/cache/PhabricatorCaches.php
+++ b/src/applications/cache/PhabricatorCaches.php
@@ -1,286 +1,347 @@
<?php
/**
* @task immutable Immutable Cache
* @task setup Setup Cache
+ * @task compress Compression
*/
final class PhabricatorCaches {
public static function getNamespace() {
return PhabricatorEnv::getEnvConfig('phabricator.cache-namespace');
}
private static function newStackFromCaches(array $caches) {
$caches = self::addNamespaceToCaches($caches);
$caches = self::addProfilerToCaches($caches);
return id(new PhutilKeyValueCacheStack())
->setCaches($caches);
}
/* -( Local Cache )-------------------------------------------------------- */
/**
* Gets an immutable cache stack.
*
* This stack trades mutability away for improved performance. Normally, it is
* APC + DB.
*
* In the general case with multiple web frontends, this stack can not be
* cleared, so it is only appropriate for use if the value of a given key is
* permanent and immutable.
*
* @return PhutilKeyValueCacheStack Best immutable stack available.
* @task immutable
*/
public static function getImmutableCache() {
static $cache;
if (!$cache) {
$caches = self::buildImmutableCaches();
$cache = self::newStackFromCaches($caches);
}
return $cache;
}
/**
* Build the immutable cache stack.
*
* @return list<PhutilKeyValueCache> List of caches.
* @task immutable
*/
private static function buildImmutableCaches() {
$caches = array();
$apc = new PhutilKeyValueCacheAPC();
if ($apc->isAvailable()) {
$caches[] = $apc;
}
$caches[] = new PhabricatorKeyValueDatabaseCache();
return $caches;
}
/* -( Repository Graph Cache )--------------------------------------------- */
public static function getRepositoryGraphL1Cache() {
static $cache;
if (!$cache) {
$caches = self::buildRepositoryGraphL1Caches();
$cache = self::newStackFromCaches($caches);
}
return $cache;
}
private static function buildRepositoryGraphL1Caches() {
$caches = array();
$request = new PhutilKeyValueCacheInRequest();
$request->setLimit(32);
$caches[] = $request;
$apc = new PhutilKeyValueCacheAPC();
if ($apc->isAvailable()) {
$caches[] = $apc;
}
return $caches;
}
public static function getRepositoryGraphL2Cache() {
static $cache;
if (!$cache) {
$caches = self::buildRepositoryGraphL2Caches();
$cache = self::newStackFromCaches($caches);
}
return $cache;
}
private static function buildRepositoryGraphL2Caches() {
$caches = array();
$caches[] = new PhabricatorKeyValueDatabaseCache();
return $caches;
}
/* -( Setup Cache )-------------------------------------------------------- */
/**
* Highly specialized cache for performing setup checks. We use this cache
* to determine if we need to run expensive setup checks when the page
* loads. Without it, we would need to run these checks every time.
*
* Normally, this cache is just APC. In the absence of APC, this cache
* degrades into a slow, quirky on-disk cache.
*
* NOTE: Do not use this cache for anything else! It is not a general-purpose
* cache!
*
* @return PhutilKeyValueCacheStack Most qualified available cache stack.
* @task setup
*/
public static function getSetupCache() {
static $cache;
if (!$cache) {
$caches = self::buildSetupCaches();
$cache = self::newStackFromCaches($caches);
}
return $cache;
}
/**
* @task setup
*/
private static function buildSetupCaches() {
// In most cases, we should have APC. This is an ideal cache for our
// purposes -- it's fast and empties on server restart.
$apc = new PhutilKeyValueCacheAPC();
if ($apc->isAvailable()) {
return array($apc);
}
// If we don't have APC, build a poor approximation on disk. This is still
// much better than nothing; some setup steps are quite slow.
$disk_path = self::getSetupCacheDiskCachePath();
if ($disk_path) {
$disk = new PhutilKeyValueCacheOnDisk();
$disk->setCacheFile($disk_path);
$disk->setWait(0.1);
if ($disk->isAvailable()) {
return array($disk);
}
}
return array();
}
/**
* @task setup
*/
private static function getSetupCacheDiskCachePath() {
// The difficulty here is in choosing a path which will change on server
// restart (we MUST have this property), but as rarely as possible
// otherwise (we desire this property to give the cache the best hit rate
// we can).
// In some setups, the parent PID is more stable and longer-lived that the
// PID (e.g., under apache, our PID will be a worker while the ppid will
// be the main httpd process). If we're confident we're running under such
// a setup, we can try to use the PPID as the basis for our cache instead
// of our own PID.
$use_ppid = false;
switch (php_sapi_name()) {
case 'cli-server':
// This is the PHP5.4+ built-in webserver. We should use the pid
// (the server), not the ppid (probably a shell or something).
$use_ppid = false;
break;
case 'fpm-fcgi':
// We should be safe to use PPID here.
$use_ppid = true;
break;
case 'apache2handler':
// We're definitely safe to use the PPID.
$use_ppid = true;
break;
}
$pid_basis = getmypid();
if ($use_ppid) {
if (function_exists('posix_getppid')) {
$parent_pid = posix_getppid();
// On most systems, normal processes can never have PIDs lower than 100,
// so something likely went wrong if we we get one of these.
if ($parent_pid > 100) {
$pid_basis = $parent_pid;
}
}
}
// If possible, we also want to know when the process launched, so we can
// drop the cache if a process restarts but gets the same PID an earlier
// process had. "/proc" is not available everywhere (e.g., not on OSX), but
// check if we have it.
$epoch_basis = null;
$stat = @stat("/proc/{$pid_basis}");
if ($stat !== false) {
$epoch_basis = $stat['ctime'];
}
$tmp_dir = sys_get_temp_dir();
$tmp_path = $tmp_dir.DIRECTORY_SEPARATOR.'phabricator-setup';
if (!file_exists($tmp_path)) {
@mkdir($tmp_path);
}
$is_ok = self::testTemporaryDirectory($tmp_path);
if (!$is_ok) {
$tmp_path = $tmp_dir;
$is_ok = self::testTemporaryDirectory($tmp_path);
if (!$is_ok) {
// We can't find anywhere to write the cache, so just bail.
return null;
}
}
$tmp_name = 'setup-'.$pid_basis;
if ($epoch_basis) {
$tmp_name .= '.'.$epoch_basis;
}
$tmp_name .= '.cache';
return $tmp_path.DIRECTORY_SEPARATOR.$tmp_name;
}
/**
* @task setup
*/
private static function testTemporaryDirectory($dir) {
if (!@file_exists($dir)) {
return false;
}
if (!@is_dir($dir)) {
return false;
}
if (!@is_writable($dir)) {
return false;
}
return true;
}
private static function addProfilerToCaches(array $caches) {
foreach ($caches as $key => $cache) {
$pcache = new PhutilKeyValueCacheProfiler($cache);
$pcache->setProfiler(PhutilServiceProfiler::getInstance());
$caches[$key] = $pcache;
}
return $caches;
}
private static function addNamespaceToCaches(array $caches) {
$namespace = PhabricatorCaches::getNamespace();
if (!$namespace) {
return $caches;
}
foreach ($caches as $key => $cache) {
$ncache = new PhutilKeyValueCacheNamespace($cache);
$ncache->setNamespace($namespace);
$caches[$key] = $ncache;
}
return $caches;
}
+
+ /**
+ * Deflate a value, if deflation is available and has an impact.
+ *
+ * If the value is larger than 1KB, we have `gzdeflate()`, we successfully
+ * can deflate it, and it benefits from deflation, we deflate it. Otherwise
+ * we leave it as-is.
+ *
+ * Data can later be inflated with @{method:inflateData}.
+ *
+ * @param string String to attempt to deflate.
+ * @return string|null Deflated string, or null if it was not deflated.
+ * @task compress
+ */
+ public static function maybeDeflateData($value) {
+ $len = strlen($value);
+ if ($len <= 1024) {
+ return null;
+ }
+
+ if (!function_exists('gzdeflate')) {
+ return null;
+ }
+
+ $deflated = gzdeflate($value);
+ if ($deflated === false) {
+ return null;
+ }
+
+ $deflated_len = strlen($deflated);
+ if ($deflated_len >= ($len / 2)) {
+ return null;
+ }
+
+ return $deflated;
+ }
+
+
+ /**
+ * Inflate data previously deflated by @{method:maybeDeflateData}.
+ *
+ * @param string Deflated data, from @{method:maybeDeflateData}.
+ * @return string Original, uncompressed data.
+ * @task compress
+ */
+ public static function inflateData($value) {
+ if (!function_exists('gzinflate')) {
+ throw new Exception(
+ pht('gzinflate() is not available; unable to read deflated data!'));
+ }
+
+ $value = gzinflate($value);
+ if ($value === false) {
+ throw new Exception(pht('Failed to inflate data!'));
+ }
+
+ return $value;
+ }
+
+
}
diff --git a/src/applications/cache/PhabricatorKeyValueDatabaseCache.php b/src/applications/cache/PhabricatorKeyValueDatabaseCache.php
index 822b90ffcf..aa641ab703 100644
--- a/src/applications/cache/PhabricatorKeyValueDatabaseCache.php
+++ b/src/applications/cache/PhabricatorKeyValueDatabaseCache.php
@@ -1,185 +1,170 @@
<?php
final class PhabricatorKeyValueDatabaseCache
extends PhutilKeyValueCache {
const CACHE_FORMAT_RAW = 'raw';
const CACHE_FORMAT_DEFLATE = 'deflate';
public function setKeys(array $keys, $ttl = null) {
if ($keys) {
$map = $this->digestKeys(array_keys($keys));
$conn_w = $this->establishConnection('w');
$sql = array();
foreach ($map as $key => $hash) {
$value = $keys[$key];
list($format, $storage_value) = $this->willWriteValue($key, $value);
$sql[] = qsprintf(
$conn_w,
'(%s, %s, %s, %B, %d, %nd)',
$hash,
$key,
$format,
$storage_value,
time(),
$ttl ? (time() + $ttl) : null);
}
$guard = AphrontWriteGuard::beginScopedUnguardedWrites();
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn_w,
'INSERT INTO %T
(cacheKeyHash, cacheKey, cacheFormat, cacheData,
cacheCreated, cacheExpires) VALUES %Q
ON DUPLICATE KEY UPDATE
cacheKey = VALUES(cacheKey),
cacheFormat = VALUES(cacheFormat),
cacheData = VALUES(cacheData),
cacheCreated = VALUES(cacheCreated),
cacheExpires = VALUES(cacheExpires)',
$this->getTableName(),
$chunk);
}
unset($guard);
}
return $this;
}
public function getKeys(array $keys) {
$results = array();
if ($keys) {
$map = $this->digestKeys($keys);
$rows = queryfx_all(
$this->establishConnection('r'),
'SELECT * FROM %T WHERE cacheKeyHash IN (%Ls)',
$this->getTableName(),
$map);
$rows = ipull($rows, null, 'cacheKey');
foreach ($keys as $key) {
if (empty($rows[$key])) {
continue;
}
$row = $rows[$key];
if ($row['cacheExpires'] && ($row['cacheExpires'] < time())) {
continue;
}
try {
$results[$key] = $this->didReadValue(
$row['cacheFormat'],
$row['cacheData']);
} catch (Exception $ex) {
// Treat this as a cache miss.
phlog($ex);
}
}
}
return $results;
}
public function deleteKeys(array $keys) {
if ($keys) {
$map = $this->digestKeys($keys);
queryfx(
$this->establishConnection('w'),
'DELETE FROM %T WHERE cacheKeyHash IN (%Ls)',
$this->getTableName(),
$keys);
}
return $this;
}
public function destroyCache() {
queryfx(
$this->establishConnection('w'),
'DELETE FROM %T',
$this->getTableName());
return $this;
}
/* -( Raw Cache Access )--------------------------------------------------- */
public function establishConnection($mode) {
// TODO: This is the only concrete table we have on the database right
// now.
return id(new PhabricatorMarkupCache())->establishConnection($mode);
}
public function getTableName() {
return 'cache_general';
}
/* -( Implementation )----------------------------------------------------- */
private function digestKeys(array $keys) {
$map = array();
foreach ($keys as $key) {
$map[$key] = PhabricatorHash::digestForIndex($key);
}
return $map;
}
private function willWriteValue($key, $value) {
if (!is_string($value)) {
throw new Exception("Only strings may be written to the DB cache!");
}
static $can_deflate;
if ($can_deflate === null) {
$can_deflate = function_exists('gzdeflate') &&
PhabricatorEnv::getEnvConfig('cache.enable-deflate');
}
- // If the value is larger than 1KB, we have gzdeflate(), we successfully
- // can deflate it, and it benefits from deflation, store it deflated.
if ($can_deflate) {
- $len = strlen($value);
- if ($len > 1024) {
- $deflated = gzdeflate($value);
- if ($deflated !== false) {
- $deflated_len = strlen($deflated);
- if ($deflated_len < ($len / 2)) {
- return array(self::CACHE_FORMAT_DEFLATE, $deflated);
- }
- }
+ $deflated = PhabricatorCaches::maybeDeflateData($value);
+ if ($deflated !== null) {
+ return array(self::CACHE_FORMAT_DEFLATE, $deflated);
}
}
return array(self::CACHE_FORMAT_RAW, $value);
}
private function didReadValue($format, $value) {
switch ($format) {
case self::CACHE_FORMAT_RAW:
return $value;
case self::CACHE_FORMAT_DEFLATE:
- if (!function_exists('gzinflate')) {
- throw new Exception("No gzinflate() to read deflated cache.");
- }
- $value = gzinflate($value);
- if ($value === false) {
- throw new Exception("Failed to deflate cache.");
- }
- return $value;
+ return PhabricatorCaches::inflateData($value);
default:
throw new Exception("Unknown cache format.");
}
}
}
diff --git a/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php b/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php
index 3065658743..2b59f1871d 100644
--- a/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php
+++ b/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php
@@ -1,48 +1,62 @@
<?php
final class PhabricatorHunksManagementMigrateWorkflow
extends PhabricatorHunksManagementWorkflow {
protected function didConstruct() {
$this
->setName('migrate')
->setExamples('**migrate**')
->setSynopsis(pht('Migrate hunks to modern storage.'))
->setArguments(array());
}
public function execute(PhutilArgumentParser $args) {
$saw_any_rows = false;
$console = PhutilConsole::getConsole();
$table = new DifferentialHunkLegacy();
foreach (new LiskMigrationIterator($table) as $hunk) {
$saw_any_rows = true;
$id = $hunk->getID();
$console->writeOut("%s\n", pht('Migrating hunk %d...', $id));
$new_hunk = id(new DifferentialHunkModern())
->setChangesetID($hunk->getChangesetID())
->setOldOffset($hunk->getOldOffset())
->setOldLen($hunk->getOldLen())
->setNewOffset($hunk->getNewOffset())
->setNewLen($hunk->getNewLen())
->setChanges($hunk->getChanges())
->setDateCreated($hunk->getDateCreated())
->setDateModified($hunk->getDateModified());
$hunk->openTransaction();
$new_hunk->save();
$hunk->delete();
$hunk->saveTransaction();
+
+ $old_len = strlen($hunk->getChanges());
+ $new_len = strlen($new_hunk->getData());
+ if ($old_len) {
+ $diff_len = ($old_len - $new_len);
+ $console->writeOut(
+ "%s\n",
+ pht(
+ 'Saved %s bytes (%s).',
+ new PhutilNumber($diff_len),
+ sprintf('%.1f%%', 100 * ($diff_len / $old_len))));
+ }
+
+ break;
}
if ($saw_any_rows) {
$console->writeOut("%s\n", pht('Done.'));
} else {
$console->writeOut("%s\n", pht('No rows to migrate.'));
}
}
}
diff --git a/src/applications/differential/storage/DifferentialHunkModern.php b/src/applications/differential/storage/DifferentialHunkModern.php
index 50aa48e8c6..39168b3ebe 100644
--- a/src/applications/differential/storage/DifferentialHunkModern.php
+++ b/src/applications/differential/storage/DifferentialHunkModern.php
@@ -1,73 +1,103 @@
<?php
final class DifferentialHunkModern extends DifferentialHunk {
const DATATYPE_TEXT = 'text';
const DATATYPE_FILE = 'file';
const DATAFORMAT_RAW = 'byte';
- const DATAFORMAT_DEFLATE = 'gzde';
+ const DATAFORMAT_DEFLATED = 'gzde';
protected $dataType;
protected $dataEncoding;
protected $dataFormat;
protected $data;
+ private $rawData;
+
public function getTableName() {
return 'differential_hunk_modern';
}
public function getConfiguration() {
return array(
self::CONFIG_BINARY => array(
'data' => true,
),
) + parent::getConfiguration();
}
public function setChanges($text) {
+ $this->rawData = $text;
+
$this->dataEncoding = $this->detectEncodingForStorage($text);
$this->dataType = self::DATATYPE_TEXT;
$this->dataFormat = self::DATAFORMAT_RAW;
$this->data = $text;
return $this;
}
public function getChanges() {
return $this->getUTF8StringFromStorage(
$this->getRawData(),
$this->getDataEncoding());
}
- private function getRawData() {
+ public function save() {
+
$type = $this->getDataType();
- $data = $this->getData();
-
- switch ($type) {
- case self::DATATYPE_TEXT:
- // In this storage type, the changes are stored on the object.
- $data = $data;
- break;
- case self::DATATYPE_FILE:
- default:
- throw new Exception(
- pht('Hunk has unsupported data type "%s"!', $type));
+ $format = $this->getDataFormat();
+
+ // Before saving the data, attempt to compress it.
+ if ($type == self::DATATYPE_TEXT) {
+ if ($format == self::DATAFORMAT_RAW) {
+ $data = $this->getData();
+ $deflated = PhabricatorCaches::maybeDeflateData($data);
+ if ($deflated !== null) {
+ $this->data = $deflated;
+ $this->dataFormat = self::DATAFORMAT_DEFLATED;
+ }
+ }
}
- $format = $this->getDataFormat();
- switch ($format) {
- case self::DATAFORMAT_RAW:
- // In this format, the changes are stored as-is.
- $data = $data;
- break;
- case self::DATAFORMAT_DEFLATE:
- default:
- throw new Exception(
- pht('Hunk has unsupported data encoding "%s"!', $type));
+ return parent::save();
+ }
+
+ private function getRawData() {
+ if ($this->rawData === null) {
+ $type = $this->getDataType();
+ $data = $this->getData();
+
+ switch ($type) {
+ case self::DATATYPE_TEXT:
+ // In this storage type, the changes are stored on the object.
+ $data = $data;
+ break;
+ case self::DATATYPE_FILE:
+ default:
+ throw new Exception(
+ pht('Hunk has unsupported data type "%s"!', $type));
+ }
+
+ $format = $this->getDataFormat();
+ switch ($format) {
+ case self::DATAFORMAT_RAW:
+ // In this format, the changes are stored as-is.
+ $data = $data;
+ break;
+ case self::DATAFORMAT_DEFLATED:
+ $data = PhabricatorCaches::inflateData($data);
+ break;
+ default:
+ throw new Exception(
+ pht('Hunk has unsupported data encoding "%s"!', $type));
+ }
+
+ $this->rawData = $data;
}
- return $data;
+ return $this->rawData;
}
}

File Metadata

Mime Type
text/x-diff
Expires
Sun, Jan 19, 13:53 (3 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1125258
Default Alt Text
(20 KB)

Event Timeline

Content licensed under Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA) unless otherwise noted; code licensed under Apache 2.0 or other open source licenses. · CC BY-SA 4.0 · Apache 2.0