diff --git a/src/applications/files/remarkup/PhabricatorRemarkupRuleEmbedFile.php b/src/applications/files/remarkup/PhabricatorRemarkupRuleEmbedFile.php index 81a242c098..116d46111a 100644 --- a/src/applications/files/remarkup/PhabricatorRemarkupRuleEmbedFile.php +++ b/src/applications/files/remarkup/PhabricatorRemarkupRuleEmbedFile.php @@ -1,236 +1,188 @@ getEngine(); - $file = null; - if ($matches[1]) { - // TODO: This is pretty inefficient if there are a bunch of files. - // TODO: (T603) This isn't policy-aware and should be extending - // PhabricatorRemarkupRuleObject. - $file = id(new PhabricatorFile())->load($matches[1]); - } + $viewer = $engine->getConfig('viewer'); + $objects = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withIDs($ids) + ->execute(); - if (!$file) { - return $matches[0]; + $phids_key = self::KEY_EMBED_FILE_PHIDS; + $phids = $engine->getTextMetadata($phids_key, array()); + foreach (mpull($objects, 'getPHID') as $phid) { + $phids[] = $phid; } + $engine->setTextMetadata($phids_key, $phids); - $engine = $this->getEngine(); + return $objects; + } - if ($engine->isTextMode()) { - return $engine->storeText($file->getBestURI()); - } + protected function renderObjectEmbed($object, $handle, $options) { + $options = $this->getFileOptions($options) + array( + 'name' => $object->getName(), + ); + + $is_viewable_image = $object->isViewableImage(); + $is_audio = $object->isAudio(); + $force_link = ($options['layout'] == 'link'); - $phid = $file->getPHID(); + $options['viewable'] = ($is_viewable_image || $is_audio); - $token = $engine->storeText(''); - $metadata_key = self::KEY_RULE_EMBED_FILE; - $metadata = $engine->getTextMetadata($metadata_key, array()); - $bundle = array('token' => $token); + if ($is_viewable_image && !$force_link) { + return $this->renderImageFile($object, $handle, $options); + } else if ($is_audio && !$force_link) { + return $this->renderAudioFile($object, $handle, $options); + } else { + return $this->renderFileLink($object, $handle, $options); + } + } + private function getFileOptions($option_string) { $options = array( 'size' => 'thumb', 'layout' => 'left', 'float' => false, - 'name' => null, ); - if (!empty($matches[2])) { - $matches[2] = trim($matches[2], ', '); + if ($option_string) { + $option_string = trim($option_string, ', '); $parser = new PhutilSimpleOptions(); - $options = $parser->parse($matches[2]) + $options; + $options = $parser->parse($option_string) + $options; } - $file_name = coalesce($options['name'], $file->getName()); - $options['name'] = $file_name; - $is_viewable_image = $file->isViewableImage(); - $is_audio = $file->isAudio(); + return $options; + } + + private function renderImageFile( + PhabricatorFile $file, + PhabricatorObjectHandle $handle, + array $options) { + + require_celerity_resource('lightbox-attachment-css'); $attrs = array(); - if ($is_viewable_image) { - switch ((string)$options['size']) { - case 'full': - $attrs['src'] = $file->getBestURI(); - $options['image_class'] = null; - $file_data = $file->getMetadata(); - $height = idx($file_data, PhabricatorFile::METADATA_IMAGE_HEIGHT); - if ($height) { - $attrs['height'] = $height; - } - $width = idx($file_data, PhabricatorFile::METADATA_IMAGE_WIDTH); - if ($width) { - $attrs['width'] = $width; - } + $image_class = null; + switch ((string)$options['size']) { + case 'full': + $attrs += array( + 'src' => $file->getBestURI(), + 'width' => $file->getImageWidth(), + 'height' => $file->getImageHeight(), + ); + break; + case 'thumb': + default: + $attrs['src'] = $file->getPreview220URI(); + $dimensions = + PhabricatorImageTransformer::getPreviewDimensions($file, 220); + $attrs['width'] = $dimensions['sdx']; + $attrs['height'] = $dimensions['sdy']; + $image_class = 'phabricator-remarkup-embed-image'; + break; + } + + $img = phutil_tag('img', $attrs); + + $embed = javelin_tag( + 'a', + array( + 'href' => $file->getBestURI(), + 'class' => $image_class, + 'sigil' => 'lightboxable', + 'meta' => array( + 'phid' => $file->getPHID(), + 'uri' => $file->getBestURI(), + 'viewable' => true, + ), + ), + $img); + + switch ($options['layout']) { + case 'right': + case 'center': + case 'inline': + case 'left': + $layout_class = 'phabricator-remarkup-embed-layout-'.$options['layout']; + break; + default: + $layout_class = 'phabricator-remarkup-embed-layout-left'; + break; + } + + if ($options['float']) { + switch ($options['layout']) { + case 'center': + case 'inline': break; - case 'thumb': + case 'right': + $layout_class .= ' phabricator-remarkup-embed-float-right'; + break; + case 'left': default: - $attrs['src'] = $file->getPreview220URI(); - $dimensions = - PhabricatorImageTransformer::getPreviewDimensions($file, 220); - $attrs['width'] = $dimensions['sdx']; - $attrs['height'] = $dimensions['sdy']; - $options['image_class'] = 'phabricator-remarkup-embed-image'; + $layout_class .= ' phabricator-remarkup-embed-float-left'; break; } } - $bundle['attrs'] = $attrs; - $bundle['options'] = $options; - - $bundle['meta'] = array( - 'phid' => $file->getPHID(), - 'viewable' => $is_viewable_image, - 'audio' => $is_audio, - 'uri' => $file->getBestURI(), - 'dUri' => $file->getDownloadURI(), - 'name' => $options['name'], - 'mime' => $file->getMimeType(), - ); - - if ($is_audio) { - $bundle['meta'] += array( - 'autoplay' => idx($options, 'autoplay'), - 'loop' => idx($options, 'loop'), - ); - } - - $metadata[$phid][] = $bundle; - $engine->setTextMetadata($metadata_key, $metadata); - return $token; + return phutil_tag( + 'div', + array( + 'class' => $layout_class, + ), + $embed); } - public function didMarkupText() { - $engine = $this->getEngine(); - - $metadata_key = self::KEY_RULE_EMBED_FILE; - $metadata = $engine->getTextMetadata($metadata_key, array()); - - if (!$metadata) { - return; + private function renderAudioFile( + PhabricatorFile $file, + PhabricatorObjectHandle $handle, + array $options) { + + if (idx($options, 'autoplay')) { + $preload = 'auto'; + $autoplay = 'autoplay'; + } else { + $preload = 'none'; + $autoplay = null; } - $file_phids = array(); - foreach ($metadata as $phid => $bundles) { - foreach ($bundles as $data) { - - $options = $data['options']; - $meta = $data['meta']; - - $is_image = idx($meta, 'viewable'); - $is_audio = idx($meta, 'audio'); - - if ((!$is_image && !$is_audio) || $options['layout'] == 'link') { - $link = id(new PhabricatorFileLinkView()) - ->setFilePHID($meta['phid']) - ->setFileName($meta['name']) - ->setFileDownloadURI($meta['dUri']) - ->setFileViewURI($meta['uri']) - ->setFileViewable($meta['viewable']); - $embed = $link->render(); - $engine->overwriteStoredText($data['token'], $embed); - continue; - } - - if ($is_audio) { - if (idx($options, 'autoplay')) { - $preload = 'auto'; - $autoplay = 'autoplay'; - } else { - $preload = 'none'; - $autoplay = null; - } - - $link = phutil_tag( - 'audio', - array( - 'controls' => 'controls', - 'preload' => $preload, - 'autoplay' => $autoplay, - 'loop' => idx($options, 'loop') ? 'loop' : null, - ), - phutil_tag( - 'source', - array( - 'src' => $meta['uri'], - 'type' => $meta['mime'], - ))); - $engine->overwriteStoredText($data['token'], $link); - continue; - } - - require_celerity_resource('lightbox-attachment-css'); - $img = phutil_tag('img', $data['attrs']); - - $embed = javelin_tag( - 'a', - array( - 'href' => $meta['uri'], - 'class' => $options['image_class'], - 'sigil' => 'lightboxable', - 'mustcapture' => true, - 'meta' => $meta, - ), - $img); - - $layout_class = null; - switch ($options['layout']) { - case 'right': - case 'center': - case 'inline': - case 'left': - $layout_class = 'phabricator-remarkup-embed-layout-'. - $options['layout']; - break; - default: - $layout_class = 'phabricator-remarkup-embed-layout-left'; - break; - } - - if ($options['float']) { - switch ($options['layout']) { - case 'center': - case 'inline': - break; - case 'right': - $layout_class .= ' phabricator-remarkup-embed-float-right'; - break; - case 'left': - default: - $layout_class .= ' phabricator-remarkup-embed-float-left'; - break; - } - } - - if ($layout_class) { - $embed = phutil_tag( - 'div', - array( - 'class' => $layout_class, - ), - $embed); - } - - $engine->overwriteStoredText($data['token'], $embed); - } - $file_phids[] = $phid; - } - $engine->setTextMetadata(self::KEY_EMBED_FILE_PHIDS, $file_phids); - $engine->setTextMetadata($metadata_key, array()); + return phutil_tag( + 'audio', + array( + 'controls' => 'controls', + 'preload' => $preload, + 'autoplay' => $autoplay, + 'loop' => idx($options, 'loop') ? 'loop' : null, + ), + phutil_tag( + 'source', + array( + 'src' => $file->getBestURI(), + 'type' => $file->getMimeType(), + ))); + } + + private function renderFileLink( + PhabricatorFile $file, + PhabricatorObjectHandle $handle, + array $options) { + + return id(new PhabricatorFileLinkView()) + ->setFilePHID($file->getPHID()) + ->setFileName($options['name']) + ->setFileDownloadURI($file->getDownloadURI()) + ->setFileViewURI($file->getBestURI()) + ->setFileViewable($options['viewable']); } } diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php index c25bbdc835..fd50087c95 100644 --- a/src/applications/files/storage/PhabricatorFile.php +++ b/src/applications/files/storage/PhabricatorFile.php @@ -1,916 +1,930 @@ true, self::CONFIG_SERIALIZATION => array( 'metadata' => self::SERIALIZATION_JSON, ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorFilePHIDTypeFile::TYPECONST); } public function save() { if (!$this->getSecretKey()) { $this->setSecretKey($this->generateSecretKey()); } if (!$this->getMailKey()) { $this->setMailKey(Filesystem::readRandomCharacters(20)); } return parent::save(); } public static function readUploadedFileData($spec) { if (!$spec) { throw new Exception("No file was uploaded!"); } $err = idx($spec, 'error'); if ($err) { throw new PhabricatorFileUploadException($err); } $tmp_name = idx($spec, 'tmp_name'); $is_valid = @is_uploaded_file($tmp_name); if (!$is_valid) { throw new Exception("File is not an uploaded file."); } $file_data = Filesystem::readFile($tmp_name); $file_size = idx($spec, 'size'); if (strlen($file_data) != $file_size) { throw new Exception("File size disagrees with uploaded size."); } self::validateFileSize(strlen($file_data)); return $file_data; } public static function newFromPHPUpload($spec, array $params = array()) { $file_data = self::readUploadedFileData($spec); $file_name = nonempty( idx($params, 'name'), idx($spec, 'name')); $params = array( 'name' => $file_name, ) + $params; return self::newFromFileData($file_data, $params); } public static function newFromXHRUpload($data, array $params = array()) { self::validateFileSize(strlen($data)); return self::newFromFileData($data, $params); } private static function validateFileSize($size) { $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit'); if (!$limit) { return; } $limit = phabricator_parse_bytes($limit); if ($size > $limit) { throw new PhabricatorFileUploadException(-1000); } } /** * Given a block of data, try to load an existing file with the same content * if one exists. If it does not, build a new file. * * This method is generally used when we have some piece of semi-trusted data * like a diff or a file from a repository that we want to show to the user. * We can't just dump it out because it may be dangerous for any number of * reasons; instead, we need to serve it through the File abstraction so it * ends up on the CDN domain if one is configured and so on. However, if we * simply wrote a new file every time we'd potentially end up with a lot * of redundant data in file storage. * * To solve these problems, we use file storage as a cache and reuse the * same file again if we've previously written it. * * NOTE: This method unguards writes. * * @param string Raw file data. * @param dict Dictionary of file information. */ public static function buildFromFileDataOrHash( $data, array $params = array()) { $file = id(new PhabricatorFile())->loadOneWhere( 'name = %s AND contentHash = %s LIMIT 1', self::normalizeFileName(idx($params, 'name')), self::hashFileContent($data)); if (!$file) { $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $file = PhabricatorFile::newFromFileData($data, $params); unset($unguarded); } return $file; } public static function newFileFromContentHash($hash, $params) { // Check to see if a file with same contentHash exist $file = id(new PhabricatorFile())->loadOneWhere( 'contentHash = %s LIMIT 1', $hash); if ($file) { // copy storageEngine, storageHandle, storageFormat $copy_of_storage_engine = $file->getStorageEngine(); $copy_of_storage_handle = $file->getStorageHandle(); $copy_of_storage_format = $file->getStorageFormat(); $copy_of_byteSize = $file->getByteSize(); $copy_of_mimeType = $file->getMimeType(); $file_name = idx($params, 'name'); $file_name = self::normalizeFileName($file_name); $file_ttl = idx($params, 'ttl'); $authorPHID = idx($params, 'authorPHID'); $new_file = new PhabricatorFile(); $new_file->setName($file_name); $new_file->setByteSize($copy_of_byteSize); $new_file->setAuthorPHID($authorPHID); $new_file->setTtl($file_ttl); $new_file->setContentHash($hash); $new_file->setStorageEngine($copy_of_storage_engine); $new_file->setStorageHandle($copy_of_storage_handle); $new_file->setStorageFormat($copy_of_storage_format); $new_file->setMimeType($copy_of_mimeType); $new_file->copyDimensions($file); $new_file->save(); return $new_file; } return $file; } private static function buildFromFileData($data, array $params = array()) { $selector = PhabricatorEnv::newObjectFromConfig('storage.engine-selector'); if (isset($params['storageEngines'])) { $engines = $params['storageEngines']; } else { $selector = PhabricatorEnv::newObjectFromConfig( 'storage.engine-selector'); $engines = $selector->selectStorageEngines($data, $params); } assert_instances_of($engines, 'PhabricatorFileStorageEngine'); if (!$engines) { throw new Exception("No valid storage engines are available!"); } $file = new PhabricatorFile(); $data_handle = null; $engine_identifier = null; $exceptions = array(); foreach ($engines as $engine) { $engine_class = get_class($engine); try { list($engine_identifier, $data_handle) = $file->writeToEngine( $engine, $data, $params); // We stored the file somewhere so stop trying to write it to other // places. break; } catch (PhabricatorFileStorageConfigurationException $ex) { // If an engine is outright misconfigured (or misimplemented), raise // that immediately since it probably needs attention. throw $ex; } catch (Exception $ex) { phlog($ex); // If an engine doesn't work, keep trying all the other valid engines // in case something else works. $exceptions[$engine_class] = $ex; } } if (!$data_handle) { throw new PhutilAggregateException( "All storage engines failed to write file:", $exceptions); } $file_name = idx($params, 'name'); $file_name = self::normalizeFileName($file_name); $file_ttl = idx($params, 'ttl'); // If for whatever reason, authorPHID isn't passed as a param // (always the case with newFromFileDownload()), store a '' $authorPHID = idx($params, 'authorPHID'); $file->setName($file_name); $file->setByteSize(strlen($data)); $file->setAuthorPHID($authorPHID); $file->setTtl($file_ttl); $file->setContentHash(self::hashFileContent($data)); $file->setStorageEngine($engine_identifier); $file->setStorageHandle($data_handle); // TODO: This is probably YAGNI, but allows for us to do encryption or // compression later if we want. $file->setStorageFormat(self::STORAGE_FORMAT_RAW); $file->setIsExplicitUpload(idx($params, 'isExplicitUpload') ? 1 : 0); if (isset($params['mime-type'])) { $file->setMimeType($params['mime-type']); } else { $tmp = new TempFile(); Filesystem::writeFile($tmp, $data); $file->setMimeType(Filesystem::getMimeType($tmp)); } try { $file->updateDimensions(false); } catch (Exception $ex) { // Do nothing } $file->save(); return $file; } public static function newFromFileData($data, array $params = array()) { $hash = self::hashFileContent($data); $file = self::newFileFromContentHash($hash, $params); if ($file) { return $file; } return self::buildFromFileData($data, $params); } public function migrateToEngine(PhabricatorFileStorageEngine $engine) { if (!$this->getID() || !$this->getStorageHandle()) { throw new Exception( "You can not migrate a file which hasn't yet been saved."); } $data = $this->loadFileData(); $params = array( 'name' => $this->getName(), ); list($new_identifier, $new_handle) = $this->writeToEngine( $engine, $data, $params); $old_engine = $this->instantiateStorageEngine(); $old_handle = $this->getStorageHandle(); $this->setStorageEngine($new_identifier); $this->setStorageHandle($new_handle); $this->save(); $old_engine->deleteFile($old_handle); return $this; } private function writeToEngine( PhabricatorFileStorageEngine $engine, $data, array $params) { $engine_class = get_class($engine); $data_handle = $engine->writeFile($data, $params); if (!$data_handle || strlen($data_handle) > 255) { // This indicates an improperly implemented storage engine. throw new PhabricatorFileStorageConfigurationException( "Storage engine '{$engine_class}' executed writeFile() but did ". "not return a valid handle ('{$data_handle}') to the data: it ". "must be nonempty and no longer than 255 characters."); } $engine_identifier = $engine->getEngineIdentifier(); if (!$engine_identifier || strlen($engine_identifier) > 32) { throw new PhabricatorFileStorageConfigurationException( "Storage engine '{$engine_class}' returned an improper engine ". "identifier '{$engine_identifier}': it must be nonempty ". "and no longer than 32 characters."); } return array($engine_identifier, $data_handle); } public static function newFromFileDownload($uri, array $params = array()) { // Make sure we're allowed to make a request first if (!PhabricatorEnv::getEnvConfig('security.allow-outbound-http')) { throw new Exception("Outbound HTTP requests are disabled!"); } $uri = new PhutilURI($uri); $protocol = $uri->getProtocol(); switch ($protocol) { case 'http': case 'https': break; default: // Make sure we are not accessing any file:// URIs or similar. return null; } $timeout = 5; list($file_data) = id(new HTTPSFuture($uri)) ->setTimeout($timeout) ->resolvex(); $params = $params + array( 'name' => basename($uri), ); return self::newFromFileData($file_data, $params); } public static function normalizeFileName($file_name) { $pattern = "@[\\x00-\\x19#%&+!~'\$\"\/=\\\\?<> ]+@"; $file_name = preg_replace($pattern, '_', $file_name); $file_name = preg_replace('@_+@', '_', $file_name); $file_name = trim($file_name, '_'); $disallowed_filenames = array( '.' => 'dot', '..' => 'dotdot', '' => 'file', ); $file_name = idx($disallowed_filenames, $file_name, $file_name); return $file_name; } public function delete() { // We want to delete all the rows which mark this file as the transformation // of some other file (since we're getting rid of it). We also delete all // the transformations of this file, so that a user who deletes an image // doesn't need to separately hunt down and delete a bunch of thumbnails and // resizes of it. $outbound_xforms = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms( array( array( 'originalPHID' => $this->getPHID(), 'transform' => true, ), )) ->execute(); foreach ($outbound_xforms as $outbound_xform) { $outbound_xform->delete(); } $inbound_xforms = id(new PhabricatorTransformedFile())->loadAllWhere( 'transformedPHID = %s', $this->getPHID()); $this->openTransaction(); foreach ($inbound_xforms as $inbound_xform) { $inbound_xform->delete(); } $ret = parent::delete(); $this->saveTransaction(); // Check to see if other files are using storage $other_file = id(new PhabricatorFile())->loadAllWhere( 'storageEngine = %s AND storageHandle = %s AND storageFormat = %s AND id != %d LIMIT 1', $this->getStorageEngine(), $this->getStorageHandle(), $this->getStorageFormat(), $this->getID()); // If this is the only file using the storage, delete storage if (!$other_file) { $engine = $this->instantiateStorageEngine(); try { $engine->deleteFile($this->getStorageHandle()); } catch (Exception $ex) { // In the worst case, we're leaving some data stranded in a storage // engine, which is fine. phlog($ex); } } return $ret; } public static function hashFileContent($data) { return sha1($data); } public function loadFileData() { $engine = $this->instantiateStorageEngine(); $data = $engine->readFile($this->getStorageHandle()); switch ($this->getStorageFormat()) { case self::STORAGE_FORMAT_RAW: $data = $data; break; default: throw new Exception("Unknown storage format."); } return $data; } public function getViewURI() { if (!$this->getPHID()) { throw new Exception( "You must save a file before you can generate a view URI."); } $name = phutil_escape_uri($this->getName()); $path = '/file/data/'.$this->getSecretKey().'/'.$this->getPHID().'/'.$name; return PhabricatorEnv::getCDNURI($path); } public function getInfoURI() { return '/file/info/'.$this->getPHID().'/'; } public function getBestURI() { if ($this->isViewableInBrowser()) { return $this->getViewURI(); } else { return $this->getInfoURI(); } } public function getDownloadURI() { $uri = id(new PhutilURI($this->getViewURI())) ->setQueryParam('download', true); return (string) $uri; } public function getProfileThumbURI() { $path = '/file/xform/thumb-profile/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function getThumb60x45URI() { $path = '/file/xform/thumb-60x45/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function getThumb160x120URI() { $path = '/file/xform/thumb-160x120/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function getPreview140URI() { $path = '/file/xform/preview-140/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function getPreview220URI() { $path = '/file/xform/preview-220/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function getThumb220x165URI() { $path = '/file/xform/thumb-220x165/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function getThumb280x210URI() { $path = '/file/xform/thumb-280x210/'.$this->getPHID().'/' .$this->getSecretKey().'/'; return PhabricatorEnv::getCDNURI($path); } public function isViewableInBrowser() { return ($this->getViewableMimeType() !== null); } public function isViewableImage() { if (!$this->isViewableInBrowser()) { return false; } $mime_map = PhabricatorEnv::getEnvConfig('files.image-mime-types'); $mime_type = $this->getMimeType(); return idx($mime_map, $mime_type); } public function isAudio() { if (!$this->isViewableInBrowser()) { return false; } $mime_map = PhabricatorEnv::getEnvConfig('files.audio-mime-types'); $mime_type = $this->getMimeType(); return idx($mime_map, $mime_type); } public function isTransformableImage() { // NOTE: The way the 'gd' extension works in PHP is that you can install it // with support for only some file types, so it might be able to handle // PNG but not JPEG. Try to generate thumbnails for whatever we can. Setup // warns you if you don't have complete support. $matches = null; $ok = preg_match( '@^image/(gif|png|jpe?g)@', $this->getViewableMimeType(), $matches); if (!$ok) { return false; } switch ($matches[1]) { case 'jpg'; case 'jpeg': return function_exists('imagejpeg'); break; case 'png': return function_exists('imagepng'); break; case 'gif': return function_exists('imagegif'); break; default: throw new Exception('Unknown type matched as image MIME type.'); } } public static function getTransformableImageFormats() { $supported = array(); if (function_exists('imagejpeg')) { $supported[] = 'jpg'; } if (function_exists('imagepng')) { $supported[] = 'png'; } if (function_exists('imagegif')) { $supported[] = 'gif'; } return $supported; } protected function instantiateStorageEngine() { return self::buildEngine($this->getStorageEngine()); } public static function buildEngine($engine_identifier) { $engines = self::buildAllEngines(); foreach ($engines as $engine) { if ($engine->getEngineIdentifier() == $engine_identifier) { return $engine; } } throw new Exception( "Storage engine '{$engine_identifier}' could not be located!"); } public static function buildAllEngines() { $engines = id(new PhutilSymbolLoader()) ->setType('class') ->setConcreteOnly(true) ->setAncestorClass('PhabricatorFileStorageEngine') ->selectAndLoadSymbols(); $results = array(); foreach ($engines as $engine_class) { $results[] = newv($engine_class['name'], array()); } return $results; } public function getViewableMimeType() { $mime_map = PhabricatorEnv::getEnvConfig('files.viewable-mime-types'); $mime_type = $this->getMimeType(); $mime_parts = explode(';', $mime_type); $mime_type = trim(reset($mime_parts)); return idx($mime_map, $mime_type); } public function getDisplayIconForMimeType() { $mime_map = PhabricatorEnv::getEnvConfig('files.icon-mime-types'); $mime_type = $this->getMimeType(); return idx($mime_map, $mime_type, 'docs_file'); } public function validateSecretKey($key) { return ($key == $this->getSecretKey()); } public function generateSecretKey() { return Filesystem::readRandomCharacters(20); } public function updateDimensions($save = true) { if (!$this->isViewableImage()) { throw new Exception( "This file is not a viewable image."); } if (!function_exists("imagecreatefromstring")) { throw new Exception( "Cannot retrieve image information."); } $data = $this->loadFileData(); $img = imagecreatefromstring($data); if ($img === false) { throw new Exception( "Error when decoding image."); } $this->metadata[self::METADATA_IMAGE_WIDTH] = imagesx($img); $this->metadata[self::METADATA_IMAGE_HEIGHT] = imagesy($img); if ($save) { $this->save(); } return $this; } public function copyDimensions(PhabricatorFile $file) { $metadata = $file->getMetadata(); $width = idx($metadata, self::METADATA_IMAGE_WIDTH); if ($width) { $this->metadata[self::METADATA_IMAGE_WIDTH] = $width; } $height = idx($metadata, self::METADATA_IMAGE_HEIGHT); if ($height) { $this->metadata[self::METADATA_IMAGE_HEIGHT] = $height; } return $this; } public static function getMetadataName($metadata) { switch ($metadata) { case self::METADATA_IMAGE_WIDTH: $name = pht('Width'); break; case self::METADATA_IMAGE_HEIGHT: $name = pht('Height'); break; default: $name = ucfirst($metadata); break; } return $name; } /** * Load (or build) the {@class:PhabricatorFile} objects for builtin file * resources. The builtin mechanism allows files shipped with Phabricator * to be treated like normal files so that APIs do not need to special case * things like default images or deleted files. * * Builtins are located in `resources/builtin/` and identified by their * name. * * @param PhabricatorUser Viewing user. * @param list List of builtin file names. * @return dict Dictionary of named builtins. */ public static function loadBuiltins(PhabricatorUser $user, array $names) { $specs = array(); foreach ($names as $name) { $specs[] = array( 'originalPHID' => PhabricatorPHIDConstants::PHID_VOID, 'transform' => 'builtin:'.$name, ); } $files = id(new PhabricatorFileQuery()) ->setViewer($user) ->withTransforms($specs) ->execute(); $files = mpull($files, null, 'getName'); $root = dirname(phutil_get_library_root('phabricator')); $root = $root.'/resources/builtin/'; $build = array(); foreach ($names as $name) { if (isset($files[$name])) { continue; } // This is just a sanity check to prevent loading arbitrary files. if (basename($name) != $name) { throw new Exception("Invalid builtin name '{$name}'!"); } $path = $root.$name; if (!Filesystem::pathExists($path)) { throw new Exception("Builtin '{$path}' does not exist!"); } $data = Filesystem::readFile($path); $params = array( 'name' => $name, 'ttl' => time() + (60 * 60 * 24 * 7), ); $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $file = PhabricatorFile::newFromFileData($data, $params); $xform = id(new PhabricatorTransformedFile()) ->setOriginalPHID(PhabricatorPHIDConstants::PHID_VOID) ->setTransform('builtin:'.$name) ->setTransformedPHID($file->getPHID()) ->save(); unset($unguarded); $file->attachObjectPHIDs(array()); $file->attachObjects(array()); $files[$name] = $file; } return $files; } /** * Convenience wrapper for @{method:loadBuiltins}. * * @param PhabricatorUser Viewing user. * @param string Single builtin name to load. * @return PhabricatorFile Corresponding builtin file. */ public static function loadBuiltin(PhabricatorUser $user, $name) { return idx(self::loadBuiltins($user, array($name)), $name); } public function getObjects() { return $this->assertAttached($this->objects); } public function attachObjects(array $objects) { $this->objects = $objects; return $this; } public function getObjectPHIDs() { return $this->assertAttached($this->objectPHIDs); } public function attachObjectPHIDs(array $object_phids) { $this->objectPHIDs = $object_phids; return $this; } + public function getImageHeight() { + if (!$this->isViewableImage()) { + return null; + } + return idx($this->metadata, self::METADATA_IMAGE_HEIGHT); + } + + public function getImageWidth() { + if (!$this->isViewableImage()) { + return null; + } + return idx($this->metadata, self::METADATA_IMAGE_WIDTH); + } + /* -( PhabricatorPolicyInterface Implementation )-------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { // TODO: Implement proper per-object policies. return PhabricatorPolicies::POLICY_USER; } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { $viewer_phid = $viewer->getPHID(); if ($viewer_phid) { if ($this->getAuthorPHID() == $viewer_phid) { return true; } } switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: // If you can see any object this file is attached to, you can see // the file. return (count($this->getObjects()) > 0); } return false; } public function describeAutomaticCapability($capability) { $out = array(); $out[] = pht('The user who uploaded a file can always view and edit it.'); switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: $out[] = pht( 'Files attached to objects are visible to users who can view '. 'those objects.'); break; } return $out; } /* -( PhabricatorSubscribableInterface Implementation )-------------------- */ public function isAutomaticallySubscribed($phid) { return ($this->authorPHID == $phid); } /* -( PhabricatorTokenReceiverInterface )---------------------------------- */ public function getUsersToNotifyOfTokenGiven() { return array( $this->getAuthorPHID(), ); } } diff --git a/src/infrastructure/markup/PhabricatorMarkupEngine.php b/src/infrastructure/markup/PhabricatorMarkupEngine.php index 1f1fb88f95..f909c81c07 100644 --- a/src/infrastructure/markup/PhabricatorMarkupEngine.php +++ b/src/infrastructure/markup/PhabricatorMarkupEngine.php @@ -1,566 +1,566 @@ addObject($comment, $field); * } * * Now, call @{method:process} to perform the actual cache/rendering * step. This is a heavyweight call which does batched data access and * transforms the markup into output. * * $engine->process(); * * Finally, do something with the results: * * $results = array(); * foreach ($comments as $comment) { * $results[] = $engine->getOutput($comment, $field); * } * * If you have a single object to render, you can use the convenience method * @{method:renderOneObject}. * * @task markup Markup Pipeline * @task engine Engine Construction */ final class PhabricatorMarkupEngine { private $objects = array(); private $viewer; - private $version = 7; + private $version = 8; /* -( Markup Pipeline )---------------------------------------------------- */ /** * Convenience method for pushing a single object through the markup * pipeline. * * @param PhabricatorMarkupInterface The object to render. * @param string The field to render. * @param PhabricatorUser User viewing the markup. * @return string Marked up output. * @task markup */ public static function renderOneObject( PhabricatorMarkupInterface $object, $field, PhabricatorUser $viewer) { return id(new PhabricatorMarkupEngine()) ->setViewer($viewer) ->addObject($object, $field) ->process() ->getOutput($object, $field); } /** * Queue an object for markup generation when @{method:process} is * called. You can retrieve the output later with @{method:getOutput}. * * @param PhabricatorMarkupInterface The object to render. * @param string The field to render. * @return this * @task markup */ public function addObject(PhabricatorMarkupInterface $object, $field) { $key = $this->getMarkupFieldKey($object, $field); $this->objects[$key] = array( 'object' => $object, 'field' => $field, ); return $this; } /** * Process objects queued with @{method:addObject}. You can then retrieve * the output with @{method:getOutput}. * * @return this * @task markup */ public function process() { $keys = array(); foreach ($this->objects as $key => $info) { if (!isset($info['markup'])) { $keys[] = $key; } } if (!$keys) { return; } $objects = array_select_keys($this->objects, $keys); // Build all the markup engines. We need an engine for each field whether // we have a cache or not, since we still need to postprocess the cache. $engines = array(); foreach ($objects as $key => $info) { $engines[$key] = $info['object']->newMarkupEngine($info['field']); $engines[$key]->setConfig('viewer', $this->viewer); } // Load or build the preprocessor caches. $blocks = $this->loadPreprocessorCaches($engines, $objects); $blocks = mpull($blocks, 'getCacheData'); $this->engineCaches = $blocks; // Finalize the output. foreach ($objects as $key => $info) { $engine = $engines[$key]; $field = $info['field']; $object = $info['object']; $output = $engine->postprocessText($blocks[$key]); $output = $object->didMarkupText($field, $output, $engine); $this->objects[$key]['output'] = $output; } return $this; } /** * Get the output of markup processing for a field queued with * @{method:addObject}. Before you can call this method, you must call * @{method:process}. * * @param PhabricatorMarkupInterface The object to retrieve. * @param string The field to retrieve. * @return string Processed output. * @task markup */ public function getOutput(PhabricatorMarkupInterface $object, $field) { $key = $this->getMarkupFieldKey($object, $field); $this->requireKeyProcessed($key); return $this->objects[$key]['output']; } /** * Retrieve engine metadata for a given field. * * @param PhabricatorMarkupInterface The object to retrieve. * @param string The field to retrieve. * @param string The engine metadata field to retrieve. * @param wild Optional default value. * @task markup */ public function getEngineMetadata( PhabricatorMarkupInterface $object, $field, $metadata_key, $default = null) { $key = $this->getMarkupFieldKey($object, $field); $this->requireKeyProcessed($key); return idx($this->engineCaches[$key]['metadata'], $metadata_key, $default); } /** * @task markup */ private function requireKeyProcessed($key) { if (empty($this->objects[$key])) { throw new Exception( "Call addObject() before using results (key = '{$key}')."); } if (!isset($this->objects[$key]['output'])) { throw new Exception( "Call process() before using results."); } } /** * @task markup */ private function getMarkupFieldKey( PhabricatorMarkupInterface $object, $field) { return $object->getMarkupFieldKey($field).'@'.$this->version; } /** * @task markup */ private function loadPreprocessorCaches(array $engines, array $objects) { $blocks = array(); $use_cache = array(); foreach ($objects as $key => $info) { if ($info['object']->shouldUseMarkupCache($info['field'])) { $use_cache[$key] = true; } } if ($use_cache) { try { $blocks = id(new PhabricatorMarkupCache())->loadAllWhere( 'cacheKey IN (%Ls)', array_keys($use_cache)); $blocks = mpull($blocks, null, 'getCacheKey'); } catch (Exception $ex) { phlog($ex); } } foreach ($objects as $key => $info) { if (isset($blocks[$key])) { // If we already have a preprocessing cache, we don't need to rebuild // it. continue; } $text = $info['object']->getMarkupText($info['field']); $data = $engines[$key]->preprocessText($text); // NOTE: This is just debugging information to help sort out cache issues. // If one machine is misconfigured and poisoning caches you can use this // field to hunt it down. $metadata = array( 'host' => php_uname('n'), ); $blocks[$key] = id(new PhabricatorMarkupCache()) ->setCacheKey($key) ->setCacheData($data) ->setMetadata($metadata); if (isset($use_cache[$key])) { // This is just filling a cache and always safe, even on a read pathway. $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $blocks[$key]->replace(); unset($unguarded); } } return $blocks; } /** * Set the viewing user. Used to implement object permissions. * * @param PhabricatorUser The viewing user. * @return this * @task markup */ public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } /* -( Engine Construction )------------------------------------------------ */ /** * @task engine */ public static function newManiphestMarkupEngine() { return self::newMarkupEngine(array( )); } /** * @task engine */ public static function newPhrictionMarkupEngine() { return self::newMarkupEngine(array( 'header.generate-toc' => true, )); } /** * @task engine */ public static function newPhameMarkupEngine() { return self::newMarkupEngine(array( 'macros' => false, )); } /** * @task engine */ public static function newFeedMarkupEngine() { return self::newMarkupEngine( array( 'macros' => false, 'youtube' => false, )); } /** * @task engine */ public static function newDifferentialMarkupEngine(array $options = array()) { return self::newMarkupEngine(array( 'custom-inline' => PhabricatorEnv::getEnvConfig( 'differential.custom-remarkup-rules'), 'custom-block' => PhabricatorEnv::getEnvConfig( 'differential.custom-remarkup-block-rules'), 'differential.diff' => idx($options, 'differential.diff'), )); } /** * @task engine */ public static function newDiffusionMarkupEngine(array $options = array()) { return self::newMarkupEngine(array( )); } /** * @task engine */ public static function getEngine($ruleset = 'default') { static $engines = array(); if (isset($engines[$ruleset])) { return $engines[$ruleset]; } $engine = null; switch ($ruleset) { case 'default': $engine = self::newMarkupEngine(array()); break; case 'diviner': $engine = self::newMarkupEngine(array()); $engine->setConfig('preserve-linebreaks', false); // $engine->setConfig('diviner.renderer', new DivinerDefaultRenderer()); $engine->setConfig('header.generate-toc', true); break; default: throw new Exception("Unknown engine ruleset: {$ruleset}!"); } $engines[$ruleset] = $engine; return $engine; } /** * @task engine */ private static function getMarkupEngineDefaultConfiguration() { return array( 'pygments' => PhabricatorEnv::getEnvConfig('pygments.enabled'), 'youtube' => PhabricatorEnv::getEnvConfig( 'remarkup.enable-embedded-youtube'), 'custom-inline' => array(), 'custom-block' => array(), 'differential.diff' => null, 'header.generate-toc' => false, 'macros' => true, 'uri.allowed-protocols' => PhabricatorEnv::getEnvConfig( 'uri.allowed-protocols'), 'syntax-highlighter.engine' => PhabricatorEnv::getEnvConfig( 'syntax-highlighter.engine'), 'preserve-linebreaks' => true, ); } /** * @task engine */ public static function newMarkupEngine(array $options) { $options += self::getMarkupEngineDefaultConfiguration(); $engine = new PhutilRemarkupEngine(); $engine->setConfig('preserve-linebreaks', $options['preserve-linebreaks']); $engine->setConfig('pygments.enabled', $options['pygments']); $engine->setConfig( 'uri.allowed-protocols', $options['uri.allowed-protocols']); $engine->setConfig('differential.diff', $options['differential.diff']); $engine->setConfig('header.generate-toc', $options['header.generate-toc']); $engine->setConfig( 'syntax-highlighter.engine', $options['syntax-highlighter.engine']); $rules = array(); $rules[] = new PhutilRemarkupRuleEscapeRemarkup(); $rules[] = new PhutilRemarkupRuleMonospace(); $custom_rule_classes = $options['custom-inline']; if ($custom_rule_classes) { foreach ($custom_rule_classes as $custom_rule_class) { $rules[] = newv($custom_rule_class, array()); } } $rules[] = new PhutilRemarkupRuleDocumentLink(); if ($options['youtube']) { $rules[] = new PhabricatorRemarkupRuleYoutube(); } $applications = PhabricatorApplication::getAllInstalledApplications(); foreach ($applications as $application) { foreach ($application->getRemarkupRules() as $rule) { $rules[] = $rule; } } $rules[] = new PhutilRemarkupRuleHyperlink(); if ($options['macros']) { $rules[] = new PhabricatorRemarkupRuleImageMacro(); $rules[] = new PhabricatorRemarkupRuleMeme(); } $rules[] = new PhutilRemarkupRuleBold(); $rules[] = new PhutilRemarkupRuleItalic(); $rules[] = new PhutilRemarkupRuleDel(); $blocks = array(); $blocks[] = new PhutilRemarkupEngineRemarkupQuotesBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupLiteralBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupHeaderBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupHorizontalRuleBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupListBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupCodeBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupNoteBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupTableBlockRule(); $blocks[] = new PhutilRemarkupEngineRemarkupSimpleTableBlockRule(); $custom_block_rule_classes = $options['custom-block']; if ($custom_block_rule_classes) { foreach ($custom_block_rule_classes as $custom_block_rule_class) { $blocks[] = newv($custom_block_rule_class, array()); } } $blocks[] = new PhutilRemarkupEngineRemarkupDefaultBlockRule(); foreach ($blocks as $block) { $block->setMarkupRules($rules); } $engine->setBlockRules($blocks); return $engine; } public static function extractPHIDsFromMentions(array $content_blocks) { $mentions = array(); $engine = self::newDifferentialMarkupEngine(); $engine->setConfig('viewer', PhabricatorUser::getOmnipotentUser()); foreach ($content_blocks as $content_block) { $engine->markupText($content_block); $phids = $engine->getTextMetadata( PhabricatorRemarkupRuleMention::KEY_MENTIONED, array()); $mentions += $phids; } return $mentions; } public static function extractFilePHIDsFromEmbeddedFiles( array $content_blocks) { $files = array(); $engine = self::newDifferentialMarkupEngine(); $engine->setConfig('viewer', PhabricatorUser::getOmnipotentUser()); foreach ($content_blocks as $content_block) { $engine->markupText($content_block); $ids = $engine->getTextMetadata( PhabricatorRemarkupRuleEmbedFile::KEY_EMBED_FILE_PHIDS, array()); $files += $ids; } return $files; } /** * Produce a corpus summary, in a way that shortens the underlying text * without truncating it somewhere awkward. * * TODO: We could do a better job of this. * * @param string Remarkup corpus to summarize. * @return string Summarized corpus. */ public static function summarize($corpus) { // Major goals here are: // - Don't split in the middle of a character (utf-8). // - Don't split in the middle of, e.g., **bold** text, since // we end up with hanging '**' in the summary. // - Try not to pick an image macro, header, embedded file, etc. // - Hopefully don't return too much text. We don't explicitly limit // this right now. $blocks = preg_split("/\n *\n\s*/", trim($corpus)); $best = null; foreach ($blocks as $block) { // This is a test for normal spaces in the block, i.e. a heuristic to // distinguish standard paragraphs from things like image macros. It may // not work well for non-latin text. We prefer to summarize with a // paragraph of normal words over an image macro, if possible. $has_space = preg_match('/\w\s\w/', $block); // This is a test to find embedded images and headers. We prefer to // summarize with a normal paragraph over a header or an embedded object, // if possible. $has_embed = preg_match('/^[{=]/', $block); if ($has_space && !$has_embed) { // This seems like a good summary, so return it. return $block; } if (!$best) { // This is the first block we found; if everything is garbage just // use the first block. $best = $block; } } return $best; } }