Page MenuHomePhorge
Files F2895696

No OneTemporary
Actions

View Options

diff --git a/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php b/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
index cd0ed346fc..154c266da2 100644
--- a/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
+++ b/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
@@ -1,168 +1,304 @@
<?php
final class PhabricatorBulkManagementExportWorkflow
extends PhabricatorBulkManagementWorkflow {
protected function didConstruct() {
$this
->setName('export')
->setExamples('**export** [options]')
->setSynopsis(
pht('Export data to a flat file (JSON, CSV, Excel, etc).'))
->setArguments(
array(
array(
'name' => 'class',
'param' => 'class',
'help' => pht(
'SearchEngine class to export data from.'),
),
array(
'name' => 'format',
'param' => 'format',
'help' => pht('Export format.'),
),
array(
'name' => 'query',
'param' => 'key',
'help' => pht(
- 'Export the data selected by this query.'),
+ 'Export the data selected by one or more queries.'),
+ 'repeat' => true,
),
array(
'name' => 'output',
'param' => 'path',
'help' => pht(
'Write output to a file. If omitted, output will be sent to '.
'stdout.'),
),
array(
'name' => 'overwrite',
'help' => pht(
'If the output file already exists, overwrite it instead of '.
'raising an error.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$viewer = $this->getViewer();
- $class = $args->getArg('class');
-
- if (!strlen($class)) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Specify a search engine class to export data from with '.
- '"--class".'));
- }
-
- if (!is_subclass_of($class, 'PhabricatorApplicationSearchEngine')) {
- throw new PhutilArgumentUsageException(
- pht(
- 'SearchEngine class ("%s") is unknown.',
- $class));
- }
-
- $engine = newv($class, array())
- ->setViewer($viewer);
-
- if (!$engine->canExport()) {
- throw new PhutilArgumentUsageException(
- pht(
- 'SearchEngine class ("%s") does not support data export.',
- $class));
- }
-
- $query_key = $args->getArg('query');
- if (!strlen($query_key)) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Specify a query to export with "--query".'));
- }
-
- if ($engine->isBuiltinQuery($query_key)) {
- $saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
- } else if ($query_key) {
- $saved_query = id(new PhabricatorSavedQueryQuery())
- ->setViewer($viewer)
- ->withQueryKeys(array($query_key))
- ->executeOne();
- } else {
- $saved_query = null;
- }
-
- if (!$saved_query) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Failed to load saved query ("%s").',
- $query_key));
- }
+ list($engine, $queries) = $this->newQueries($args);
$format_key = $args->getArg('format');
if (!strlen($format_key)) {
throw new PhutilArgumentUsageException(
pht(
'Specify an export format with "--format".'));
}
$all_formats = PhabricatorExportFormat::getAllExportFormats();
$format = idx($all_formats, $format_key);
if (!$format) {
throw new PhutilArgumentUsageException(
pht(
'Unknown export format ("%s"). Known formats are: %s.',
$format_key,
implode(', ', array_keys($all_formats))));
}
if (!$format->isExportFormatEnabled()) {
throw new PhutilArgumentUsageException(
pht(
'Export format ("%s") is not enabled.',
$format_key));
}
$is_overwrite = $args->getArg('overwrite');
$output_path = $args->getArg('output');
if (!strlen($output_path) && $is_overwrite) {
throw new PhutilArgumentUsageException(
pht(
'Flag "--overwrite" has no effect without "--output".'));
}
if (!$is_overwrite) {
if (Filesystem::pathExists($output_path)) {
throw new PhutilArgumentUsageException(
pht(
'Output path already exists. Use "--overwrite" to overwrite '.
'it.'));
}
}
+ // If we have more than one query, execute the queries to figure out which
+ // results they hit, then build a synthetic query for all those results
+ // using the IDs.
+ if (count($queries) > 1) {
+ $saved_query = $this->newUnionQuery($engine, $queries);
+ } else {
+ $saved_query = head($queries);
+ }
+
$export_engine = id(new PhabricatorExportEngine())
->setViewer($viewer)
->setTitle(pht('Export'))
->setFilename(pht('export'))
->setSearchEngine($engine)
->setSavedQuery($saved_query)
->setExportFormat($format);
$file = $export_engine->exportFile();
$iterator = $file->getFileDataIterator();
if (strlen($output_path)) {
foreach ($iterator as $chunk) {
Filesystem::appendFile($output_path, $chunk);
}
} else {
foreach ($iterator as $chunk) {
echo $chunk;
}
}
return 0;
}
+ private function newQueries(PhutilArgumentParser $args) {
+ $viewer = $this->getViewer();
+
+ $query_keys = $args->getArg('query');
+ if (!$query_keys) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Specify one or more queries to export with "--query".'));
+ }
+
+ $engine_classes = id(new PhutilClassMapQuery())
+ ->setAncestorClass('PhabricatorApplicationSearchEngine')
+ ->execute();
+
+ $class = $args->getArg('class');
+ if (strlen($class)) {
+
+ $class_list = array();
+ foreach ($engine_classes as $class_name => $engine_object) {
+ $can_export = id(clone $engine_object)
+ ->setViewer($viewer)
+ ->canExport();
+ if ($can_export) {
+ $class_list[] = $class_name;
+ }
+ }
+
+ sort($class_list);
+ $class_list = implode(', ', $class_list);
+
+ $matches = array();
+ foreach ($engine_classes as $class_name => $engine_object) {
+ if (stripos($class_name, $class) !== false) {
+ if (strtolower($class_name) == strtolower($class)) {
+ $matches = array($class_name);
+ break;
+ } else {
+ $matches[] = $class_name;
+ }
+ }
+ }
+
+ if (!$matches) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'No search engines match "%s". Available engines which support '.
+ 'data export are: %s.',
+ $class,
+ $class_list));
+ } else if (count($matches) > 1) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Multiple search engines match "%s": %s.',
+ $class,
+ implode(', ', $matches)));
+ } else {
+ $class = head($matches);
+ }
+
+ $engine = newv($class, array())
+ ->setViewer($viewer);
+ } else {
+ $engine = null;
+ }
+
+ $queries = array();
+ foreach ($query_keys as $query_key) {
+ if ($engine) {
+ if ($engine->isBuiltinQuery($query_key)) {
+ $queries[$query_key] = $engine->buildSavedQueryFromBuiltin(
+ $query_key);
+ continue;
+ }
+ }
+
+ $saved_query = id(new PhabricatorSavedQueryQuery())
+ ->setViewer($viewer)
+ ->withQueryKeys(array($query_key))
+ ->executeOne();
+ if (!$saved_query) {
+ if (!$engine) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Query "%s" is unknown. To run a builtin query like "all" or '.
+ '"active", also specify the search engine with "--class".',
+ $query_key));
+ } else {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Query "%s" is not a recognized query for class "%s".',
+ $query_key,
+ get_class($engine)));
+ }
+ }
+
+ $queries[$query_key] = $saved_query;
+ }
+
+ // If we don't have an engine from "--class", fill it in by looking at the
+ // class of the first query.
+ if (!$engine) {
+ foreach ($queries as $query) {
+ $engine = newv($query->getEngineClassName(), array())
+ ->setViewer($viewer);
+ break;
+ }
+ }
+
+ $engine_class = get_class($engine);
+
+ foreach ($queries as $query) {
+ $query_class = $query->getEngineClassName();
+ if ($query_class !== $engine_class) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Specified queries use different engines: query "%s" uses '.
+ 'engine "%s", not "%s". All queries must run on the same '.
+ 'engine.',
+ $query->getQueryKey(),
+ $query_class,
+ $engine_class));
+ }
+ }
+
+ if (!$engine->canExport()) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'SearchEngine class ("%s") does not support data export.',
+ $engine_class));
+ }
+
+ return array($engine, $queries);
+ }
+
+ private function newUnionQuery(
+ PhabricatorApplicationSearchEngine $engine,
+ array $queries) {
+
+ assert_instances_of($queries, 'PhabricatorSavedQuery');
+
+ $engine = clone $engine;
+
+ $ids = array();
+ foreach ($queries as $saved_query) {
+ $page_size = 1000;
+ $page_cursor = null;
+ do {
+ $query = $engine->buildQueryFromSavedQuery($saved_query);
+ $pager = $engine->newPagerForSavedQuery($saved_query);
+ $pager->setPageSize($page_size);
+
+ if ($page_cursor !== null) {
+ $pager->setAfterID($page_cursor);
+ }
+
+ $objects = $engine->executeQuery($query, $pager);
+ $page_cursor = $pager->getNextPageID();
+
+ foreach ($objects as $object) {
+ $ids[] = $object->getID();
+ }
+ } while ($pager->getHasMoreResults());
+ }
+
+ // When we're merging multiple different queries, override any query order
+ // and just put the combined result list in ID order. At time of writing,
+ // we can't merge the result sets together while retaining the overall sort
+ // order even if they all used the same order, and it's meaningless to try
+ // to retain orders if the queries had different orders in the first place.
+ rsort($ids);
+
+ return id($engine->newSavedQuery())
+ ->setParameter('ids', $ids);
+ }
+
}
diff --git a/src/infrastructure/export/engine/PhabricatorExportEngine.php b/src/infrastructure/export/engine/PhabricatorExportEngine.php
index f2c6a1270b..f29eaf0f60 100644
--- a/src/infrastructure/export/engine/PhabricatorExportEngine.php
+++ b/src/infrastructure/export/engine/PhabricatorExportEngine.php
@@ -1,168 +1,168 @@
<?php
final class PhabricatorExportEngine
extends Phobject {
private $viewer;
private $searchEngine;
private $savedQuery;
private $exportFormat;
private $filename;
private $title;
public function setViewer(PhabricatorUser $viewer) {
$this->viewer = $viewer;
return $this;
}
public function getViewer() {
return $this->viewer;
}
public function setSearchEngine(
PhabricatorApplicationSearchEngine $search_engine) {
$this->searchEngine = $search_engine;
return $this;
}
public function getSearchEngine() {
return $this->searchEngine;
}
public function setSavedQuery(PhabricatorSavedQuery $saved_query) {
$this->savedQuery = $saved_query;
return $this;
}
public function getSavedQuery() {
return $this->savedQuery;
}
public function setExportFormat(
PhabricatorExportFormat $export_format) {
$this->exportFormat = $export_format;
return $this;
}
public function getExportFormat() {
return $this->exportFormat;
}
public function setFilename($filename) {
$this->filename = $filename;
return $this;
}
public function getFilename() {
return $this->filename;
}
public function setTitle($title) {
$this->title = $title;
return $this;
}
public function getTitle() {
return $this->title;
}
public function newBulkJob(AphrontRequest $request) {
$viewer = $this->getViewer();
$engine = $this->getSearchEngine();
$saved_query = $this->getSavedQuery();
$format = $this->getExportFormat();
$params = array(
'engineClass' => get_class($engine),
'queryKey' => $saved_query->getQueryKey(),
'formatKey' => $format->getExportFormatKey(),
'title' => $this->getTitle(),
'filename' => $this->getFilename(),
);
$job = PhabricatorWorkerBulkJob::initializeNewJob(
$viewer,
new PhabricatorExportEngineBulkJobType(),
$params);
// We queue these jobs directly into STATUS_WAITING without requiring
// a confirmation from the user.
$xactions = array();
$xactions[] = id(new PhabricatorWorkerBulkJobTransaction())
->setTransactionType(PhabricatorWorkerBulkJobTransaction::TYPE_STATUS)
->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING);
$editor = id(new PhabricatorWorkerBulkJobEditor())
->setActor($viewer)
->setContentSourceFromRequest($request)
->setContinueOnMissingFields(true)
->applyTransactions($job, $xactions);
return $job;
}
public function exportFile() {
$viewer = $this->getViewer();
$engine = $this->getSearchEngine();
$saved_query = $this->getSavedQuery();
$format = $this->getExportFormat();
$title = $this->getTitle();
$filename = $this->getFilename();
$query = $engine->buildQueryFromSavedQuery($saved_query);
$extension = $format->getFileExtension();
$mime_type = $format->getMIMEContentType();
$filename = $filename.'.'.$extension;
$format = id(clone $format)
->setViewer($viewer)
->setTitle($title);
$field_list = $engine->newExportFieldList();
$field_list = mpull($field_list, null, 'getKey');
$format->addHeaders($field_list);
- // Iterate over the query results in large page so we don't have to hold
+ // Iterate over the query results in large pages so we don't have to hold
// too much stuff in memory.
$page_size = 1000;
$page_cursor = null;
do {
$pager = $engine->newPagerForSavedQuery($saved_query);
$pager->setPageSize($page_size);
if ($page_cursor !== null) {
$pager->setAfterID($page_cursor);
}
$objects = $engine->executeQuery($query, $pager);
$objects = array_values($objects);
$page_cursor = $pager->getNextPageID();
$export_data = $engine->newExport($objects);
for ($ii = 0; $ii < count($objects); $ii++) {
$format->addObject($objects[$ii], $field_list, $export_data[$ii]);
}
} while ($pager->getHasMoreResults());
$export_result = $format->newFileData();
// We have all the data in one big string and aren't actually
// streaming it, but pretending that we are allows us to actviate
// the chunk engine and store large files.
$iterator = new ArrayIterator(array($export_result));
$source = id(new PhabricatorIteratorFileUploadSource())
->setName($filename)
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
->setMIMEType($mime_type)
->setRelativeTTL(phutil_units('60 minutes in seconds'))
->setAuthorPHID($viewer->getPHID())
->setIterator($iterator);
return $source->uploadFile();
}
}

File Metadata

Mime Type
text/x-diff
Expires
Jan 19 2025, 21:53 (6 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1129139
Default Alt Text
(15 KB)

Event Timeline

Content licensed under Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA) unless otherwise noted; code licensed under Apache 2.0 or other open source licenses. · CC BY-SA 4.0 · Apache 2.0