Skip to content
Snippets Groups Projects
Commit a0115226 authored by Peter Rotich's avatar Peter Rotich
Browse files

Export: Make Export Fast Again

This commit addresses longstanding memory and speed issues with ticket
export especially when exporting large set of tickets.

The implementation borrows heavily on Custom Columns, which introduced the
idea of using placeholder field to parse the data from query (db) instead of
making models to do the  parsing which led to related models getting fetched
too -- and hence the memory issues.

The commit also addresses caching of choices for selection fields and such
to avoid expensive trips to database for each row!
parent 2f7ab755
No related branches found
No related tags found
No related merge requests found
......@@ -1421,6 +1421,9 @@ implements CustomListItem, TemplateVariable, Searchable {
}
function display() {
return $this->getLocalName();
return sprintf('<a class="preview" href="#"
data-preview="#list/%d/items/%d/preview">%s</a>',
$this->getListId(),
......
......@@ -574,7 +574,7 @@ class CustomQueue extends VerySimpleModel {
continue;
$name = $f->get('name') ?: 'field_'.$f->get('id');
$key = 'cdata.'.$name;
$key = 'cdata__'.$name;
$cdata[$key] = $f->getLocal('label');
}
......@@ -582,22 +582,22 @@ class CustomQueue extends VerySimpleModel {
$fields = array(
'number' => __('Ticket Number'),
'created' => __('Date Created'),
'cdata.subject' => __('Subject'),
'user.name' => __('From'),
'user.default_email.address' => __('From Email'),
'cdata.:priority.priority_desc' => __('Priority'),
'dept::getLocalName' => __('Department'),
'topic::getName' => __('Help Topic'),
'cdata__subject' => __('Subject'),
'user__name' => __('From'),
'user__emails__address' => __('From Email'),
'cdata__priority' => __('Priority'),
'dept_id' => __('Department'),
'topic_id' => __('Help Topic'),
'source' => __('Source'),
'status::getName' =>__('Current Status'),
'status__id' =>__('Current Status'),
'lastupdate' => __('Last Updated'),
'est_duedate' => __('SLA Due Date'),
'duedate' => __('Due Date'),
'closed' => __('Closed Date'),
'isoverdue' => __('Overdue'),
'isanswered' => __('Answered'),
'staff::getName' => __('Agent Assigned'),
'team::getName' => __('Team Assigned'),
'staff_id' => __('Agent Assigned'),
'team_id' => __('Team Assigned'),
'thread_count' => __('Thread Count'),
'reopen_count' => __('Reopen Count'),
'attachment_count' => __('Attachment Count'),
......@@ -629,6 +629,22 @@ class CustomQueue extends VerySimpleModel {
return $fields;
}
function getExportColumns($fields=array()) {
$columns = array();
$fields = $fields ?: $this->getExportFields();
$i = 0;
foreach ($fields as $path => $label) {
$c = QueueColumn::placeholder(array(
'id' => $i++,
'heading' => $label,
'primary' => $path,
));
$c->setQueue($this);
$columns[$path] = $c;
}
return $columns;
}
function getStandardColumns() {
return $this->getColumns();
}
......@@ -775,14 +791,13 @@ class CustomQueue extends VerySimpleModel {
}
function export($options=array()) {
global $thisstaff;
if (!($query=$this->getBasicQuery()))
return false;
if (!($fields=$this->getExportFields()))
if (!$thisstaff
|| !($query=$this->getBasicQuery())
|| !($fields=$this->getExportFields()))
return false;
$filename = sprintf('%s Tickets-%s.csv',
$this->getName(),
strftime('%Y%m%d'));
......@@ -799,14 +814,45 @@ class CustomQueue extends VerySimpleModel {
$filename ="$filename.csv";
}
if (isset($opts['delimiter']))
if (isset($opts['delimiter']) && !$options['delimiter'])
$options['delimiter'] = $opts['delimiter'];
}
// Apply columns
$columns = $this->getExportColumns($fields);
$headers = array(); // Reset fields based on validity of columns
foreach ($columns as $column) {
$query = $column->mangleQuery($query, $this->getRoot());
$headers[] = $column->getHeading();
}
// Apply visibility
if (!$this->ignoreVisibilityConstraints($thisstaff))
$query->filter($thisstaff->getTicketsVisibility());
// Render Util
$render = function ($row) use($columns) {
if (!$row) return false;
return Export::saveTickets($query, $fields, $filename, 'csv',
$options);
$record = array();
foreach ($columns as $path => $column) {
$record[] = (string) $column->from_query($row) ?:
$row[$path] ?: '';
}
return $record;
};
$delimiter = $options['delimiter'] ?:
Internationalization::getCSVDelimiter();
$output = fopen('php://output', 'w');
Http::download($filename, "text/csv");
fputs($output, chr(0xEF) . chr(0xBB) . chr(0xBF));
fputcsv($output, $headers, $delimiter);
foreach ($query as $row)
fputcsv($output, $render($row), $delimiter);
fclose($output);
exit();
}
/**
......@@ -1984,6 +2030,7 @@ extends VerySimpleModel {
var $_annotations;
var $_conditions;
var $_queue; // Apparent queue if being inherited
var $_fields;
function getId() {
return $this->id;
......@@ -2022,6 +2069,25 @@ extends VerySimpleModel {
$this->_queue = $queue;
}
function getFields() {
if (!isset($this->_fields)) {
$root = ($q = $this->getQueue()) ? $q->getRoot() : 'Ticket';
$fields = CustomQueue::getSearchableFields($root);
$primary = CustomQueue::getOrmPath($this->primary);
$secondary = CustomQueue::getOrmPath($this->secondary);
if (($F = $fields[$primary]) && (list(,$field) = $F))
$this->_fields[$primary] = $field;
if (($F = $fields[$secondary]) && (list(,$field) = $F))
$this->_fields[$secondary] = $field;
}
return $this->_fields;
}
function getField($path=null) {
$fields = $this->getFields();
return @$fields[$path ?: $this->primary];
}
function getWidth() {
return $this->width ?: 100;
}
......@@ -2089,29 +2155,36 @@ extends VerySimpleModel {
}
function renderBasicValue($row) {
$root = ($q = $this->getQueue()) ? $q->getRoot() : 'Ticket';
$fields = CustomQueue::getSearchableFields($root);
$fields = $this->getFields();
$primary = CustomQueue::getOrmPath($this->primary);
$secondary = CustomQueue::getOrmPath($this->secondary);
// Return a lazily ::display()ed value so that the value to be
// rendered by the field could be changed or display()ed when
// converted to a string.
if (($F = $fields[$primary])
&& (list(,$field) = $F)
&& ($T = $field->from_query($row, $primary))
&& ($T = $F->from_query($row, $primary))
) {
return new LazyDisplayWrapper($field, $T);
return new LazyDisplayWrapper($F, $T);
}
if (($F = $fields[$secondary])
&& (list(,$field) = $F)
&& ($T = $field->from_query($row, $secondary))
&& ($T = $F->from_query($row, $secondary))
) {
return new LazyDisplayWrapper($field, $T);
return new LazyDisplayWrapper($F, $T);
}
return new LazyDisplayWrapper($field, '');
return new LazyDisplayWrapper($F, '');
}
function from_query($row) {
if (!($f = $this->getField($this->primary)))
return '';
$val = $f->to_php($f->from_query($row, $this->primary));
if (is_numeric($val))
$val = $f->display($val);
return $val;
}
function applyTruncate($text, $row) {
......@@ -2155,14 +2228,12 @@ extends VerySimpleModel {
function mangleQuery($query, $root=null) {
// Basic data
$fields = CustomQueue::getSearchableFields($root ?: $this->getQueue()->getRoot());
if ($primary = $fields[$this->primary]) {
list(,$field) = $primary;
$fields = $this->getFields();
if ($field = $fields[$this->primary]) {
$query = $this->addToQuery($query, $field,
CustomQueue::getOrmPath($this->primary, $query));
}
if ($secondary = $fields[$this->secondary]) {
list(,$field) = $secondary;
if ($field = $fields[$this->secondary]) {
$query = $this->addToQuery($query, $field,
CustomQueue::getOrmPath($this->secondary, $query));
}
......
......@@ -1004,21 +1004,30 @@ class AdvancedSearchSelectionField extends ChoiceField {
}
class HelpTopicChoiceField extends AdvancedSearchSelectionField {
static $_topics;
function hasIdValue() {
return true;
}
function getChoices($verbose=false) {
return Topic::getHelpTopics(false, Topic::DISPLAY_DISABLED);
if (!isset($this->_topics))
$this->_topics = Topic::getHelpTopics(false, Topic::DISPLAY_DISABLED);
return $this->_topics;
}
}
require_once INCLUDE_DIR . 'class.dept.php';
class DepartmentChoiceField extends AdvancedSearchSelectionField {
var $_choices = null;
static $_depts;
var $_choices;
function getChoices($verbose=false) {
return Dept::getDepartments();
if (!isset($this->_depts))
$this->_depts = Dept::getDepartments();
return $this->_depts;
}
function getQuickFilterChoices() {
......@@ -1241,13 +1250,23 @@ trait ZeroMeansUnset {
class AgentSelectionField extends AdvancedSearchSelectionField {
use ZeroMeansUnset;
static $_agents;
function getChoices($verbose=false) {
return array('M' => __('Me')) + Staff::getStaffMembers();
if (!isset($this->_agents)) {
$this->_agents = array('M' => __('Me')) +
Staff::getStaffMembers();
}
return $this->_agents;
}
function toString($value) {
$choices = $this->getChoices();
$selection = array();
if (!is_array($value))
$value = array($value => $value);
foreach ($value as $k => $v)
if (isset($choices[$k]))
$selection[] = $choices[$k];
......@@ -1278,9 +1297,13 @@ class AgentSelectionField extends AdvancedSearchSelectionField {
}
class DepartmentManagerSelectionField extends AgentSelectionField {
static $_members;
function getChoices($verbose=false) {
return Staff::getStaffMembers();
if (isset($this->_members))
$this->_members = Staff::getStaffMembers();
return $this->_members;
}
function getSearchQ($method, $value, $name=false) {
......@@ -1289,9 +1312,14 @@ class DepartmentManagerSelectionField extends AgentSelectionField {
}
class TeamSelectionField extends AdvancedSearchSelectionField {
static $_teams;
function getChoices($verbose=false) {
return array('T' => __('One of my teams')) + Team::getTeams();
if (!isset($this->_teams))
$this->_teams = array('T' => __('One of my teams')) +
Team::getTeams();
return $this->_teams;
}
function getSearchQ($method, $value, $name=false) {
......@@ -1315,6 +1343,19 @@ class TeamSelectionField extends AdvancedSearchSelectionField {
$reverse = $reverse ? '-' : '';
return $query->order_by("{$reverse}team__name");
}
function toString($value) {
$choices = $this->getChoices();
$selection = array();
if (!is_array($value))
$value = array($value => $value);
foreach ($value as $k => $v)
if (isset($choices[$k]))
$selection[] = $choices[$k];
return $selection ? implode(',', $selection) :
parent::toString($value);
}
}
class TicketStateChoiceField extends AdvancedSearchSelectionField {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment