From a01152262d880131c8ef8575f349b3da66286f88 Mon Sep 17 00:00:00 2001
From: Peter Rotich <peter@enhancesoft.com>
Date: Mon, 27 Aug 2018 04:28:33 +0000
Subject: [PATCH] Export: Make Export Fast Again
This commit addresses longstanding memory and speed issues with ticket
export especially when exporting large set of tickets.
The implementation borrows heavily on Custom Columns, which introduced the
idea of using placeholder field to parse the data from query (db) instead of
making models to do the parsing which led to related models getting fetched
too -- and hence the memory issues.
The commit also addresses caching of choices for selection fields and such
to avoid expensive trips to database for each row!
---
include/class.list.php | 3 +
include/class.queue.php | 137 +++++++++++++++++++++++++++++----------
include/class.search.php | 53 +++++++++++++--
3 files changed, 154 insertions(+), 39 deletions(-)
diff --git a/include/class.list.php b/include/class.list.php
index e5c84fbd9..f8055805c 100644
--- a/include/class.list.php
+++ b/include/class.list.php
@@ -1421,6 +1421,9 @@ implements CustomListItem, TemplateVariable, Searchable {
}
function display() {
+
+ return $this->getLocalName();
+
return sprintf('<a class="preview" href="#"
data-preview="#list/%d/items/%d/preview">%s</a>',
$this->getListId(),
diff --git a/include/class.queue.php b/include/class.queue.php
index b4343d43e..2e1b3fa5a 100644
--- a/include/class.queue.php
+++ b/include/class.queue.php
@@ -574,7 +574,7 @@ class CustomQueue extends VerySimpleModel {
continue;
$name = $f->get('name') ?: 'field_'.$f->get('id');
- $key = 'cdata.'.$name;
+ $key = 'cdata__'.$name;
$cdata[$key] = $f->getLocal('label');
}
@@ -582,22 +582,22 @@ class CustomQueue extends VerySimpleModel {
$fields = array(
'number' => __('Ticket Number'),
'created' => __('Date Created'),
- 'cdata.subject' => __('Subject'),
- 'user.name' => __('From'),
- 'user.default_email.address' => __('From Email'),
- 'cdata.:priority.priority_desc' => __('Priority'),
- 'dept::getLocalName' => __('Department'),
- 'topic::getName' => __('Help Topic'),
+ 'cdata__subject' => __('Subject'),
+ 'user__name' => __('From'),
+ 'user__emails__address' => __('From Email'),
+ 'cdata__priority' => __('Priority'),
+ 'dept_id' => __('Department'),
+ 'topic_id' => __('Help Topic'),
'source' => __('Source'),
- 'status::getName' =>__('Current Status'),
+ 'status__id' =>__('Current Status'),
'lastupdate' => __('Last Updated'),
'est_duedate' => __('SLA Due Date'),
'duedate' => __('Due Date'),
'closed' => __('Closed Date'),
'isoverdue' => __('Overdue'),
'isanswered' => __('Answered'),
- 'staff::getName' => __('Agent Assigned'),
- 'team::getName' => __('Team Assigned'),
+ 'staff_id' => __('Agent Assigned'),
+ 'team_id' => __('Team Assigned'),
'thread_count' => __('Thread Count'),
'reopen_count' => __('Reopen Count'),
'attachment_count' => __('Attachment Count'),
@@ -629,6 +629,22 @@ class CustomQueue extends VerySimpleModel {
return $fields;
}
+ function getExportColumns($fields=array()) {
+ $columns = array();
+ $fields = $fields ?: $this->getExportFields();
+ $i = 0;
+ foreach ($fields as $path => $label) {
+ $c = QueueColumn::placeholder(array(
+ 'id' => $i++,
+ 'heading' => $label,
+ 'primary' => $path,
+ ));
+ $c->setQueue($this);
+ $columns[$path] = $c;
+ }
+ return $columns;
+ }
+
function getStandardColumns() {
return $this->getColumns();
}
@@ -775,14 +791,13 @@ class CustomQueue extends VerySimpleModel {
}
function export($options=array()) {
+ global $thisstaff;
- if (!($query=$this->getBasicQuery()))
- return false;
-
- if (!($fields=$this->getExportFields()))
+ if (!$thisstaff
+ || !($query=$this->getBasicQuery())
+ || !($fields=$this->getExportFields()))
return false;
-
$filename = sprintf('%s Tickets-%s.csv',
$this->getName(),
strftime('%Y%m%d'));
@@ -799,14 +814,45 @@ class CustomQueue extends VerySimpleModel {
$filename ="$filename.csv";
}
- if (isset($opts['delimiter']))
+ if (isset($opts['delimiter']) && !$options['delimiter'])
$options['delimiter'] = $opts['delimiter'];
}
+ // Apply columns
+ $columns = $this->getExportColumns($fields);
+ $headers = array(); // Reset fields based on validity of columns
+ foreach ($columns as $column) {
+ $query = $column->mangleQuery($query, $this->getRoot());
+ $headers[] = $column->getHeading();
+ }
+
+ // Apply visibility
+ if (!$this->ignoreVisibilityConstraints($thisstaff))
+ $query->filter($thisstaff->getTicketsVisibility());
+
+ // Render Util
+ $render = function ($row) use($columns) {
+ if (!$row) return false;
- return Export::saveTickets($query, $fields, $filename, 'csv',
- $options);
+ $record = array();
+ foreach ($columns as $path => $column) {
+ $record[] = (string) $column->from_query($row) ?:
+ $row[$path] ?: '';
+ }
+ return $record;
+ };
+
+ $delimiter = $options['delimiter'] ?:
+ Internationalization::getCSVDelimiter();
+ $output = fopen('php://output', 'w');
+ Http::download($filename, "text/csv");
+ fputs($output, chr(0xEF) . chr(0xBB) . chr(0xBF));
+ fputcsv($output, $headers, $delimiter);
+ foreach ($query as $row)
+ fputcsv($output, $render($row), $delimiter);
+ fclose($output);
+ exit();
}
/**
@@ -1984,6 +2030,7 @@ extends VerySimpleModel {
var $_annotations;
var $_conditions;
var $_queue; // Apparent queue if being inherited
+ var $_fields;
function getId() {
return $this->id;
@@ -2022,6 +2069,25 @@ extends VerySimpleModel {
$this->_queue = $queue;
}
+ function getFields() {
+ if (!isset($this->_fields)) {
+ $root = ($q = $this->getQueue()) ? $q->getRoot() : 'Ticket';
+ $fields = CustomQueue::getSearchableFields($root);
+ $primary = CustomQueue::getOrmPath($this->primary);
+ $secondary = CustomQueue::getOrmPath($this->secondary);
+ if (($F = $fields[$primary]) && (list(,$field) = $F))
+ $this->_fields[$primary] = $field;
+ if (($F = $fields[$secondary]) && (list(,$field) = $F))
+ $this->_fields[$secondary] = $field;
+ }
+ return $this->_fields;
+ }
+
+ function getField($path=null) {
+ $fields = $this->getFields();
+ return @$fields[$path ?: $this->primary];
+ }
+
function getWidth() {
return $this->width ?: 100;
}
@@ -2089,29 +2155,36 @@ extends VerySimpleModel {
}
function renderBasicValue($row) {
- $root = ($q = $this->getQueue()) ? $q->getRoot() : 'Ticket';
- $fields = CustomQueue::getSearchableFields($root);
+ $fields = $this->getFields();
$primary = CustomQueue::getOrmPath($this->primary);
$secondary = CustomQueue::getOrmPath($this->secondary);
// Return a lazily ::display()ed value so that the value to be
// rendered by the field could be changed or display()ed when
// converted to a string.
-
if (($F = $fields[$primary])
- && (list(,$field) = $F)
- && ($T = $field->from_query($row, $primary))
+ && ($T = $F->from_query($row, $primary))
) {
- return new LazyDisplayWrapper($field, $T);
+ return new LazyDisplayWrapper($F, $T);
}
if (($F = $fields[$secondary])
- && (list(,$field) = $F)
- && ($T = $field->from_query($row, $secondary))
+ && ($T = $F->from_query($row, $secondary))
) {
- return new LazyDisplayWrapper($field, $T);
+ return new LazyDisplayWrapper($F, $T);
}
- return new LazyDisplayWrapper($field, '');
+ return new LazyDisplayWrapper($F, '');
+ }
+
+ function from_query($row) {
+ if (!($f = $this->getField($this->primary)))
+ return '';
+
+ $val = $f->to_php($f->from_query($row, $this->primary));
+ if (is_numeric($val))
+ $val = $f->display($val);
+
+ return $val;
}
function applyTruncate($text, $row) {
@@ -2155,14 +2228,12 @@ extends VerySimpleModel {
function mangleQuery($query, $root=null) {
// Basic data
- $fields = CustomQueue::getSearchableFields($root ?: $this->getQueue()->getRoot());
- if ($primary = $fields[$this->primary]) {
- list(,$field) = $primary;
+ $fields = $this->getFields();
+ if ($field = $fields[$this->primary]) {
$query = $this->addToQuery($query, $field,
CustomQueue::getOrmPath($this->primary, $query));
}
- if ($secondary = $fields[$this->secondary]) {
- list(,$field) = $secondary;
+ if ($field = $fields[$this->secondary]) {
$query = $this->addToQuery($query, $field,
CustomQueue::getOrmPath($this->secondary, $query));
}
diff --git a/include/class.search.php b/include/class.search.php
index e4c6111b5..51629ccdb 100644
--- a/include/class.search.php
+++ b/include/class.search.php
@@ -1004,21 +1004,30 @@ class AdvancedSearchSelectionField extends ChoiceField {
}
class HelpTopicChoiceField extends AdvancedSearchSelectionField {
+ static $_topics;
+
function hasIdValue() {
return true;
}
function getChoices($verbose=false) {
- return Topic::getHelpTopics(false, Topic::DISPLAY_DISABLED);
+ if (!isset($this->_topics))
+ $this->_topics = Topic::getHelpTopics(false, Topic::DISPLAY_DISABLED);
+
+ return $this->_topics;
}
}
require_once INCLUDE_DIR . 'class.dept.php';
class DepartmentChoiceField extends AdvancedSearchSelectionField {
- var $_choices = null;
+ static $_depts;
+ var $_choices;
function getChoices($verbose=false) {
- return Dept::getDepartments();
+ if (!isset($this->_depts))
+ $this->_depts = Dept::getDepartments();
+
+ return $this->_depts;
}
function getQuickFilterChoices() {
@@ -1241,13 +1250,23 @@ trait ZeroMeansUnset {
class AgentSelectionField extends AdvancedSearchSelectionField {
use ZeroMeansUnset;
+ static $_agents;
+
function getChoices($verbose=false) {
- return array('M' => __('Me')) + Staff::getStaffMembers();
+ if (!isset($this->_agents)) {
+ $this->_agents = array('M' => __('Me')) +
+ Staff::getStaffMembers();
+ }
+ return $this->_agents;
}
function toString($value) {
+
$choices = $this->getChoices();
$selection = array();
+ if (!is_array($value))
+ $value = array($value => $value);
+
foreach ($value as $k => $v)
if (isset($choices[$k]))
$selection[] = $choices[$k];
@@ -1278,9 +1297,13 @@ class AgentSelectionField extends AdvancedSearchSelectionField {
}
class DepartmentManagerSelectionField extends AgentSelectionField {
+ static $_members;
function getChoices($verbose=false) {
- return Staff::getStaffMembers();
+ if (isset($this->_members))
+ $this->_members = Staff::getStaffMembers();
+
+ return $this->_members;
}
function getSearchQ($method, $value, $name=false) {
@@ -1289,9 +1312,14 @@ class DepartmentManagerSelectionField extends AgentSelectionField {
}
class TeamSelectionField extends AdvancedSearchSelectionField {
+ static $_teams;
function getChoices($verbose=false) {
- return array('T' => __('One of my teams')) + Team::getTeams();
+ if (!isset($this->_teams))
+ $this->_teams = array('T' => __('One of my teams')) +
+ Team::getTeams();
+
+ return $this->_teams;
}
function getSearchQ($method, $value, $name=false) {
@@ -1315,6 +1343,19 @@ class TeamSelectionField extends AdvancedSearchSelectionField {
$reverse = $reverse ? '-' : '';
return $query->order_by("{$reverse}team__name");
}
+
+ function toString($value) {
+ $choices = $this->getChoices();
+ $selection = array();
+ if (!is_array($value))
+ $value = array($value => $value);
+ foreach ($value as $k => $v)
+ if (isset($choices[$k]))
+ $selection[] = $choices[$k];
+ return $selection ? implode(',', $selection) :
+ parent::toString($value);
+ }
+
}
class TicketStateChoiceField extends AdvancedSearchSelectionField {
--
GitLab