From b0ff694bd812cc0794d5e1ff3b0818328a1d6cdf Mon Sep 17 00:00:00 2001
From: Jared Hancock <jared@osticket.com>
Date: Fri, 16 Oct 2015 15:39:58 -0500
Subject: [PATCH] search: Use field searching system from custom queues

---
 include/ajax.search.php                       |  54 +-
 include/class.queue.php                       | 116 +---
 include/class.search.php                      | 502 ++++++++++--------
 include/class.thread.php                      |  18 +-
 include/class.ticket.php                      |  47 +-
 include/i18n/en_US/queue.yaml                 |  94 ++++
 include/staff/queue.inc.php                   |   4 +-
 include/staff/queues-ticket.inc.php           |   3 +-
 .../advanced-search-criteria.tmpl.php         |  15 +-
 .../templates/queue-column-condition.tmpl.php |   2 +-
 .../streams/core/98ad7d55-00000000.patch.sql  |   1 +
 scp/tickets.php                               |   9 +-
 12 files changed, 454 insertions(+), 411 deletions(-)
 create mode 100644 include/i18n/en_US/queue.yaml

diff --git a/include/ajax.search.php b/include/ajax.search.php
index db9fab815..7e0452dd2 100644
--- a/include/ajax.search.php
+++ b/include/ajax.search.php
@@ -28,9 +28,12 @@ class SearchAjaxAPI extends AjaxController {
         if (!$thisstaff)
             Http::response(403, 'Agent login required');
 
-        $search = SavedSearch::create();
-        $form = $search->getFormFromSession('advsearch') ?: $search->getForm();
-        $matches = SavedSearch::getSupportedTicketMatches();
+        $search = SavedSearch::create(array(
+            'root' => 'T',
+        ));
+        $search->config = $_SESSION['advsearch'];
+        $form = $search->getForm();
+        $matches = $search->getSupportedMatches();
 
         include STAFFINC_DIR . 'templates/advanced-search.tmpl.php';
     }
@@ -41,37 +44,13 @@ class SearchAjaxAPI extends AjaxController {
         if (!$thisstaff)
             Http::response(403, 'Agent login required');
 
-        @list($type, $id) = explode('!', $name, 2);
-
-        switch (strtolower($type)) {
-        case ':ticket':
-        case ':user':
-        case ':organization':
-        case ':field':
-            // Support nested field ids for list properties and such
-            if (strpos($id, '.') !== false)
-                list(,$id) = explode('!', $id, 2);
-            if (!($field = DynamicFormField::lookup($id)))
-                Http::response(404, 'No such field: ', print_r($id, true));
-
-            $impl = $field->getImpl();
-            $impl->set('label', sprintf('%s / %s',
-                $field->form->getLocal('title'), $field->getLocal('label')
-            ));
-            break;
-
-        default:
-            $extended = SavedSearch::getExtendedTicketFields();
-
-            if (isset($extended[$name])) {
-                $impl = $extended[$name];
-                break;
-            }
-            Http::response(400, 'No such field type');
-        }
+        $search = SavedSearch::create(array('root'=>'T'));
+        $searchable = $search->getSupportedMatches();
+        if (!($F = $searchable[$name]))
+            Http::response(404, 'No such field: ', print_r($id, true));
 
-        $fields = SavedSearch::getSearchField($impl, $name);
-        $form = new SimpleForm($fields);
+        $fields = SavedSearch::getSearchField($F, $name);
+        $form = new AdvancedSearchForm($fields);
         // Check the box to search the field by default
         if ($F = $form->getField("{$name}+search"))
             $F->value = true;
@@ -83,24 +62,21 @@ class SearchAjaxAPI extends AjaxController {
         return $this->encode(array(
             'success' => true,
             'html' => $html,
-            // Send the current formfield UID to be resent with the next
-            // addField request and set above
-            'ff_uid' => FormField::$uid,
         ));
     }
 
     function doSearch() {
         global $thisstaff;
 
-        $search = SavedSearch::create();
+        $search = SavedSearch::create(array('root' => 'T'));
 
         $form = $search->getForm($_POST);
         if (!$form->isValid()) {
-            $matches = SavedSearch::getSupportedTicketMatches();
+            $matches = $search->getSupportedMatches();
             include STAFFINC_DIR . 'templates/advanced-search.tmpl.php';
             return;
         }
-        $_SESSION['advsearch'] = $form->getState();
+        $_SESSION['advsearch'] = $search->isolateCriteria($form->getClean());
 
         Http::response(200, $this->encode(array(
             'redirect' => 'tickets.php?queue=adhoc',
diff --git a/include/class.queue.php b/include/class.queue.php
index 51bc6ff66..66781c5e0 100644
--- a/include/class.queue.php
+++ b/include/class.queue.php
@@ -58,61 +58,8 @@ class CustomQueue extends SavedSearch {
 
     function getColumns() {
         if (!count($this->columns)) {
-            if ($this->columns_id
-                && ($q = CustomQueue::lookup($this->columns_id))
-            ) {
-                // Use columns from cited queue
-                return $q->getColumns();
-            }
-
-            // Last resort — use standard columns
-            foreach (array(
-                new QueueColumn(array(
-                    "id" => 1,
-                    "heading" => "Number",
-                    "primary" => 'number',
-                    "width" => 100,
-                    "filter" => "link:ticket",
-                    "annotations" => '[{"c":"TicketSourceDecoration","p":"b"}]',
-                    "conditions" => '[{"crit":["isanswered","set",null],"prop":{"font-weight":"bold"}}]',
-                )),
-                new QueueColumn(array(
-                    "id" => 2,
-                    "heading" => "Created",
-                    "primary" => 'created',
-                    "width" => 100,
-                )),
-                new QueueColumn(array(
-                    "id" => 3,
-                    "heading" => "Subject",
-                    "primary" => 'cdata__subject',
-                    "width" => 250,
-                    "filter" => "link:ticket",
-                    "annotations" => '[{"c":"TicketThreadCount","p":">"},{"c":"ThreadAttachmentCount","p":"a"},{"c":"OverdueFlagDecoration","p":"<"}]',
-                    "truncate" => 'ellipsis',
-                )),
-                new QueueColumn(array(
-                    "id" => 4,
-                    "heading" => "From",
-                    "primary" => 'user__name',
-                    "width" => 150,
-                )),
-                new QueueColumn(array(
-                    "id" => 5,
-                    "heading" => "Priority",
-                    "primary" => 'cdata__priority',
-                    "width" => 120,
-                )),
-                new QueueColumn(array(
-                    "id" => 6,
-                    "heading" => "Assignee",
-                    "primary" => 'assignee',
-                    "secondary" => 'team__name',
-                    "width" => 100,
-                )),
-            ) as $c) {
+            foreach (parent::getColumns() as $c)
                 $this->addColumn($c);
-            }
         }
         return $this->columns;
     }
@@ -176,8 +123,7 @@ class CustomQueue extends SavedSearch {
             $root = $this->getRoot();
             $query = $root::objects();
         }
-        $form = $form ?: $this->loadFromState($this->getCriteria());
-        return $this->mangleQuerySet($query, $form);
+        return $this->mangleQuerySet($query);
     }
 
     /**
@@ -188,7 +134,7 @@ class CustomQueue extends SavedSearch {
      * Returns:
      * <QuerySet> instance
      */
-    function getQuery($form=false, $quick_filter=false) {
+    function getQuery($form=false, $quick_filter=null) {
         // Start with basic criteria
         $query = $this->getBasicQuery($form);
 
@@ -196,7 +142,7 @@ class CustomQueue extends SavedSearch {
         if (isset($quick_filter)
             && ($qf = $this->getQuickFilterField($quick_filter))
         ) {
-            $this->filter = @QueueColumn::getOrmPath($this->filter, $query);
+            $this->filter = @SavedSearch::getOrmPath($this->filter, $query);
             $query = $qf->applyQuickFilter($query, $quick_filter,
                 $this->filter); 
         }
@@ -224,16 +170,16 @@ class CustomQueue extends SavedSearch {
         }
     }
 
-    function update($vars, &$errors) {
+    function update($vars, &$errors=array()) {
         // TODO: Move this to SavedSearch::update() and adjust
         //       AjaxSearch::_saveSearch()
         $form = $this->getForm($vars);
-        $form->setSource($vars);
         if (!$vars || !$form->isValid()) {
             $errors['criteria'] = __('Validation errors exist on criteria');
         }
         else {
-            $this->config = JsonDataEncoder::encode($form->getState());
+            $this->config = JsonDataEncoder::encode(
+                $this->isolateCriteria($form->getClean()));
         }
 
         // Set basic queue information
@@ -522,7 +468,7 @@ class QueueColumnCondition {
 
         // XXX: Move getOrmPath to be more of a utility
         // Ensure the special join is created to support custom data joins
-        $name = @QueueColumn::getOrmPath($name, $query);
+        $name = @SavedSearch::getOrmPath($name, $query);
 
         $name2 = null;
         if (preg_match('/__answers!\d+__/', $name)) {
@@ -700,7 +646,7 @@ extends VerySimpleModel {
         'ordering' => array('sort'),
         'joins' => array(
             'queue' => array(
-                'constraint' => array('queue_id' => 'CustomQueue.id'),
+                'constraint' => array('queue_id' => 'SavedSearch.id'),
             ),
         ),
     );
@@ -708,9 +654,6 @@ extends VerySimpleModel {
     var $_annotations;
     var $_conditions;
 
-    function __onload() {
-    }
-
     function getId() {
         return $this->id;
     }
@@ -756,10 +699,10 @@ extends VerySimpleModel {
     }
 
     function renderBasicValue($row) {
-        $root = $this->getQueue()->getRoot();
+        $root = ($q = $this->getQueue()) ? $q->getRoot() : 'Ticket';
         $fields = SavedSearch::getSearchableFields($root);
-        $primary = $this->getOrmPath($this->primary);
-        $secondary = $this->getOrmPath($this->secondary);
+        $primary = SavedSearch::getOrmPath($this->primary);
+        $secondary = SavedSearch::getOrmPath($this->secondary);
 
         // TODO: Consider data filter if configured
 
@@ -808,12 +751,12 @@ extends VerySimpleModel {
         if ($primary = $fields[$this->primary]) {
             list(,$field) = $primary;
             $query = $this->addToQuery($query, $field,
-                $this->getOrmPath($this->primary, $query));
+                SavedSearch::getOrmPath($this->primary, $query));
         }
         if ($secondary = $fields[$this->secondary]) {
             list(,$field) = $secondary;
             $query = $this->addToQuery($query, $field,
-                $this->getOrmPath($this->secondary, $query));
+                SavedSearch::getOrmPath($this->secondary, $query));
         }
 
         if ($filter = $this->getFilter())
@@ -837,34 +780,6 @@ extends VerySimpleModel {
             array('id' => $this->id));
     }
 
-    function getOrmPath($name, $query=null) {
-        // Special case for custom data `__answers!id__value`. Only add the
-        // join and constraint on the query the first pass, when the query
-        // being mangled is received.
-        $path = array();
-        if ($query && preg_match('/^(.+?)__(answers!(\d+))/', $name, $path)) {
-            // Add a join to the model of the queryset where the custom data
-            // is forked from — duplicate the 'answers' join and add the
-            // constraint to the query based on the field_id
-            // $path[1] - part before the answers (user__org__entries)
-            // $path[2] - answers!xx join part
-            // $path[3] - the `xx` part of the answers!xx join component
-            $root = $query->model;
-            $meta = $root::getMeta()->getByPath($path[1]);
-            $joins = $meta['joins'];
-            if (!isset($joins[$path[2]])) {
-                $meta->addJoin($path[2], $joins['answers']);
-            }
-            // Ensure that the query join through answers!xx is only for the
-            // records which match field_id=xx
-            $query->constrain(array("{$path[1]}__{$path[2]}" =>
-                array("{$path[1]}__{$path[2]}__field_id" => (int) $path[3])
-            ));
-            // Leave $name unchanged
-        }
-        return $name;
-    }
-
     function getAnnotations() {
         if (!isset($this->_annotations)) {
             $this->_annotations = array();
@@ -935,8 +850,7 @@ extends VerySimpleModel {
                 if (!isset($fields[$name]))
                     // No such field exists for this queue root type
                     continue;
-                list(,$field) = $fields[$name];
-                $parts = SavedSearch::getSearchField($field, $name);
+                $parts = SavedSearch::getSearchField($fields[$name], $name);
                 $search_form = new SimpleForm($parts, $vars, array('id' => $id));
                 $search_form->getField("{$name}+search")->value = true;
                 $crit = $search_form->getClean();
diff --git a/include/class.search.php b/include/class.search.php
index f83860689..29a48fa5c 100644
--- a/include/class.search.php
+++ b/include/class.search.php
@@ -670,6 +670,7 @@ class SavedSearch extends VerySimpleModel {
     const FLAG_INHERIT_CRITERIA = 0x0008; // Include criteria from parent
 
     var $criteria;
+    private $columns;
 
     static function forStaff(Staff $agent) {
         return static::objects()->filter(Q::any(array(
@@ -703,48 +704,23 @@ class SavedSearch extends VerySimpleModel {
 
     function getCriteria() {
         if (!isset($this->criteria)) {
-            $this->criteria = JsonDataParser::decode($this->config);
-        }
-        return $this->criteria;
-    }
-
-    function getSearchForm() {
-        if ($state = JsonDataParser::parse($this->config)) {
-            $form = $this->loadFromState($state);
-            $form->loadState($state);
-            return $form;
-        }
-        return $this->getForm();
-    }
-
-    function loadFromState($source=false) {
-        // Pull out 'other' fields from the state so the fields will be
-        // added to the form. The state will be loaded below
-        $state = $source ?: array();
-        foreach ($state as $k=>$v) {
-            $info = array();
-            if (!preg_match('/^:(\w+)(?:!(\d+))?\+search/', $k, $info)) {
-                continue;
+            $old = @$this->config[0] === '{';
+            $this->criteria = is_string($this->config)
+                ? JsonDataParser::decode($this->config)
+                : $this->config;
+            // Auto-upgrade criteria to new format
+            if ($old) {
+                // TODO: Upgrade old ORM path names
+                $this->criteria = $this->isolateCriteria($this->criteria);
             }
-            list($k,) = explode('+', $k, 2);
-            $state['fields'][] = $k;
         }
-        return $this->getForm($state);
+        return $this->criteria ?: array();
     }
 
-    function getFormFromSession($key) {
-        if (isset($_SESSION[$key])) {
-            return $this->loadFromState($_SESSION[$key]);
-        }
-    }
-
-    function getForm($source=false) {
-        // XXX: Ensure that the UIDs generated for these fields are
-        //      consistent between requests
-
+    function getForm($source=null) {
         $searchable = $this->getCurrentSearchFields($source);
         $fields = array(
-            'keywords' => new TextboxField(array(
+            ':keywords' => new TextboxField(array(
                 'id' => 3001,
                 'configuration' => array(
                     'size' => 40,
@@ -755,8 +731,8 @@ class SavedSearch extends VerySimpleModel {
                 ),
             )),
         );
-        foreach ($searchable as $name=>$field) {
-            $fields = array_merge($fields, self::getSearchField($field, $name));
+        foreach ($searchable as $path=>$field) {
+            $fields = array_merge($fields, self::getSearchField($field, $path));
         }
 
         // Don't send the state as the souce because it is not in the
@@ -769,139 +745,77 @@ class SavedSearch extends VerySimpleModel {
                 if (substr($F->get('name'), -7) == '+search' && $F->getClean())
                     $selected += 1;
                 // Consider keyword searches
-                elseif ($F->get('name') == 'keywords' && $F->getClean())
+                elseif ($F->get('name') == ':keywords' && $F->getClean())
                     $selected += 1;
             }
             if (!$selected)
                 $form->addError(__('No fields selected for searching'));
         });
-        if ($source)
-            $form->loadState($source);
+
+        // Load state from current configuraiton
+        foreach ($this->getCriteria() as $I) {
+            list($path, $method, $value) = $I;
+            if (!($F = $form->getField("{$path}+search")))
+                continue;
+            $F->value = true;
+
+            if (!($F = $form->getField("{$path}+method")))
+                continue;
+            $F->value = $method;
+
+            if ($value && ($F = $form->getField("{$path}+{$method}")))
+                $F->value = $value;
+        }
         return $form;
     }
 
-    function getCurrentSearchFields($source=false) {
-        $core = array(
-            'status_id' =>  new TicketStatusChoiceField(array(
-                'id' => 3101,
-                'label' => __('Status'),
-            )),
-            'dept_id'   =>  new DepartmentChoiceField(array(
-                'id' => 3102,
-                'label' => __('Department'),
-            )),
-            'assignee'  =>  new AssigneeChoiceField(array(
-                'id' => 3103,
-                'label' => __('Assignee'),
-            )),
-            'topic_id'  =>  new HelpTopicChoiceField(array(
-                'id' => 3104,
-                'label' => __('Help Topic'),
-            )),
-            'created'   =>  new DateTimeField(array(
-                'id' => 3105,
-                'label' => __('Created'),
-            )),
-            'est_duedate'   =>  new DateTimeField(array(
-                'id' => 3106,
-                'label' => __('Due Date'),
-            )),
+    function getCurrentSearchFields($source=array()) {
+        static $basic = array(
+            'Ticket' => array(
+                'status__state',
+                'dept_id',
+                'assignee',
+                'topic_id',
+                'created',
+                'est_duedate',
+            )
         );
 
-        // Add 'other' fields added dynamically
-        if (is_array($source) && isset($source['fields'])) {
-            $extended = self::getExtendedTicketFields();
-            foreach ($source['fields'] as $f) {
-                $info = array();
-                if (isset($extended[$f])) {
-                    $core[$f] = $extended[$f];
-                    continue;
-                }
-                if (!preg_match('/^:(\w+)!(\d+)/', $f, $info)) {
-                    continue;
-                }
-                $id = $info[2];
-                if (is_numeric($id) && ($field = DynamicFormField::lookup($id))) {
-                    $impl = $field->getImpl();
-                    $impl->set('label', sprintf('%s / %s',
-                        $field->form->getLocal('title'), $field->getLocal('label')
-                    ));
-                    $core[":{$info[1]}!{$info[2]}"] = $impl;
-                }
-            }
+        $all = $this->getSupportedMatches();
+        $core = array();
+        foreach ($basic[$this->getRoot()] as $path)
+            if (isset($all[$path]))
+                $core[$path] = $all[$path];
+
+        // Add others from current configuration
+        foreach ($this->getCriteria() as $C) {
+            list($path) = $C;
+            if (isset($all[$path]))
+                $core[$path] = $all[$path];
         }
+
+        if (isset($source['fields']))
+            foreach ($source['fields'] as $path)
+                if (isset($all[$path]))
+                    $core[$path] = $all[$path];
+
         return $core;
     }
-    static function getSupportedTicketMatches() {
-        // User information
-        $matches = array(
-            __('Ticket Built-In') => SavedSearch::getExtendedTicketFields(),
-        );
-        foreach (array('ticket'=>'TicketForm', 'user'=>'UserForm', 'organization'=>'OrganizationForm') as $k=>$F) {
-            $form = $F::objects()->one();
-            $fields = &$matches[$form->getLocal('title')];
-            foreach ($form->getFields() as $f) {
-                if (!$f->hasData() || $f->isPresentationOnly())
-                    continue;
-                $fields[":$k!".$f->get('id')] = __(ucfirst($k)).' / '.$f->getLocal('label');
-                /* TODO: Support matches on list item properties
-                if (($fi = $f->getImpl()) && $fi->hasSubFields()) {
-                    foreach ($fi->getSubFields() as $p) {
-                        $fields[":$k.".$f->get('id').'.'.$p->get('id')]
-                            = __(ucfirst($k)).' / '.$f->getLocal('label').' / '.$p->getLocal('label');
-                    }
-                }
-                */
-            }
-        }
-        $fields = &$matches[__('Custom Forms')];
-        foreach (DynamicForm::objects()->filter(array('type'=>'G')) as $form) {
-            foreach ($form->getFields() as $f) {
-                if (!$f->hasData() || $f->isPresentationOnly())
-                    continue;
-                $key = sprintf(':field!%d', $f->get('id'), $f->get('id'));
-                $fields[$key] = $form->getLocal('title').' / '.$f->getLocal('label');
-            }
-        }
-        return $matches;
-    }
 
-    static function getExtendedTicketFields() {
-        return array(
-#            ':user' =>       new UserChoiceField(array(
-#                'label' => __('Ticket Owner'),
-#            )),
-#            ':org' =>        new OrganizationChoiceField(array(
-#                'label' => __('Organization'),
-#            )),
-            ':closed' =>     new DatetimeField(array(
-                'id' => 3204,
-                'label' => __('Closed Date'),
-            )),
-            ':thread__lastresponse' => new DatetimeField(array(
-                'id' => 3205,
-                'label' => __('Last Response'),
-            )),
-            ':thread__lastmessage' => new DatetimeField(array(
-                'id' => 3206,
-                'label' => __('Last Message'),
-            )),
-            ':source' =>     new TicketSourceChoiceField(array(
-                'id' => 3201,
-                'label' => __('Source'),
-            )),
-            ':state' =>      new TicketStateChoiceField(array(
-                'id' => 3202,
-                'label' => __('State'),
-            )),
-            ':flags' =>      new TicketFlagChoiceField(array(
-                'id' => 3203,
-                'label' => __('Flags'),
-            )),
-        );
+    function getSupportedMatches() {
+        return static::getSearchableFields($this->getRoot());
     }
 
     /**
+     * Trace ORM fields from a base object and retrieve a complete list of
+     * fields which can be used in an ORM query based on the base object.
+     * The base object must implement Searchable interface and extend from
+     * VerySimpleModel. Then all joins from the object are also inspected,
+     * and any which implement the Searchable interface are traversed and
+     * automatically added to the list. The resulting list is cached based
+     * on the $base class, so multiple calls for the same $base return
+     * quickly.
+     *
      * Parameters:
      * $base - Class, name of a class implementing Searchable
      * $recurse - int, number of levels to recurse, default is 2
@@ -910,7 +824,7 @@ class SavedSearch extends VerySimpleModel {
      *      forms
      */
     static function getSearchableFields($base, $recurse=2, $cache=true,
-        $customData=true
+        $customData=true, $exclude=array()
     ) {
         static $cache, $otherFields;
 
@@ -934,11 +848,13 @@ class SavedSearch extends VerySimpleModel {
         }
 
         if ($recurse) {
+            $exclude[$base] = 1;
             foreach ($base::getMeta('joins') as $path=>$j) {
                 $fc = $j['fkey'][0];
-                if ($fc == $base || $j['list'] || $j['reverse'])
+                if (isset($exclude[$fc]) || $j['list'])
                     continue;
-                foreach (static::getSearchableFields($fc, $recurse-1, false)
+                foreach (static::getSearchableFields($fc, $recurse-1, false,
+                    true, $exclude)
                 as $path2=>$F) {
                     if (is_array($F)) {
                         list($label, $field) = $F;
@@ -978,19 +894,19 @@ class SavedSearch extends VerySimpleModel {
         return $fields;
     }
 
-    static function getSearchField($field, $name) {
-        $baseId = $field->getId() * 20;
+    static function getSearchField($F, $name) {
+        list($label, $field) = $F;
+
         $pieces = array();
         $pieces["{$name}+search"] = new BooleanField(array(
-            'id' => $baseId + 50000,
+            'id' => sprintf('%u', crc32($name)) >> 1,
             'configuration' => array(
-                'desc' => $field->getLocal('label'),
+                'desc' => $label ?: $field->getLocal('label'),
                 'classes' => 'inline',
             ),
         ));
         $methods = $field->getSearchMethods();
         $pieces["{$name}+method"] = new ChoiceField(array(
-            'id' => $baseId + 50001,
             'choices' => $methods,
             'default' => key($methods),
             'visibility' => new VisibilityConstraint(new Q(array(
@@ -1002,7 +918,6 @@ class SavedSearch extends VerySimpleModel {
             if (!$w)
                 continue;
             list($class, $args) = $w;
-            $args['id'] = $baseId + 50002 + $offs++;
             $args['required'] = true;
             $args['__searchval__'] = true;
             $args['visibility'] = new VisibilityConstraint(new Q(array(
@@ -1013,17 +928,16 @@ class SavedSearch extends VerySimpleModel {
         return $pieces;
     }
 
-    /**
-     * Collect information on the search form.
-     *
-     * Returns:
-     * (<array(name => array('field' => <FormField>, 'method' => <string>,
-     *      'value' => <mixed>, 'active' => <bool>))>), which will help to
-     * explain each field active in the search form.
-     */
+    function getField($path) {
+        $searchable = $this->getSupportedMatches();
+        return $searchable[$path];
+    }
+
+    // Remove this and adjust advanced-search-criteria template to use the
+    // getCriteria() list and getField()
     function getSearchFields($form=false) {
         $form = $form ?: $this->getForm();
-        $searchable = $this->getCurrentSearchFields($form->state);
+        $searchable = $this->getCurrentSearchFields();
         $info = array();
         foreach ($form->getFields() as $f) {
             if (substr($f->get('name'), -7) == '+search') {
@@ -1032,7 +946,7 @@ class SavedSearch extends VerySimpleModel {
                 // Determine the search method and fetch the original field
                 if (($M = $form->getField("{$name}+method"))
                     && ($method = $M->getClean())
-                    && ($field = $searchable[$name])
+                    && (list(,$field) = $searchable[$name])
                 ) {
                     // Request the field to generate a search Q for the
                     // search method and given value
@@ -1050,6 +964,104 @@ class SavedSearch extends VerySimpleModel {
         return $info;
     }
 
+    /**
+     * Take the criteria from the SavedSearch fields setup and isolate the
+     * field name being search, the method used for searhing, and the method-
+     * specific data entered in the UI.
+     */
+    function isolateCriteria($criteria, $root=null) {
+        $searchable = static::getSearchableFields($root ?: $this->getRoot());
+        $items = array();
+        if (!$criteria)
+            return null;
+        foreach ($criteria as $k=>$v) {
+            if (substr($k, -7) === '+method') {
+                list($name,) = explode('+', $k, 2);
+                if (!isset($searchable[$name]))
+                    continue;
+
+                // Require checkbox to be checked too
+                if (!$criteria["{$name}+search"])
+                    continue;
+
+                // Lookup the field to search this condition
+                list($label, $field) = $searchable[$name];
+
+                // Get the search method and value
+                $method = $v;
+                // Not all search methods require a value
+                $value = $criteria["{$name}+{$method}"];
+
+                $items[] = array($name, $method, $value);
+            }
+        }
+        return $items;
+    }
+
+    function getColumns() {
+        if ($this->columns_id
+            && ($q = CustomQueue::lookup($this->columns_id))
+        ) {
+            // Use columns from cited queue
+            return $q->getColumns();
+        }
+
+        if (isset($this->columns))
+            return $this->columns;
+
+        // Last resort — use standard columns
+        $this->columns = array(
+            QueueColumn::create(array(
+                "id" => 1,
+                "heading" => "Number",
+                "primary" => 'number',
+                "width" => 100,
+                "filter" => "link:ticketP",
+                "annotations" => '[{"c":"TicketSourceDecoration","p":"b"}]',
+                "conditions" => '[{"crit":["isanswered","set",null],"prop":{"font-weight":"bold"}}]',
+            )),
+            QueueColumn::create(array(
+                "id" => 2,
+                "heading" => "Created",
+                "primary" => 'created',
+                "width" => 100,
+            )),
+            QueueColumn::create(array(
+                "id" => 3,
+                "heading" => "Subject",
+                "primary" => 'cdata__subject',
+                "width" => 250,
+                "filter" => "link:ticket",
+                "annotations" => '[{"c":"TicketThreadCount","p":">"},{"c":"ThreadAttachmentCount","p":"a"},{"c":"OverdueFlagDecoration","p":"<"}]',
+                "truncate" => 'ellipsis',
+            )),
+            QueueColumn::create(array(
+                "id" => 4,
+                "heading" => "From",
+                "primary" => 'user__name',
+                "width" => 150,
+            )),
+            QueueColumn::create(array(
+                "id" => 5,
+                "heading" => "Priority",
+                "primary" => 'cdata__priority',
+                "width" => 120,
+            )),
+            QueueColumn::create(array(
+                "id" => 6,
+                "heading" => "Assignee",
+                "primary" => 'assignee',
+                "secondary" => 'team__name',
+                "width" => 100,
+            )),
+        );
+        
+        foreach ($this->columns as $c)
+            $c->queue = $this;
+
+        return $this->columns;
+    }
+
     /**
      * Get a description of a field in a search. Expects an entry from the
      * array retrieved in ::getSearchFields()
@@ -1058,68 +1070,82 @@ class SavedSearch extends VerySimpleModel {
         return $info['field']->describeSearch($info['method'], $info['value'], $name);
     }
 
+    function getQuery() {
+        $root = $this->getRoot();
+        $base = $root::objects();
+        $query = $this->mangleQuerySet($base);
+
+        // Apply column, annotations and conditions additions
+        foreach ($this->getColumns() as $C) {
+            $query = $C->mangleQuery($query);
+        }
+        return $query;
+    }
+
     function mangleQuerySet(QuerySet $qs, $form=false) {
-        $form = $form ?: $this->getForm();
-        $searchable = $this->getCurrentSearchFields($form->state);
         $qs = clone $qs;
+        $searchable = $this->getSupportedMatches();
 
         // Figure out fields to search on
-        foreach ($this->getSearchFields($form) as $name=>$info) {
-            if (!$info['active'])
-                continue;
-            $field = $info['field'];
-            $filter = new Q();
-            if ($name[0] == ':') {
-                // This was an 'other' field, fetch a special "name"
-                // for it which will be the ORM join path
-                static $other_paths = array(
-                    ':ticket' => 'cdata__',
-                    ':user' => 'user__cdata__',
-                    ':organization' => 'user__org__cdata__',
-                );
-                $column = $field->get('name') ?: 'field_'.$field->get('id');
-                list($type,$id) = explode('!', $name, 2);
-                // XXX: Last mile — find a better idea
-                switch (array($type, $column)) {
-                case array(':user', 'name'):
-                    $name = 'user__name';
-                    break;
-                case array(':user', 'email'):
-                    $name = 'user__emails__address';
-                    break;
-                case array(':organization', 'name'):
-                    $name = 'user__org__name';
-                    break;
-                default:
-                    if ($type == ':field' && $id) {
-                        $name = 'entries__answers__value';
-                        $filter->add(array('entries__answers__field_id' => $id));
-                        break;
-                    }
-                    if ($OP = $other_paths[$type])
-                        $name = $OP . $column;
-                    else
-                        $name = substr($name, 1);
-                }
+        foreach ($this->getCriteria() as $I) {
+            list($name, $method, $value) = $I;
+
+            // Consider keyword searching
+            if ($name === ':keywords') {
+                global $ost;
+                $qs = $ost->searcher->find($keywords, $qs);
             }
+            else {
+                // XXX: Move getOrmPath to be more of a utility
+                // Ensure the special join is created to support custom data joins
+                $name = @static::getOrmPath($name, $qs);
+
+                $name2 = null;
+                if (preg_match('/__answers!\d+__/', $name)) {
+                    // Ensure that only one record is returned from the join through
+                    // the entry and answers joins
+                    $name2 = $this->getAnnotationName().'2';
+                    $query->annotate(array($name2 => SqlAggregate::MAX($name)));
+                }
 
-            // Add the criteria to the QuerySet
-            if ($Q = $field->getSearchQ($info['method'], $info['value'], $name)) {
-                $filter->add($Q);
-                $qs = $qs->filter($filter);
+                // Fetch a criteria Q for the query
+                if (list(,$field) = $searchable[$name])
+                    if ($q = $field->getSearchQ($method, $value, $name2 ?: $name))
+                        $qs = $qs->filter($q);
             }
         }
+        return $qs;
+    }
 
-        // Consider keyword searching
-        if ($keywords = $form->getField('keywords')->getClean()) {
-            global $ost;
-
-            $qs = $ost->searcher->find($keywords, $qs);
+    function getOrmPath($name, $query=null) {
+        // Special case for custom data `__answers!id__value`. Only add the
+        // join and constraint on the query the first pass, when the query
+        // being mangled is received.
+        $path = array();
+        if ($query && preg_match('/^(.+?)__(answers!(\d+))/', $name, $path)) {
+            // Add a join to the model of the queryset where the custom data
+            // is forked from — duplicate the 'answers' join and add the
+            // constraint to the query based on the field_id
+            // $path[1] - part before the answers (user__org__entries)
+            // $path[2] - answers!xx join part
+            // $path[3] - the `xx` part of the answers!xx join component
+            $root = $query->model;
+            $meta = $root::getMeta()->getByPath($path[1]);
+            $joins = $meta['joins'];
+            if (!isset($joins[$path[2]])) {
+                $meta->addJoin($path[2], $joins['answers']);
+            }
+            // Ensure that the query join through answers!xx is only for the
+            // records which match field_id=xx
+            $query->constrain(array("{$path[1]}__{$path[2]}" =>
+                array("{$path[1]}__{$path[2]}__field_id" => (int) $path[3])
+            ));
+            // Leave $name unchanged
         }
-
-        return $qs;
+        return $name;
     }
 
+
     function checkAccess(Staff $agent) {
         return $agent->getId() == $this->staff_id
             || $this->hasFlag(self::FLAG_PUBLIC);
@@ -1165,15 +1191,21 @@ class SavedSearch extends VerySimpleModel {
     }
 }
 
-class AdvancedSearchForm extends SimpleForm {
-    var $state;
+class AdhocSearch
+extends SavedSearch {
+    function getName() {
+        return __('Ad-Hoc Search');
+    }
 
-    function __construct($fields, $state) {
-        parent::__construct($fields);
-        $this->state = $state;
+    function getHref() {
+        return 'tickets.php?queue=adhoc';
     }
 }
 
+class AdvancedSearchForm extends SimpleForm {
+    static $id = 1337;
+}
+
 // Advanced search special fields
 
 class HelpTopicChoiceField extends ChoiceField {
@@ -1318,6 +1350,12 @@ class AgentSelectionField extends ChoiceField {
     }
 }
 
+class TeamSelectionField extends ChoiceField {
+    function getChoices() {
+        return Team::getTeams();
+    }
+}
+
 class TicketStateChoiceField extends ChoiceField {
     function getChoices($verbose=false) {
         return array(
@@ -1370,13 +1408,7 @@ class TicketFlagChoiceField extends ChoiceField {
 
 class TicketSourceChoiceField extends ChoiceField {
     function getChoices($verbose=false) {
-        return array(
-            'web' => __('Web'),
-            'email' => __('Email'),
-            'phone' => __('Phone'),
-            'api' => __('API'),
-            'other' => __('Other'),
-        );
+        return Ticket::getSources();
     }
 
     function getSearchMethods() {
diff --git a/include/class.thread.php b/include/class.thread.php
index 256b09cfd..af92d3a5f 100644
--- a/include/class.thread.php
+++ b/include/class.thread.php
@@ -19,7 +19,8 @@ include_once(INCLUDE_DIR.'class.draft.php');
 include_once(INCLUDE_DIR.'class.role.php');
 
 //Ticket thread.
-class Thread extends VerySimpleModel {
+class Thread extends VerySimpleModel
+implements Searchable {
     static $meta = array(
         'table' => THREAD_TABLE,
         'pk' => array('id'),
@@ -509,6 +510,21 @@ class Thread extends VerySimpleModel {
         return null;
     }
 
+    static function getSearchableFields() {
+        return array(
+            'lastmessage' => new DatetimeField(array(
+                'label' => __('Last Message'),
+            )),
+            'lastresponse' => new DatetimeField(array(
+                'label' => __('Last Response'),
+            )),
+        );
+    }
+
+    static function supportsCustomData() {
+        false;
+    }
+
     function delete() {
 
         //Self delete
diff --git a/include/class.ticket.php b/include/class.ticket.php
index 184963136..4a424f672 100644
--- a/include/class.ticket.php
+++ b/include/class.ticket.php
@@ -41,6 +41,8 @@ implements RestrictedAccess, Threadable, Searchable {
     static $meta = array(
         'table' => TICKET_TABLE,
         'pk' => array('ticket_id'),
+        'select_related' => array('topic', 'staff', 'user', 'team', 'dept',
+            'sla', 'thread', 'user__default_email', 'status'),
         'joins' => array(
             'user' => array(
                 'constraint' => array('user_id' => 'User.id')
@@ -101,7 +103,6 @@ implements RestrictedAccess, Threadable, Searchable {
     const PERM_CLOSE    = 'ticket.close';
     const PERM_DELETE   = 'ticket.delete';
 
-
     static protected $perms = array(
             self::PERM_CREATE => array(
                 'title' =>
@@ -1860,20 +1861,10 @@ implements RestrictedAccess, Threadable, Searchable {
             'number' => new TextboxField(array(
                 'label' => __('Ticket Number')
             )),
-            'ip_address' => new TextboxField(array(
-                'label' => __('IP Address'),
-                'configuration' => array('validator' => 'ip'),
-            )),
-            'source' => new TicketSourceChoiceField(array(
-                'label' => __('Ticket Source'),
-            )),
-            'isoverdue' => new BooleanField(array(
-                'label' => __('Overdue'),
-            )),
-            'isanswered' => new BooleanField(array(
-                'label' => __('Answered'),
+            'created' => new DatetimeField(array(
+                'label' => __('Create Date'),
             )),
-            'duedate' => new DatetimeField(array(
+            'est_duedate' => new DatetimeField(array(
                 'label' => __('Due Date'),
             )),
             'reopened' => new DatetimeField(array(
@@ -1885,12 +1876,34 @@ implements RestrictedAccess, Threadable, Searchable {
             'lastupdate' => new DatetimeField(array(
                 'label' => __('Last Update'),
             )),
-            'created' => new DatetimeField(array(
-                'label' => __('Create Date'),
-            )),
             'assignee' => new AssigneeChoiceField(array(
                 'label' => __('Assignee'),
             )),
+            'staff_id' => new AgentSelectionField(array(
+                'label' => __('Assigned Staff'),
+            )),
+            'team_id' => new TeamSelectionField(array(
+                'label' => __('Assigned Team'),
+            )),
+            'dept_id' => new DepartmentChoiceField(array(
+                'label' => __('Department'),
+            )),
+            'topic_id' => new HelpTopicChoiceField(array(
+                'label' => __('Help Topic'),
+            )),
+            'source' => new TicketSourceChoiceField(array(
+                'label' => __('Ticket Source'),
+            )),
+            'isoverdue' => new BooleanField(array(
+                'label' => __('Overdue'),
+            )),
+            'isanswered' => new BooleanField(array(
+                'label' => __('Answered'),
+            )),
+            'ip_address' => new TextboxField(array(
+                'label' => __('IP Address'),
+                'configuration' => array('validator' => 'ip'),
+            )),
         );
         $tform = TicketForm::getInstance();
         foreach ($tform->getFields() as $F) {
diff --git a/include/i18n/en_US/queue.yaml b/include/i18n/en_US/queue.yaml
new file mode 100644
index 000000000..ce7b6a344
--- /dev/null
+++ b/include/i18n/en_US/queue.yaml
@@ -0,0 +1,94 @@
+#
+# Basic queues for the initial ticket system. Queues installed for
+# - Open / All
+# - Open / Answered
+# - Open / Unanswered
+# - Open / Unassigned
+# - Open / Mine
+# - Closed / All
+#
+# Fields:
+# id:       
+# parent_id:
+# flags:
+#   0x01:   FLAG_PUBLIC
+#   0x02:   FLAG_QUEUE (should be set for everything here)
+#   0x04:   FLAG_CONTAINER (should be set for top-level queues)
+#   0x08:   FLAG_INHERIT (inherit criteria from parent)
+#   0x10:   FLAG_DEFAULT (default queue for parent container)
+#   0x20:   FLAG_DRAFT
+# staff_id: User owner of the queue
+# sort:     Manual sort order
+# title:    Display name of the queue
+# config:   Criteria configuration
+# filter:   Quick filter field
+# root:     Object type of the queue listing
+#   'T':    Tickets
+#   'A':    Tasks
+#
+# Columns are not necessary and a default list is used if no columns are
+# specified.
+#
+# columns:  Array of column instances with these fields
+#   flags:      (unused)
+#   sort:       Manual sort order of the queue
+#   heading:    Display name of the column header
+#   primary:    Data source for the field
+#   secondary:  Backup data source / default text
+#   width:      Width weight of the column
+#   filter:     What the field should link to
+#     'link:ticket':    Ticket
+#     'link:user':      User
+#     'link:org':       Organization
+#     'link:ticketP':   Ticket with hover preview
+#   truncate:
+#     'wrap':   Fold words on multiple lines
+#   annotations:
+#     c:        Annotation class name
+#     p:        Placement
+#       'a':    After column text
+#       'b':    Before column text
+#       '<':    Float to start (left)
+#       '>':    Float to end (right)
+#   conditions:
+#     crit:     Criteria for the condiditon, in the form of [field, method, value]
+#     prop:     Array of CSS properties to apply to the field
+#       'font-weight':
+#       'font-style':
+#       ...
+#   extra:      (future use and for plugins)
+---
+- id: 1
+  title: Open
+  flags: 0x03
+  sort: 1
+  root: T 
+  config: '[["status__state","includes",{"open":"Open"}]]'
+
+- id: 2
+  title: Closed
+  flags: 0x03
+  sort: 2
+  root: T
+  config: '[["status__state","includes",{"closed":"Closed"}]]'
+
+- title: Unanswered
+  parent_id: 1
+  flags: 0x0b
+  root: T
+  sort: 1
+  config: '[["status__state","includes",{"open":"Open"}],["answered","nset",null]]'
+
+- title: Unassigned
+  parent_id: 1
+  flags: 0x0b
+  root: T
+  sort: 2
+  config: '[["assignee","unassigned",null]]
+
+- title: My Tickets
+  parent_id: 1
+  flags: 0x0b
+  root: T
+  sort: 4
+  config: '[["assignee","includes",{"M":"Me", "T":"One of my teams"}]]'
diff --git a/include/staff/queue.inc.php b/include/staff/queue.inc.php
index bff6a39b7..d873ccba3 100644
--- a/include/staff/queue.inc.php
+++ b/include/staff/queue.inc.php
@@ -63,9 +63,9 @@ else {
         <div class="error"><?php echo $errors['criteria']; ?></div>
         <div class="advanced-search">
 <?php
-            $form = $queue->getSearchForm();
+            $form = $queue->getForm();
             $search = $queue;
-            $matches = SavedSearch::getSupportedTicketMatches();
+            $matches = $queue->getSupportedMatches();
             include STAFFINC_DIR . 'templates/advanced-search-criteria.tmpl.php';
 ?>
         </div>
diff --git a/include/staff/queues-ticket.inc.php b/include/staff/queues-ticket.inc.php
index f051d5b7c..ef12f8873 100644
--- a/include/staff/queues-ticket.inc.php
+++ b/include/staff/queues-ticket.inc.php
@@ -91,7 +91,8 @@ $emitLevel = function($queues, $level=0) use ($all_queues, &$emitLevel) {
         <td width="63%" colspan="<?php echo max(1, 5-$level); ?>"><a
           href="queues.php?id=<?php echo $q->getId(); ?>"><?php
           echo Format::htmlchars($q->getFullName()); ?></a></td>
-        <td><?php echo Format::htmlchars($q->staff->getName()); ?></td>
+        <td><?php echo Format::htmlchars($q->staff ? $q->staff->getName() :
+        __('SYSTEM')); ?></td>
         <td><?php echo Format::htmlchars($q->getStatus()); ?></td>
         <td><?php echo Format::date($q->created); ?></td>
       </tr>
diff --git a/include/staff/templates/advanced-search-criteria.tmpl.php b/include/staff/templates/advanced-search-criteria.tmpl.php
index ea343245a..29bea883a 100644
--- a/include/staff/templates/advanced-search-criteria.tmpl.php
+++ b/include/staff/templates/advanced-search-criteria.tmpl.php
@@ -75,15 +75,11 @@ if (!$first_field)
     <option value="">— <?php echo __('Add Other Field'); ?> —</option>
 <?php
 if (is_array($matches)) {
-foreach ($matches as $name => $fields) { ?>
-    <optgroup label="<?php echo $name; ?>">
-<?php
-    foreach ($fields as $id => $desc) { ?>
-        <option value="<?php echo $id; ?>" <?php
-            if (isset($state[$id])) echo 'disabled="disabled"';
-        ?>><?php echo ($desc instanceof FormField ? $desc->getLocal('label') : $desc); ?></option>
-<?php } ?>
-    </optgroup>
+foreach ($matches as $path => $F) {
+    list($label, $field) = $F; ?>
+    <option value="<?php echo $path; ?>" <?php
+        if (isset($state[$path])) echo 'disabled="disabled"';
+        ?>><?php echo $label; ?></option>
 <?php }
 } ?>
 </select>
@@ -98,7 +94,6 @@ $(function() {
       success: function(json) {
         if (!json.success)
           return false;
-        ff_uid = json.ff_uid;
         $(that).find(':selected').prop('disabled', true);
         $('#extra-fields').append($(json.html));
       }
diff --git a/include/staff/templates/queue-column-condition.tmpl.php b/include/staff/templates/queue-column-condition.tmpl.php
index d9861890b..40f6d6fb6 100644
--- a/include/staff/templates/queue-column-condition.tmpl.php
+++ b/include/staff/templates/queue-column-condition.tmpl.php
@@ -20,7 +20,7 @@
   <?php echo $label ?: $field->getLabel(); ?>
   <div class="advanced-search">
 <?php
-$parts = SavedSearch::getSearchField($field, $field_name);
+$parts = SavedSearch::getSearchField(array($label, $field), $field_name);
 // Drop the search checkbox field
 unset($parts["{$field_name}+search"]);
 list(, $crit_method, $crit_value) = $condition->getCriteria();
diff --git a/include/upgrader/streams/core/98ad7d55-00000000.patch.sql b/include/upgrader/streams/core/98ad7d55-00000000.patch.sql
index 22530ecb5..b253c9efd 100644
--- a/include/upgrader/streams/core/98ad7d55-00000000.patch.sql
+++ b/include/upgrader/streams/core/98ad7d55-00000000.patch.sql
@@ -1,6 +1,7 @@
 /**
  * @version v1.11
  * @signature 00000000000000000000000000000000
+ * @title Custom Queues, Columns
  *
  * Add custom queues, custom columns, and quick filter capabilities to the
  * system.
diff --git a/scp/tickets.php b/scp/tickets.php
index 4a5c2a6d6..297e285e2 100644
--- a/scp/tickets.php
+++ b/scp/tickets.php
@@ -393,10 +393,11 @@ as $q) {
 
 if (isset($_SESSION['advsearch'])) {
         // XXX: De-duplicate and simplify this code
-    $adhoc = SavedSearch::create(array('title' => __("Advanced Search")));
-    $form = $adhoc->getFormFromSession('advsearch');
-    $adhoc->config = $form->getState();
-
+    $adhoc = SavedSearch::create(array(
+        'title' => __("Advanced Search"),
+        'root' => 'T',
+    ));
+    $adhoc->config = $_SESSION['advsearch'];
     if ($_REQUEST['queue'] == 'adhoc')
         $queue = $adhoc;
 }
-- 
GitLab