From a9c283ec7fa4d5cbd67fcb08461df5d3b91f73fb Mon Sep 17 00:00:00 2001
From: Jared Hancock <gravydish@gmail.com>
Date: Fri, 19 Oct 2018 02:22:53 +0000
Subject: [PATCH] search: Refactor full-text search
This patch adds a few changes. First, the full-text search query is
refactored so that MySQL can focus first on finding matching records in
its full-text search index (in the %_search table) in a subquery. Then
it can focus on matching records against tickets in an outer query
(whether ticket data, thread entries, user info, or organization inf was
matched). This dramatically speeds up the query for large datasets by
about 20x (about three seconds instead of sixty-something).
Secondly, results are now sorted based on relevance, so the best hits
sort a the top of the list. This is accomplished by adding the
`relevance` to the query sort list via the extra() method. Then, since
no GROUP BY clause is really necessary, it is not added to the query,
which will allow the results not to be re-sorted by the ticket_id. (That
is, the remain sorted by the search relevance).
Third, the relevance has a lower limit of 0.3. Good hits seem to have a
much larger value (like hundreds), so setting a small value will help to
remove hits with barely any relevance to the search terms.
---
include/class.orm.php | 7 ++-----
include/class.search.php | 6 ++++--
include/staff/templates/queue-tickets.tmpl.php | 2 +-
3 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/include/class.orm.php b/include/class.orm.php
index 8393aeabb..b95735e0d 100644
--- a/include/class.orm.php
+++ b/include/class.orm.php
@@ -2629,10 +2629,7 @@ class SqlCompiler {
foreach ($queryset->extra['tables'] as $S) {
$join = ' JOIN ';
// Left joins require an ON () clause
- if ($lastparen = strrpos($S, '(')) {
- if (preg_match('/\bon\b/i', substr($S, $lastparen - 4, 4)))
- $join = ' LEFT' . $join;
- }
+ // TODO: Have a way to indicate a LEFT JOIN
$sql .= $join.$S;
}
}
@@ -3087,7 +3084,7 @@ class MySqlCompiler extends SqlCompiler {
}
}
// If no group by has been set yet, use the root model pk
- if (!$group_by && !$queryset->aggregated && !$queryset->distinct) {
+ if (!$group_by && !$queryset->aggregated && !$queryset->distinct && $need_group_by) {
foreach ($meta['pk'] as $pk)
$group_by[] = $rootAlias .'.'. $pk;
}
diff --git a/include/class.search.php b/include/class.search.php
index db7ef05c6..f5034d1f3 100644
--- a/include/class.search.php
+++ b/include/class.search.php
@@ -384,9 +384,11 @@ class MysqlSearchBackend extends SearchBackend {
$criteria->extra(array(
'tables' => array(
str_replace(array(':', '{}'), array(TABLE_PREFIX, $search),
- "(SELECT COALESCE(Z3.`object_id`, Z5.`ticket_id`, Z8.`ticket_id`) as `ticket_id`, SUM({}) AS `relevance` FROM `:_search` Z1 LEFT JOIN `:thread_entry` Z2 ON (Z1.`object_type` = 'H' AND Z1.`object_id` = Z2.`id`) LEFT JOIN `:thread` Z3 ON (Z2.`thread_id` = Z3.`id` AND Z3.`object_type` = 'T') LEFT JOIN `:ticket` Z5 ON (Z1.`object_type` = 'T' AND Z1.`object_id` = Z5.`ticket_id`) LEFT JOIN `:user` Z6 ON (Z6.`id` = Z1.`object_id` and Z1.`object_type` = 'U') LEFT JOIN `:organization` Z7 ON (Z7.`id` = Z1.`object_id` AND Z7.`id` = Z6.`org_id` AND Z1.`object_type` = 'O') LEFT JOIN :ticket Z8 ON (Z8.`user_id` = Z6.`id`) WHERE {} GROUP BY `ticket_id`) Z1"),
- )
+ "(SELECT COALESCE(Z3.`object_id`, Z5.`ticket_id`, Z8.`ticket_id`) as `ticket_id`, Z1.relevance FROM (SELECT Z1.`object_id`, Z1.`object_type`, {} AS `relevance` FROM `:_search` Z1 WHERE {} > 0.3 ORDER BY relevance DESC) Z1 LEFT JOIN `:thread_entry` Z2 ON (Z1.`object_type` = 'H' AND Z1.`object_id` = Z2.`id`) LEFT JOIN `:thread` Z3 ON (Z2.`thread_id` = Z3.`id` AND Z3.`object_type` = 'T') LEFT JOIN `:ticket` Z5 ON (Z1.`object_type` = 'T' AND Z1.`object_id` = Z5.`ticket_id`) LEFT JOIN `:user` Z6 ON (Z6.`id` = Z1.`object_id` and Z1.`object_type` = 'U') LEFT JOIN `:organization` Z7 ON (Z7.`id` = Z1.`object_id` AND Z7.`id` = Z6.`org_id` AND Z1.`object_type` = 'O') LEFT JOIN `:ticket` Z8 ON (Z8.`user_id` = Z6.`id`)) Z1"),
+ ),
));
+ $criteria->extra(array('order_by' => array(array(new SqlCode('Z1.relevance', 'DESC')))));
+
$criteria->filter(array('ticket_id'=>new SqlCode('Z1.`ticket_id`')));
break;
diff --git a/include/staff/templates/queue-tickets.tmpl.php b/include/staff/templates/queue-tickets.tmpl.php
index 50a81b024..fe82ff9a2 100644
--- a/include/staff/templates/queue-tickets.tmpl.php
+++ b/include/staff/templates/queue-tickets.tmpl.php
@@ -82,7 +82,7 @@ $tickets = $pageNav->paginateSimple($tickets);
// criteria, sort, limit, and offset from the outer query.
$criteria = clone $tickets;
$criteria->annotations = $criteria->related = $criteria->aggregated = [];
-$tickets->constraints = $tickets->extra['tables'] = [];
+$tickets->constraints = $tickets->extra = [];
$tickets = $tickets->filter(['ticket_id__in' => $criteria->values_flat('ticket_id')])
->limit(false)->offset(false)->order_by(false);
# Index hint should be used on the $criteria query only
--
GitLab