Bug 38694: Add ESBoostFieldMatch option to Elasticsearch

What this patch does is:
1 - Wraps the existing search code in a a "bool" compound query as a "must". This should not affect
    relevancy or results of the existing searches.
2 - Before we clean/truncate terms, loop through the passed in search terms and indexes to build a
    new 'should' query, using the 'match' on the specified index/field that is added to the 'bool' query from above.
    This means that if a result from the original query is also returned here, that item will be boosted in the result.
    For searches on 'keyword' or 'title', or if no index is set, we use 'title-cover' as the most narrow form of title

This query isn't going to help when users enter CCL (i.e. ti:To die for) and it won't boost titles from 505, series, etc
when doing a general search. Nor will it have a detrimental effect, it will only boost field matches

To test:
1 - Add a record with
    245 $a novel
2 - Add a record with
    245 $a A novel : $b about things / $c by me
3 - Search for: novel
    - Above records are returned lower in the list (results #6 and #14 for me)
4 - Search for: a novel
    - Above records are returned lower in the list (results #6 and #14 for me)
5 - Apply patch, restart all, enable ESBoostFieldMatch option
6 - Repeat searches, note exact titles are boosted
    - For 'novel' record from #1 is #1 result and other is second
    - For 'a novel' record form #2 is #1 result and other is third
7 - Disable the new pref and repeat steps 3 and 4 and get results as before patch
8 - Experiment with other searches, turning pref on and off to verify relevant titles are boosted when enabled
9 - Search results when disabled should return as before the patch

Signed-off-by: Matthias Meusburger <matthias.meusburger@biblibre.com>
Signed-off-by: Thomas Klausner <domm@plix.at>
Signed-off-by: Katrin Fischer <katrin.fischer@bsz-bw.de>

Author: kidclamp

Koha/SearchEngine/Elasticsearch/QueryBuilder.pm

@@ -203,17 +203,19 @@ sub build_query {
     if ( $options{whole_record} ) {
         push @$fields, 'marc_data_array.*';
     }
-    $res->{query} = {
-        query_string => {
-            query            => $query,
-            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
-            default_operator => 'AND',
-            fields           => $fields,
-            lenient          => JSON::true,
-            analyze_wildcard => JSON::true,
-        }
+    my $query_string = {
+        query            => $query,
+        fuzziness        => $fuzzy_enabled ? 'auto' : '0',
+        default_operator => 'AND',
+        fields           => $fields,
+        lenient          => JSON::true,
+        analyze_wildcard => JSON::true,
     };
-    $res->{query}->{query_string}->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields');
+    $query_string->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields');
+
+    $res->{query} = { bool => { must => [ { query_string => $query_string } ] } };
+
+    $res->{query}->{bool}->{should} = $options{field_match_boost_query} if $options{field_match_boost_query};
 
     if ( $options{sort} ) {
         foreach my $sort ( @{ $options{sort} } ) {
@@ -288,6 +290,10 @@ sub build_query_compat {
     } else {
         my @sort_params  = $self->_convert_sort_fields(@$sort_by);
         my @index_params = $self->_convert_index_fields(@$indexes);
+        my $field_match_boost_query =
+            C4::Context->preference('ESBoostFieldMatch')
+            ? $self->_build_field_match_boost_query( { operands => $operands, indexes => \@index_params } )
+            : [];
         $limits = $self->_fix_limit_special_cases($orig_limits);
         if ( $params->{suppress} ) { push @$limits, "suppress:false"; }
 
@@ -328,12 +334,13 @@ sub build_query_compat {
         # If there's no query on the left, let's remove the junk left behind
         $query_str =~ s/^ AND //;
         my %options;
-        $options{sort}            = \@sort_params;
-        $options{is_opac}         = $params->{is_opac};
-        $options{weighted_fields} = $params->{weighted_fields};
-        $options{whole_record}    = $params->{whole_record};
-        $options{skip_facets}     = $params->{skip_facets};
-        $query                    = $self->build_query( $query_str, %options );
+        $options{sort}                    = \@sort_params;
+        $options{is_opac}                 = $params->{is_opac};
+        $options{weighted_fields}         = $params->{weighted_fields};
+        $options{whole_record}            = $params->{whole_record};
+        $options{skip_facets}             = $params->{skip_facets};
+        $options{field_match_boost_query} = $field_match_boost_query if @$field_match_boost_query;
+        $query                            = $self->build_query( $query_str, %options );
     }
 
     # We roughly emulate the CGI parameters of the zebra query builder
@@ -369,6 +376,30 @@ sub build_query_compat {
     );
 }
 
+=head2 _build_field_match_boost_query
+
+    my ($query, $query_str) = $builder->_build_field_match_boost_query({ operands => \@operands, indexes => \@indexes)
+
+This will build an array of match queries for terms and indexes passed in and return a reference to the array.
+
+=cut
+
+sub _build_field_match_boost_query {
+    my ( $self, $params ) = @_;
+    my $indexes  = $params->{indexes};
+    my $operands = $params->{operands};
+
+    my @boost_query;
+    my $ea = each_array( @$operands, @$indexes );
+    while ( my ( $operand, $index ) = $ea->() ) {
+        next unless $operand;
+        $index = $index->{field} if ref $index eq 'HASH';
+        $index = 'title-cover'   if ( !$index || $index eq 'kw' || $index eq 'ti' || $index eq 'title' );
+        push @boost_query, { match => { $index => { query => $operand } } };
+    }
+    return \@boost_query;
+}
+
 =head2 build_authorities_query
 
     my $query = $builder->build_authorities_query(\%search);

installer/data/mysql/atomicupdate/bug38694.pl

@@ -0,0 +1,21 @@
+use Modern::Perl;
+use Koha::Installer::Output qw(say_warning say_success say_info);
+
+return {
+    bug_number  => "38694",
+    description => "Add ESBoostFieldMatch system preference",
+    up          => sub {
+        my ($args) = @_;
+        my ( $dbh, $out ) = @$args{qw(dbh out)};
+
+        # Do you stuffs here
+        $dbh->do(
+            q{
+           INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES
+           ('ESBoostFieldMatch', '0', NULL, 'Add a "match" query to es when searching, will follow indexes chosen in advanced search, or use title-cover for generic keyword or title index search', 'YesNo')
+       }
+        );
+
+        say_success( $out, "Added new system preference 'ESBoostFieldMatch'" );
+    },
+};

installer/data/mysql/mandatory/sysprefs.sql

@@ -261,6 +261,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `
 ('ERMProviderEbscoApiKey', '', '', 'API key for EBSCO', 'free'),
 ('ERMProviderEbscoCustomerID', '', '', 'Customer ID for EBSCO', 'free'),
 ('ERMProviders', 'local', 'local|ebsco', 'Set the providers for the ERM module', 'Choice'),
+('ESBoostFieldMatch', '0', NULL, 'Add a "match" query to es when searching, will follow indexes chosen in advanced search, or use title-cover for generic keyword or title index search', 'YesNo'),
 ('ESPreventAutoTruncate', 'barcode|control-number|control-number-identifier|date-of-acquisition|date-of-publication|date-time-last-modified|identifier-standard|isbn|issn|itype|lc-card-number|number-local-acquisition|other-control-number|record-control-number', NULL, 'List of searchfields (separated by | or ,) that should not be autotruncated by Elasticsearch even if QueryAutoTruncate is set to Yes', 'free'),
 ('ExcludeHolidaysFromMaxPickUpDelay', '0', NULL, 'If ON, reserves max pickup delay takes into accountthe closed days.', 'YesNo'),
 ('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),

koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref

@@ -95,6 +95,14 @@ Searching:
                   0: Disable
             - "the cross_fields option for Elasticsearch searches, supported in Elasticsearch 6.X and above."
             - See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#type-cross-fields">Elasticsearch cross_fields documentation</a>.
+        -
+            - pref: ESBoostFieldMatch
+              default: 0
+              choices:
+                  1: Enable
+                  0: Disable
+            - "adding a second 'match' field search to the ES query to boost relevancy. Keyword and title fields will be boosted with title-cover, other fields will boost directly."
+            - "This will not boost CCL style searches, only standard or advanced searches"
         -
             - pref: SavedSearchFilters
               default: 0