search_test.go

//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bleve

import (
	"context"
	"encoding/json"
	"fmt"
	"math"
	"math/rand"
	"os"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/blevesearch/bleve/v2/analysis"
	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
	"github.com/blevesearch/bleve/v2/analysis/analyzer/simple"
	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
	html_char_filter "github.com/blevesearch/bleve/v2/analysis/char/html"
	regexp_char_filter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
	"github.com/blevesearch/bleve/v2/analysis/datetime/flexible"
	"github.com/blevesearch/bleve/v2/analysis/datetime/iso"
	"github.com/blevesearch/bleve/v2/analysis/datetime/percent"
	"github.com/blevesearch/bleve/v2/analysis/datetime/sanitized"
	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds"
	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds"
	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds"
	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds"
	"github.com/blevesearch/bleve/v2/analysis/lang/en"
	"github.com/blevesearch/bleve/v2/analysis/token/length"
	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
	"github.com/blevesearch/bleve/v2/analysis/token/shingle"
	"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
	"github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace"
	"github.com/blevesearch/bleve/v2/document"
	"github.com/blevesearch/bleve/v2/geo"
	"github.com/blevesearch/bleve/v2/index/scorch"
	"github.com/blevesearch/bleve/v2/index/upsidedown"
	"github.com/blevesearch/bleve/v2/mapping"
	"github.com/blevesearch/bleve/v2/search"
	"github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi"
	"github.com/blevesearch/bleve/v2/search/highlight/highlighter/html"
	"github.com/blevesearch/bleve/v2/search/query"
	index "github.com/blevesearch/bleve_index_api"
)

func TestSortedFacetedQuery(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	indexMapping := NewIndexMapping()
	indexMapping.TypeField = "type"
	indexMapping.DefaultAnalyzer = "en"
	documentMapping := NewDocumentMapping()
	indexMapping.AddDocumentMapping("hotel", documentMapping)

	contentFieldMapping := NewTextFieldMapping()
	contentFieldMapping.Index = true
	contentFieldMapping.DocValues = true
	documentMapping.AddFieldMappingsAt("content", contentFieldMapping)
	documentMapping.AddFieldMappingsAt("country", contentFieldMapping)

	index, err := New(tmpIndexPath, indexMapping)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err := index.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	if err := index.Index("1", map[string]interface{}{
		"country": "india",
		"content": "k",
	}); err != nil {
		t.Fatal(err)
	}

	if err := index.Index("2", map[string]interface{}{
		"country": "india",
		"content": "l",
	}); err != nil {
		t.Fatal(err)
	}

	if err := index.Index("3", map[string]interface{}{
		"country": "india",
		"content": "k",
	}); err != nil {
		t.Fatal(err)
	}

	d, err := index.DocCount()
	if err != nil {
		t.Fatal(err)
	}

	if d != 3 {
		t.Errorf("expected 3, got %d", d)
	}

	query := NewMatchPhraseQuery("india")
	query.SetField("country")
	searchRequest := NewSearchRequest(query)
	searchRequest.SortBy([]string{"content"})
	fr := NewFacetRequest("content", 100)
	searchRequest.AddFacet("content_facet", fr)

	searchResults, err := index.Search(searchRequest)
	if err != nil {
		t.Fatal(err)
	}

	expectedResults := map[string]int{"k": 2, "l": 1}

	for _, v := range searchResults.Facets {
		for _, v1 := range v.Terms.Terms() {
			if v1.Count != expectedResults[v1.Term] {
				t.Errorf("expected %d, got %d", expectedResults[v1.Term], v1.Count)
			}
		}
	}
}

func TestMatchAllScorer(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	indexMapping := NewIndexMapping()
	indexMapping.TypeField = "type"
	indexMapping.DefaultAnalyzer = "en"
	documentMapping := NewDocumentMapping()

	contentFieldMapping := NewTextFieldMapping()
	contentFieldMapping.Index = true
	contentFieldMapping.Store = true
	documentMapping.AddFieldMappingsAt("content", contentFieldMapping)

	index, err := New(tmpIndexPath, indexMapping)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err := index.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	if err := index.Index("1", map[string]interface{}{
		"country": "india",
		"content": "k",
	}); err != nil {
		t.Fatal(err)
	}

	if err := index.Index("2", map[string]interface{}{
		"country": "india",
		"content": "l",
	}); err != nil {
		t.Fatal(err)
	}

	if err := index.Index("3", map[string]interface{}{
		"country": "india",
		"content": "k",
	}); err != nil {
		t.Fatal(err)
	}

	d, err := index.DocCount()
	if err != nil {
		t.Fatal(err)
	}
	if d != 3 {
		t.Errorf("expected 3, got %d", d)
	}

	searchRequest := NewSearchRequest(NewMatchAllQuery())
	searchRequest.Score = "none"
	searchResults, err := index.Search(searchRequest)
	if err != nil {
		t.Fatal(err)
	}

	if searchResults.Total != 3 {
		t.Fatalf("expected all the 3 docs in the index, got %v", searchResults.Total)
	}

	for _, hit := range searchResults.Hits {
		if hit.Score != 0.0 {
			t.Fatalf("expected 0 score since score = none, got %v", hit.Score)
		}
	}
}

func TestSearchResultString(t *testing.T) {
	tests := []struct {
		result *SearchResult
		str    string
	}{
		{
			result: &SearchResult{
				Request: &SearchRequest{
					Size: 10,
				},
				Total: 5,
				Took:  1 * time.Second,
				Hits: search.DocumentMatchCollection{
					&search.DocumentMatch{},
					&search.DocumentMatch{},
					&search.DocumentMatch{},
					&search.DocumentMatch{},
					&search.DocumentMatch{},
				},
			},
			str: "5 matches, showing 1 through 5, took 1s",
		},
		{
			result: &SearchResult{
				Request: &SearchRequest{
					Size: 0,
				},
				Total: 5,
				Hits:  search.DocumentMatchCollection{},
			},
			str: "5 matches",
		},
		{
			result: &SearchResult{
				Request: &SearchRequest{
					Size: 10,
				},
				Total: 0,
				Hits:  search.DocumentMatchCollection{},
			},
			str: "No matches",
		},
		// no search request
		{
			result: &SearchResult{
				Total: 3,
				Took:  500 * time.Millisecond,
				Hits: search.DocumentMatchCollection{
					&search.DocumentMatch{},
					&search.DocumentMatch{},
					&search.DocumentMatch{},
				},
			},
			str: "3 matches, took 500ms",
		},
	}

	for _, test := range tests {
		srstring := test.result.String()
		if !strings.HasPrefix(srstring, test.str) {
			t.Errorf("expected to start %s, got %s", test.str, srstring)
		}
	}
}

func TestSearchResultMerge(t *testing.T) {
	l := &SearchResult{
		Status: &SearchStatus{
			Total:      1,
			Successful: 1,
			Errors:     make(map[string]error),
		},
		Total:    1,
		MaxScore: 1,
		Hits: search.DocumentMatchCollection{
			&search.DocumentMatch{
				ID:    "a",
				Score: 1,
			},
		},
	}

	r := &SearchResult{
		Status: &SearchStatus{
			Total:      1,
			Successful: 1,
			Errors:     make(map[string]error),
		},
		Total:    1,
		MaxScore: 2,
		Hits: search.DocumentMatchCollection{
			&search.DocumentMatch{
				ID:    "b",
				Score: 2,
			},
		},
	}

	expected := &SearchResult{
		Status: &SearchStatus{
			Total:      2,
			Successful: 2,
			Errors:     make(map[string]error),
		},
		Total:    2,
		MaxScore: 2,
		Hits: search.DocumentMatchCollection{
			&search.DocumentMatch{
				ID:    "a",
				Score: 1,
			},
			&search.DocumentMatch{
				ID:    "b",
				Score: 2,
			},
		},
	}

	l.Merge(r)

	if !reflect.DeepEqual(l, expected) {
		t.Errorf("expected %#v, got %#v", expected, l)
	}
}

func TestUnmarshalingSearchResult(t *testing.T) {
	searchResponse := []byte(`{
    "status":{
      "total":1,
      "failed":1,
      "successful":0,
      "errors":{
        "default_index_362ce020b3d62b13_348f5c3c":"context deadline exceeded"
      }
    },
    "request":{
      "query":{
        "match":"emp",
        "field":"type",
        "boost":1,
        "prefix_length":0,
        "fuzziness":0
      },
    "size":10000000,
    "from":0,
    "highlight":null,
    "fields":[],
    "facets":null,
    "explain":false
  },
  "hits":null,
  "total_hits":0,
  "max_score":0,
  "took":0,
  "facets":null
}`)

	rv := &SearchResult{
		Status: &SearchStatus{
			Errors: make(map[string]error),
		},
	}
	err = json.Unmarshal(searchResponse, rv)
	if err != nil {
		t.Error(err)
	}
	if len(rv.Status.Errors) != 1 {
		t.Errorf("expected 1 error, got %d", len(rv.Status.Errors))
	}
}

func TestFacetNumericDateRangeRequests(t *testing.T) {
	drMissingErr := fmt.Errorf("date range query must specify either start, end or both for range name 'testName'")
	nrMissingErr := fmt.Errorf("numeric range query must specify either min, max or both for range name 'testName'")
	drNrErr := fmt.Errorf("facet can only contain numeric ranges or date ranges, not both")
	drNameDupErr := fmt.Errorf("date ranges contains duplicate name 'testName'")
	nrNameDupErr := fmt.Errorf("numeric ranges contains duplicate name 'testName'")
	value := float64(5)

	tests := []struct {
		facet  *FacetRequest
		result error
	}{
		{
			facet: &FacetRequest{
				Field: "Date_Range_Success_With_StartEnd",
				Size:  1,
				DateTimeRanges: []*dateTimeRange{
					{Name: "testName", Start: time.Unix(0, 0), End: time.Now()},
				},
			},
			result: nil,
		},
		{
			facet: &FacetRequest{
				Field: "Date_Range_Success_With_Start",
				Size:  1,
				DateTimeRanges: []*dateTimeRange{
					{Name: "testName", Start: time.Unix(0, 0)},
				},
			},
			result: nil,
		},
		{
			facet: &FacetRequest{
				Field: "Date_Range_Success_With_End",
				Size:  1,
				DateTimeRanges: []*dateTimeRange{
					{Name: "testName", End: time.Now()},
				},
			},
			result: nil,
		},
		{
			facet: &FacetRequest{
				Field: "Numeric_Range_Success_With_MinMax",
				Size:  1,
				NumericRanges: []*numericRange{
					{Name: "testName", Min: &value, Max: &value},
				},
			},
			result: nil,
		},
		{
			facet: &FacetRequest{
				Field: "Numeric_Range_Success_With_Min",
				Size:  1,
				NumericRanges: []*numericRange{
					{Name: "testName", Min: &value},
				},
			},
			result: nil,
		},
		{
			facet: &FacetRequest{
				Field: "Numeric_Range_Success_With_Max",
				Size:  1,
				NumericRanges: []*numericRange{
					{Name: "testName", Max: &value},
				},
			},
			result: nil,
		},
		{
			facet: &FacetRequest{
				Field: "Date_Range_Missing_Failure",
				Size:  1,
				DateTimeRanges: []*dateTimeRange{
					{Name: "testName2", Start: time.Unix(0, 0)},
					{Name: "testName1", End: time.Now()},
					{Name: "testName"},
				},
			},
			result: drMissingErr,
		},
		{
			facet: &FacetRequest{
				Field: "Numeric_Range_Missing_Failure",
				Size:  1,
				NumericRanges: []*numericRange{
					{Name: "testName2", Min: &value},
					{Name: "testName1", Max: &value},
					{Name: "testName"},
				},
			},
			result: nrMissingErr,
		},
		{
			facet: &FacetRequest{
				Field: "Numeric_And_DateRanges_Failure",
				Size:  1,
				NumericRanges: []*numericRange{
					{Name: "testName", Max: &value},
				},
				DateTimeRanges: []*dateTimeRange{
					{Name: "testName", End: time.Now()},
				},
			},
			result: drNrErr,
		},
		{
			facet: &FacetRequest{
				Field: "Numeric_Range_Name_Repeat_Failure",
				Size:  1,
				NumericRanges: []*numericRange{
					{Name: "testName", Min: &value},
					{Name: "testName", Max: &value},
				},
			},
			result: nrNameDupErr,
		},
		{
			facet: &FacetRequest{
				Field: "Date_Range_Name_Repeat_Failure",
				Size:  1,
				DateTimeRanges: []*dateTimeRange{
					{Name: "testName", Start: time.Unix(0, 0)},
					{Name: "testName", End: time.Now()},
				},
			},
			result: drNameDupErr,
		},
	}

	for _, test := range tests {
		result := test.facet.Validate()
		if !reflect.DeepEqual(result, test.result) {
			t.Errorf("expected %#v, got %#v", test.result, result)
		}
	}
}

func TestSearchResultFacetsMerge(t *testing.T) {
	lowmed := "2010-01-01"
	medhi := "2011-01-01"
	hihigher := "2012-01-01"

	fr := &search.FacetResult{
		Field:   "birthday",
		Total:   100,
		Missing: 25,
		Other:   25,
		DateRanges: []*search.DateRangeFacet{
			{
				Name:  "low",
				End:   &lowmed,
				Count: 25,
			},
			{
				Name:  "med",
				Count: 24,
				Start: &lowmed,
				End:   &medhi,
			},
			{
				Name:  "hi",
				Count: 1,
				Start: &medhi,
				End:   &hihigher,
			},
		},
	}
	frs := search.FacetResults{
		"birthdays": fr,
	}

	l := &SearchResult{
		Status: &SearchStatus{
			Total:      10,
			Successful: 1,
			Errors:     make(map[string]error),
		},
		Total:    10,
		MaxScore: 1,
	}

	r := &SearchResult{
		Status: &SearchStatus{
			Total:      1,
			Successful: 1,
			Errors:     make(map[string]error),
		},
		Total:    1,
		MaxScore: 2,
		Facets:   frs,
	}

	expected := &SearchResult{
		Status: &SearchStatus{
			Total:      11,
			Successful: 2,
			Errors:     make(map[string]error),
		},
		Total:    11,
		MaxScore: 2,
		Facets:   frs,
	}

	l.Merge(r)

	if !reflect.DeepEqual(l, expected) {
		t.Errorf("expected %#v, got %#v", expected, l)
	}
}

func TestMemoryNeededForSearchResult(t *testing.T) {
	query := NewTermQuery("blah")
	req := NewSearchRequest(query)

	var sr SearchResult
	expect := sr.Size()
	var dm search.DocumentMatch
	expect += 10 * dm.Size()

	estimate := MemoryNeededForSearchResult(req)
	if estimate != uint64(expect) {
		t.Errorf("estimate not what is expected: %v != %v", estimate, expect)
	}
}

// https://github.com/blevesearch/bleve/issues/954
func TestNestedBooleanSearchers(t *testing.T) {
	// create an index with a custom analyzer
	idxMapping := NewIndexMapping()
	if err := idxMapping.AddCustomAnalyzer("3xbla", map[string]interface{}{
		"type":          custom.Name,
		"tokenizer":     whitespace.Name,
		"token_filters": []interface{}{lowercase.Name, "stop_en"},
	}); err != nil {
		t.Fatal(err)
	}

	idxMapping.DefaultAnalyzer = "3xbla"

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	// create and insert documents as a batch
	batch := idx.NewBatch()
	matches := 0
	for i := 0; i < 100; i++ {
		hostname := fmt.Sprintf("planner_hostname_%d", i%5)
		metadata := map[string]string{"region": fmt.Sprintf("planner_us-east-%d", i%5)}

		// Expected matches
		if (hostname == "planner_hostname_1" || hostname == "planner_hostname_2") &&
			metadata["region"] == "planner_us-east-1" {
			matches++
		}

		doc := document.NewDocument(strconv.Itoa(i))
		doc.Fields = []document.Field{
			document.NewTextFieldCustom("hostname", []uint64{}, []byte(hostname),
				index.IndexField,
				&analysis.DefaultAnalyzer{
					Tokenizer: single.NewSingleTokenTokenizer(),
					TokenFilters: []analysis.TokenFilter{
						lowercase.NewLowerCaseFilter(),
					},
				},
			),
		}
		for k, v := range metadata {
			doc.AddField(document.NewTextFieldWithIndexingOptions(
				fmt.Sprintf("metadata.%s", k), []uint64{}, []byte(v), index.IndexField))
		}
		doc.CompositeFields = []*document.CompositeField{
			document.NewCompositeFieldWithIndexingOptions(
				"_all", true, []string{"text"}, []string{},
				index.IndexField|index.IncludeTermVectors),
		}

		if err = batch.IndexAdvanced(doc); err != nil {
			t.Fatal(err)
		}
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	que, err := query.ParseQuery([]byte(
		`{
			"conjuncts": [
			{
				"must": {
					"conjuncts": [
					{
						"disjuncts": [
						{
							"match": "planner_hostname_1",
							"field": "hostname"
						},
						{
							"match": "planner_hostname_2",
							"field": "hostname"
						}
						]
					}
					]
				}
			},
			{
				"must": {
					"conjuncts": [
					{
						"match": "planner_us-east-1",
						"field": "metadata.region"
					}
					]
				}
			}
			]
		}`,
	))
	if err != nil {
		t.Fatal(err)
	}

	req := NewSearchRequest(que)
	req.Size = 100
	req.Fields = []string{"hostname", "metadata.region"}
	searchResults, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}
	if matches != len(searchResults.Hits) {
		t.Fatalf("Unexpected result set, %v != %v", matches, len(searchResults.Hits))
	}
}

func TestNestedBooleanMustNotSearcherUpsidedown(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	// create an index with default settings
	idxMapping := NewIndexMapping()
	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	// create and insert documents as a batch
	batch := idx.NewBatch()

	docs := []struct {
		id              string
		hasRole         bool
		investigationId string
	}{
		{
			id:              "1@1",
			hasRole:         true,
			investigationId: "1",
		},
		{
			id:              "1@2",
			hasRole:         false,
			investigationId: "2",
		},
		{
			id:              "2@1",
			hasRole:         true,
			investigationId: "1",
		},
		{
			id:              "2@2",
			hasRole:         false,
			investigationId: "2",
		},
		{
			id:              "3@1",
			hasRole:         true,
			investigationId: "1",
		},
		{
			id:              "3@2",
			hasRole:         false,
			investigationId: "2",
		},
		{
			id:              "4@1",
			hasRole:         true,
			investigationId: "1",
		},
		{
			id:              "5@1",
			hasRole:         true,
			investigationId: "1",
		},
		{
			id:              "6@1",
			hasRole:         true,
			investigationId: "1",
		},
		{
			id:              "7@1",
			hasRole:         true,
			investigationId: "1",
		},
	}

	for i := 0; i < len(docs); i++ {
		doc := document.NewDocument(docs[i].id)
		doc.Fields = []document.Field{
			document.NewTextField("id", []uint64{}, []byte(docs[i].id)),
			document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole),
			document.NewTextField("investigationId", []uint64{}, []byte(docs[i].investigationId)),
		}

		doc.CompositeFields = []*document.CompositeField{
			document.NewCompositeFieldWithIndexingOptions(
				"_all", true, []string{"text"}, []string{},
				index.IndexField|index.IncludeTermVectors),
		}

		if err = batch.IndexAdvanced(doc); err != nil {
			t.Fatal(err)
		}
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	tq := NewTermQuery("1")
	tq.SetField("investigationId")
	// using must not, for cases that the field did not exists at all
	hasRole := NewBoolFieldQuery(true)
	hasRole.SetField("hasRole")
	noRole := NewBooleanQuery()
	noRole.AddMustNot(hasRole)
	oneRolesOrNoRoles := NewBooleanQuery()
	oneRolesOrNoRoles.AddShould(noRole)
	oneRolesOrNoRoles.SetMinShould(1)
	q := NewConjunctionQuery(tq, oneRolesOrNoRoles)

	sr := NewSearchRequestOptions(q, 100, 0, false)
	sr.Fields = []string{"hasRole"}
	sr.Highlight = NewHighlight()

	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 0 {
		t.Fatalf("Unexpected result, %v != 0", res.Total)
	}
}

func TestSearchScorchOverEmptyKeyword(t *testing.T) {
	defaultIndexType := Config.DefaultIndexType
	Config.DefaultIndexType = scorch.Name

	dmap := mapping.NewDocumentMapping()
	dmap.DefaultAnalyzer = standard.Name

	fm := mapping.NewTextFieldMapping()
	fm.Analyzer = keyword.Name

	fm1 := mapping.NewTextFieldMapping()
	fm1.Analyzer = standard.Name

	dmap.AddFieldMappingsAt("id", fm)
	dmap.AddFieldMappingsAt("name", fm1)

	imap := mapping.NewIndexMapping()
	imap.DefaultMapping = dmap
	imap.DefaultAnalyzer = standard.Name

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}

		Config.DefaultIndexType = defaultIndexType
	}()

	for i := 0; i < 10; i++ {
		err = idx.Index(fmt.Sprint(i), map[string]string{"name": fmt.Sprintf("test%d", i), "id": ""})
		if err != nil {
			t.Fatal(err)
		}
	}

	count, err := idx.DocCount()
	if err != nil {
		t.Fatal(err)
	}
	if count != 10 {
		t.Fatalf("Unexpected doc count: %v, expected 10", count)
	}

	q := query.NewWildcardQuery("test*")
	sr := NewSearchRequestOptions(q, 40, 0, false)
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 10 {
		t.Fatalf("Unexpected search hits: %v, expected 10", res.Total)
	}
}

func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) {
	defaultIndexType := Config.DefaultIndexType
	Config.DefaultIndexType = scorch.Name

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	// create an index with default settings
	idxMapping := NewIndexMapping()
	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}

		Config.DefaultIndexType = defaultIndexType
	}()

	// create and insert documents as a batch
	batch := idx.NewBatch()

	doc := document.NewDocument("1-child-0")
	doc.Fields = []document.Field{
		document.NewTextField("id", []uint64{}, []byte("1-child-0")),
		document.NewBooleanField("hasRole", []uint64{}, false),
		document.NewTextField("roles", []uint64{}, []byte("R1")),
		document.NewNumericField("type", []uint64{}, 0),
	}
	doc.CompositeFields = []*document.CompositeField{
		document.NewCompositeFieldWithIndexingOptions(
			"_all", true, []string{"text"}, []string{},
			index.IndexField|index.IncludeTermVectors),
	}

	if err = batch.IndexAdvanced(doc); err != nil {
		t.Fatal(err)
	}

	docs := []struct {
		id      string
		hasRole bool
		typ     int
	}{
		{
			id:      "16d6fa37-48fd-4dea-8b3d-a52bddf73951",
			hasRole: false,
			typ:     9,
		},
		{
			id:      "18fa9eb2-8b1f-46f0-8b56-b4c551213f78",
			hasRole: false,
			typ:     9,
		},
		{
			id:      "3085855b-d74b-474a-86c3-9bf3e4504382",
			hasRole: false,
			typ:     9,
		},
		{
			id:      "38ef5d28-0f85-4fb0-8a94-dd20751c3364",
			hasRole: false,
			typ:     9,
		},
	}

	for i := 0; i < len(docs); i++ {
		doc := document.NewDocument(docs[i].id)
		doc.Fields = []document.Field{
			document.NewTextField("id", []uint64{}, []byte(docs[i].id)),
			document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole),
			document.NewNumericField("type", []uint64{}, float64(docs[i].typ)),
		}

		doc.CompositeFields = []*document.CompositeField{
			document.NewCompositeFieldWithIndexingOptions(
				"_all", true, []string{"text"}, []string{},
				index.IndexField|index.IncludeTermVectors),
		}

		if err = batch.IndexAdvanced(doc); err != nil {
			t.Fatal(err)
		}
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	batch = idx.NewBatch()

	// Update 1st doc
	doc = document.NewDocument("1-child-0")
	doc.Fields = []document.Field{
		document.NewTextField("id", []uint64{}, []byte("1-child-0")),
		document.NewBooleanField("hasRole", []uint64{}, false),
		document.NewNumericField("type", []uint64{}, 0),
	}
	doc.CompositeFields = []*document.CompositeField{
		document.NewCompositeFieldWithIndexingOptions(
			"_all", true, []string{"text"}, []string{},
			index.IndexField|index.IncludeTermVectors),
	}

	if err = batch.IndexAdvanced(doc); err != nil {
		t.Fatal(err)
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	inclusive := true
	val := float64(9)
	q := query.NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)
	q.SetField("type")
	initialQuery := query.NewBooleanQuery(nil, nil, []query.Query{q})

	// using must not, for cases that the field did not exists at all
	hasRole := NewBoolFieldQuery(true)
	hasRole.SetField("hasRole")
	noRole := NewBooleanQuery()
	noRole.AddMustNot(hasRole)

	rq := query.NewBooleanQuery([]query.Query{initialQuery, noRole}, nil, nil)

	sr := NewSearchRequestOptions(rq, 100, 0, false)
	sr.Fields = []string{"id", "hasRole", "type"}
	sr.Highlight = NewHighlight()

	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 1 {
		t.Fatalf("Unexpected result, %v != 1", res.Total)
	}
}

func testBooleanMustNotSearcher(t *testing.T, indexName string) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	im := NewIndexMapping()
	idx, err := NewUsing(tmpIndexPath, im, indexName, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	docs := []struct {
		Name    string
		HasRole bool
	}{
		{
			Name: "13900",
		},
		{
			Name: "13901",
		},
		{
			Name: "13965",
		},
		{
			Name:    "13966",
			HasRole: true,
		},
		{
			Name:    "13967",
			HasRole: true,
		},
	}

	for _, doc := range docs {
		err := idx.Index(doc.Name, doc)
		if err != nil {
			t.Fatal(err)
		}
	}

	lhs := NewDocIDQuery([]string{"13965", "13966", "13967"})
	hasRole := NewBoolFieldQuery(true)
	hasRole.SetField("HasRole")
	rhs := NewBooleanQuery()
	rhs.AddMustNot(hasRole)

	compareLeftRightAndConjunction := func(idx Index, left, right query.Query) error {
		// left
		lr := NewSearchRequestOptions(left, 100, 0, false)
		lres, err := idx.Search(lr)
		if err != nil {
			return fmt.Errorf("error left: %v", err)
		}
		lresIds := map[string]struct{}{}
		for i := range lres.Hits {
			lresIds[lres.Hits[i].ID] = struct{}{}
		}
		// right
		rr := NewSearchRequestOptions(right, 100, 0, false)
		rres, err := idx.Search(rr)
		if err != nil {
			return fmt.Errorf("error right: %v", err)
		}
		rresIds := map[string]struct{}{}
		for i := range rres.Hits {
			rresIds[rres.Hits[i].ID] = struct{}{}
		}
		// conjunction
		cr := NewSearchRequestOptions(NewConjunctionQuery(left, right), 100, 0, false)
		cres, err := idx.Search(cr)
		if err != nil {
			return fmt.Errorf("error conjunction: %v", err)
		}
		for i := range cres.Hits {
			if _, ok := lresIds[cres.Hits[i].ID]; ok {
				if _, ok := rresIds[cres.Hits[i].ID]; !ok {
					return fmt.Errorf("error id %s missing from right", cres.Hits[i].ID)
				}
			} else {
				return fmt.Errorf("error id %s missing from left", cres.Hits[i].ID)
			}
		}
		return nil
	}

	err = compareLeftRightAndConjunction(idx, lhs, rhs)
	if err != nil {
		t.Fatal(err)
	}
}

func TestBooleanMustNotSearcherUpsidedown(t *testing.T) {
	testBooleanMustNotSearcher(t, upsidedown.Name)
}

func TestBooleanMustNotSearcherScorch(t *testing.T) {
	testBooleanMustNotSearcher(t, scorch.Name)
}

func TestQueryStringEmptyConjunctionSearcher(t *testing.T) {
	mapping := NewIndexMapping()
	mapping.DefaultAnalyzer = keyword.Name
	index, err := NewMemOnly(mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = index.Close()
	}()

	query := NewQueryStringQuery("foo:bar +baz:\"\"")
	searchReq := NewSearchRequest(query)

	_, _ = index.Search(searchReq)
}

func TestDisjunctionQueryIncorrectMin(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	// create an index with default settings
	idxMapping := NewIndexMapping()
	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	// create and insert documents as a batch
	batch := idx.NewBatch()
	docs := []struct {
		field1 string
		field2 int
	}{
		{
			field1: "one",
			field2: 1,
		},
		{
			field1: "two",
			field2: 2,
		},
	}

	for i := 0; i < len(docs); i++ {
		doc := document.NewDocument(strconv.Itoa(docs[i].field2))
		doc.Fields = []document.Field{
			document.NewTextField("field1", []uint64{}, []byte(docs[i].field1)),
			document.NewNumericField("field2", []uint64{}, float64(docs[i].field2)),
		}
		doc.CompositeFields = []*document.CompositeField{
			document.NewCompositeFieldWithIndexingOptions(
				"_all", true, []string{"text"}, []string{},
				index.IndexField|index.IncludeTermVectors),
		}
		if err = batch.IndexAdvanced(doc); err != nil {
			t.Fatal(err)
		}
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	tq := NewTermQuery("one")
	dq := NewDisjunctionQuery(tq)
	dq.SetMin(2)
	sr := NewSearchRequestOptions(dq, 1, 0, false)
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total > 0 {
		t.Fatalf("Expected 0 matches as disjunction query contains a single clause"+
			" but got: %v", res.Total)
	}
}

func TestMatchQueryPartialMatch(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)
	idx, err := New(tmpIndexPath, NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()
	doc1 := map[string]interface{}{
		"description": "Patrick is first name Stewart is last name",
	}
	doc2 := map[string]interface{}{
		"description": "Manager given name is Patrick",
	}
	batch := idx.NewBatch()
	if err = batch.Index("doc1", doc1); err != nil {
		t.Fatal(err)
	}
	if err = batch.Index("doc2", doc2); err != nil {
		t.Fatal(err)
	}
	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}
	// Test 1 - Both Docs hit, doc 1 = Full Match and doc 2 = Partial Match
	mq1 := NewMatchQuery("patrick stewart")
	mq1.SetField("description")

	sr := NewSearchRequest(mq1)
	sr.Explain = true
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 2 {
		t.Errorf("Expected 2 results, but got: %v", res.Total)
	}
	for _, hit := range res.Hits {
		switch hit.ID {
		case "doc1":
			if hit.Expl.PartialMatch {
				t.Errorf("Expected doc1 to be a full match")
			}
		case "doc2":
			if !hit.Expl.PartialMatch {
				t.Errorf("Expected doc2 to be a partial match")
			}
		default:
			t.Errorf("Unexpected document ID: %s", hit.ID)
		}
	}

	// Test 2 - Both Docs hit, doc 1 = Partial Match and doc 2 = Full Match
	mq2 := NewMatchQuery("paltric manner")
	mq2.SetField("description")
	mq2.SetFuzziness(2)

	sr = NewSearchRequest(mq2)
	sr.Explain = true
	res, err = idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 2 {
		t.Errorf("Expected 2 results, but got: %v", res.Total)
	}
	for _, hit := range res.Hits {
		switch hit.ID {
		case "doc1":
			if !hit.Expl.PartialMatch {
				t.Errorf("Expected doc1 to be a partial match")
			}
		case "doc2":
			if hit.Expl.PartialMatch {
				t.Errorf("Expected doc2 to be a full match")
			}
		default:
			t.Errorf("Unexpected document ID: %s", hit.ID)
		}
	}
	// Test 3 - Two Docs hits, both full match
	mq3 := NewMatchQuery("patrick")
	mq3.SetField("description")

	sr = NewSearchRequest(mq3)
	sr.Explain = true
	res, err = idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 2 {
		t.Errorf("Expected 2 results, but got: %v", res.Total)
	}
	for _, hit := range res.Hits {
		switch hit.ID {
		case "doc1":
			if hit.Expl.PartialMatch {
				t.Errorf("Expected doc1 to be a full match")
			}
		case "doc2":
			if hit.Expl.PartialMatch {
				t.Errorf("Expected doc2 to be a full match")
			}
		default:
			t.Errorf("Unexpected document ID: %s", hit.ID)
		}
	}
	// Test 4 - Two Docs hits, both partial match
	mq4 := NewMatchQuery("patrick stewart manager")
	mq4.SetField("description")

	sr = NewSearchRequest(mq4)
	sr.Explain = true
	res, err = idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 2 {
		t.Errorf("Expected 2 results, but got: %v", res.Total)
	}
	for _, hit := range res.Hits {
		switch hit.ID {
		case "doc1":
			if !hit.Expl.PartialMatch {
				t.Errorf("Expected doc1 to be a full match")
			}
		case "doc2":
			if !hit.Expl.PartialMatch {
				t.Errorf("Expected doc2 to be a full match")
			}
		default:
			t.Errorf("Unexpected document ID: %s", hit.ID)
		}
	}

	// Test 5 - Match Query AND operator always results in full match
	mq5 := NewMatchQuery("patrick stewart")
	mq5.SetField("description")
	mq5.SetOperator(1)

	sr = NewSearchRequest(mq5)
	sr.Explain = true
	res, err = idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}
	if res.Total != 1 {
		t.Errorf("Expected 1 result, but got: %v", res.Total)
	}
	hit := res.Hits[0]
	if hit.ID != "doc1" || hit.Expl.PartialMatch {
		t.Errorf("Expected doc1 to be a full match")
	}
}

func TestBooleanShouldMinPropagation(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc1 := map[string]interface{}{
		"dept": "queen",
		"name": "cersei lannister",
	}

	doc2 := map[string]interface{}{
		"dept": "kings guard",
		"name": "jaime lannister",
	}

	batch := idx.NewBatch()

	if err = batch.Index("doc1", doc1); err != nil {
		t.Fatal(err)
	}

	if err = batch.Index("doc2", doc2); err != nil {
		t.Fatal(err)
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	// term dictionaries in the index for field..
	//  dept: queen kings guard
	//  name: cersei jaime lannister

	// the following match query would match doc2
	mq1 := NewMatchQuery("kings guard")
	mq1.SetField("dept")

	// the following match query would match both doc1 and doc2,
	// as both docs share common lastname
	mq2 := NewMatchQuery("jaime lannister")
	mq2.SetField("name")

	bq := NewBooleanQuery()
	bq.AddShould(mq1)
	bq.AddMust(mq2)

	sr := NewSearchRequest(bq)
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 2 {
		t.Errorf("Expected 2 results, but got: %v", res.Total)
	}
}

func TestDisjunctionMinPropagation(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc1 := map[string]interface{}{
		"dept": "finance",
		"name": "xyz",
	}

	doc2 := map[string]interface{}{
		"dept": "marketing",
		"name": "xyz",
	}

	doc3 := map[string]interface{}{
		"dept": "engineering",
		"name": "abc",
	}

	batch := idx.NewBatch()

	if err = batch.Index("doc1", doc1); err != nil {
		t.Fatal(err)
	}

	if err = batch.Index("doc2", doc2); err != nil {
		t.Fatal(err)
	}

	if err = batch.Index("doc3", doc3); err != nil {
		t.Fatal(err)
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	mq1 := NewMatchQuery("finance")
	mq2 := NewMatchQuery("marketing")
	dq := NewDisjunctionQuery(mq1, mq2)
	dq.SetMin(3)

	dq2 := NewDisjunctionQuery(dq)
	dq2.SetMin(1)

	sr := NewSearchRequest(dq2)
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 0 {
		t.Fatalf("Expect 0 results, but got: %v", res.Total)
	}
}

func TestDuplicateLocationsIssue1168(t *testing.T) {
	fm1 := NewTextFieldMapping()
	fm1.Analyzer = keyword.Name
	fm1.Name = "name1"

	dm := NewDocumentStaticMapping()
	dm.AddFieldMappingsAt("name", fm1)

	m := NewIndexMapping()
	m.DefaultMapping = dm

	idx, err := NewMemOnly(m)
	if err != nil {
		t.Fatalf("bleve new err: %v", err)
	}

	err = idx.Index("x", map[string]interface{}{
		"name": "marty",
	})
	if err != nil {
		t.Fatalf("bleve index err: %v", err)
	}

	q1 := NewTermQuery("marty")
	q2 := NewTermQuery("marty")
	dq := NewDisjunctionQuery(q1, q2)

	sreq := NewSearchRequest(dq)
	sreq.Fields = []string{"*"}
	sreq.Highlight = NewHighlightWithStyle(html.Name)

	sres, err := idx.Search(sreq)
	if err != nil {
		t.Fatalf("bleve search err: %v", err)
	}
	if len(sres.Hits[0].Locations["name1"]["marty"]) != 1 {
		t.Fatalf("duplicate marty")
	}
}

func TestBooleanMustSingleMatchNone(t *testing.T) {
	idxMapping := NewIndexMapping()
	if err := idxMapping.AddCustomTokenFilter(length.Name, map[string]interface{}{
		"min":  3.0,
		"max":  5.0,
		"type": length.Name,
	}); err != nil {
		t.Fatal(err)
	}
	if err := idxMapping.AddCustomAnalyzer("custom1", map[string]interface{}{
		"type":          "custom",
		"tokenizer":     "single",
		"token_filters": []interface{}{length.Name},
	}); err != nil {
		t.Fatal(err)
	}

	idxMapping.DefaultAnalyzer = "custom1"

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc := map[string]interface{}{
		"languages_known": "Dutch",
		"dept":            "Sales",
	}

	batch := idx.NewBatch()
	if err = batch.Index("doc", doc); err != nil {
		t.Fatal(err)
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	// this is a successful match
	matchSales := NewMatchQuery("Sales")
	matchSales.SetField("dept")

	// this would spin off a MatchNoneSearcher as the
	// token filter rules out the word "French"
	matchFrench := NewMatchQuery("French")
	matchFrench.SetField("languages_known")

	bq := NewBooleanQuery()
	bq.AddShould(matchSales)
	bq.AddMust(matchFrench)

	sr := NewSearchRequest(bq)
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 0 {
		t.Fatalf("Expected 0 results but got: %v", res.Total)
	}
}

func TestBooleanMustNotSingleMatchNone(t *testing.T) {
	idxMapping := NewIndexMapping()
	if err := idxMapping.AddCustomTokenFilter(shingle.Name, map[string]interface{}{
		"min":  3.0,
		"max":  5.0,
		"type": shingle.Name,
	}); err != nil {
		t.Fatal(err)
	}
	if err := idxMapping.AddCustomAnalyzer("custom1", map[string]interface{}{
		"type":          "custom",
		"tokenizer":     "unicode",
		"token_filters": []interface{}{shingle.Name},
	}); err != nil {
		t.Fatal(err)
	}

	idxMapping.DefaultAnalyzer = "custom1"

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc := map[string]interface{}{
		"languages_known": "Dutch",
		"dept":            "Sales",
	}

	batch := idx.NewBatch()
	if err = batch.Index("doc", doc); err != nil {
		t.Fatal(err)
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	// this is a successful match
	matchSales := NewMatchQuery("Sales")
	matchSales.SetField("dept")

	// this would spin off a MatchNoneSearcher as the
	// token filter rules out the word "Dutch"
	matchDutch := NewMatchQuery("Dutch")
	matchDutch.SetField("languages_known")

	matchEngineering := NewMatchQuery("Engineering")
	matchEngineering.SetField("dept")

	bq := NewBooleanQuery()
	bq.AddShould(matchSales)
	bq.AddMustNot(matchDutch, matchEngineering)

	sr := NewSearchRequest(bq)
	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 0 {
		t.Fatalf("Expected 0 results but got: %v", res.Total)
	}
}

func TestBooleanSearchBug1185(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	of := NewTextFieldMapping()
	of.Analyzer = keyword.Name
	of.Name = "owner"

	dm := NewDocumentMapping()
	dm.AddFieldMappingsAt("owner", of)

	m := NewIndexMapping()
	m.DefaultMapping = dm

	idx, err := NewUsing(tmpIndexPath, m, "scorch", "scorch", nil)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = idx.Index("17112", map[string]interface{}{
		"owner": "marty",
		"type":  "A Demo Type",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("17139", map[string]interface{}{
		"type": "A Demo Type",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("177777", map[string]interface{}{
		"type": "x",
	})
	if err != nil {
		t.Fatal(err)
	}
	err = idx.Index("177778", map[string]interface{}{
		"type": "A Demo Type",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("17140", map[string]interface{}{
		"type": "A Demo Type",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("17000", map[string]interface{}{
		"owner": "marty",
		"type":  "x",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("17141", map[string]interface{}{
		"type": "A Demo Type",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("17428", map[string]interface{}{
		"owner": "marty",
		"type":  "A Demo Type",
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idx.Index("17113", map[string]interface{}{
		"owner": "marty",
		"type":  "x",
	})
	if err != nil {
		t.Fatal(err)
	}

	matchTypeQ := NewMatchPhraseQuery("A Demo Type")
	matchTypeQ.SetField("type")

	matchAnyOwnerRegQ := NewRegexpQuery(".+")
	matchAnyOwnerRegQ.SetField("owner")

	matchNoOwner := NewBooleanQuery()
	matchNoOwner.AddMustNot(matchAnyOwnerRegQ)

	notNoOwner := NewBooleanQuery()
	notNoOwner.AddMustNot(matchNoOwner)

	matchTypeAndNoOwner := NewConjunctionQuery()
	matchTypeAndNoOwner.AddQuery(matchTypeQ)
	matchTypeAndNoOwner.AddQuery(notNoOwner)

	req := NewSearchRequest(matchTypeAndNoOwner)
	res, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	// query 2
	matchTypeAndNoOwnerBoolean := NewBooleanQuery()
	matchTypeAndNoOwnerBoolean.AddMust(matchTypeQ)
	matchTypeAndNoOwnerBoolean.AddMustNot(matchNoOwner)

	req2 := NewSearchRequest(matchTypeAndNoOwnerBoolean)
	res2, err := idx.Search(req2)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) != len(res2.Hits) {
		t.Fatalf("expected same number of hits, got: %d and %d", len(res.Hits), len(res2.Hits))
	}
}

func TestSearchScoreNone(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := NewUsing(tmpIndexPath, NewIndexMapping(), scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc := map[string]interface{}{
		"field1": "asd fgh jkl",
		"field2": "more content blah blah",
		"id":     "doc",
	}

	if err = idx.Index("doc", doc); err != nil {
		t.Fatal(err)
	}

	q := NewQueryStringQuery("content")
	sr := NewSearchRequest(q)
	sr.IncludeLocations = true
	sr.Score = "none"

	res, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) != 1 {
		t.Fatal("unexpected number of hits")
	}

	if len(res.Hits[0].Locations) != 1 {
		t.Fatal("unexpected locations for the hit")
	}

	if res.Hits[0].Score != 0 {
		t.Fatal("unexpected score for the hit")
	}
}

func TestGeoDistanceIssue1301(t *testing.T) {
	shopMapping := NewDocumentMapping()
	shopMapping.AddFieldMappingsAt("GEO", NewGeoPointFieldMapping())
	shopIndexMapping := NewIndexMapping()
	shopIndexMapping.DefaultMapping = shopMapping

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := NewUsing(tmpIndexPath, shopIndexMapping, scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	for i, g := range []string{"wecpkbeddsmf", "wecpk8tne453", "wecpkb80s09t"} {
		if err = idx.Index(strconv.Itoa(i), map[string]interface{}{
			"ID":  i,
			"GEO": g,
		}); err != nil {
			t.Fatal(err)
		}
	}

	// Not setting "Field" for the following query, targets it against the _all
	// field and this is returning inconsistent results, when there's another
	// field indexed along with the geopoint which is numeric.
	// As reported in: https://github.com/blevesearch/bleve/issues/1301
	lat, lon := 22.371154, 114.112603
	q := NewGeoDistanceQuery(lon, lat, "1km")

	req := NewSearchRequest(q)
	sr, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	if sr.Total != 3 {
		t.Fatalf("Size expected: 3, actual %d\n", sr.Total)
	}
}

func TestSearchHighlightingWithRegexpReplacement(t *testing.T) {
	idxMapping := NewIndexMapping()
	if err := idxMapping.AddCustomCharFilter(regexp_char_filter.Name, map[string]interface{}{
		"regexp":  `([a-z])\s+(\d)`,
		"replace": "ooooo$1-$2",
		"type":    regexp_char_filter.Name,
	}); err != nil {
		t.Fatal(err)
	}
	if err := idxMapping.AddCustomAnalyzer("regexp_replace", map[string]interface{}{
		"type":      custom.Name,
		"tokenizer": "unicode",
		"char_filters": []string{
			regexp_char_filter.Name,
		},
	}); err != nil {
		t.Fatal(err)
	}

	idxMapping.DefaultAnalyzer = "regexp_replace"
	idxMapping.StoreDynamic = true

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := NewUsing(tmpIndexPath, idxMapping, scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc := map[string]interface{}{
		"status": "fool 10",
	}

	batch := idx.NewBatch()
	if err = batch.Index("doc", doc); err != nil {
		t.Fatal(err)
	}

	if err = idx.Batch(batch); err != nil {
		t.Fatal(err)
	}

	query := NewMatchQuery("fool 10")
	sreq := NewSearchRequest(query)
	sreq.Fields = []string{"*"}
	sreq.Highlight = NewHighlightWithStyle(ansi.Name)

	sres, err := idx.Search(sreq)
	if err != nil {
		t.Fatal(err)
	}

	if sres.Total != 1 {
		t.Fatalf("Expected 1 hit, got: %v", sres.Total)
	}
}

func TestAnalyzerInheritance(t *testing.T) {
	tests := []struct {
		name       string
		mappingStr string
		doc        map[string]interface{}
		queryField string
		queryTerm  string
	}{
		{
			/*
				index_mapping: keyword
				default_mapping: ""
					-> child field (should inherit keyword)
			*/
			name: "Child field to inherit index mapping's default analyzer",
			mappingStr: `{"default_mapping":{"enabled":true,"dynamic":false,"properties":` +
				`{"city":{"enabled":true,"dynamic":false,"fields":[{"name":"city","type":"text",` +
				`"store":false,"index":true}]}}},"default_analyzer":"keyword"}`,
			doc:        map[string]interface{}{"city": "San Francisco"},
			queryField: "city",
			queryTerm:  "San Francisco",
		},
		{
			/*
				index_mapping: standard
				default_mapping: keyword
				    -> child field (should inherit keyword)
			*/
			name: "Child field to inherit default mapping's default analyzer",
			mappingStr: `{"default_mapping":{"enabled":true,"dynamic":false,"properties":` +
				`{"city":{"enabled":true,"dynamic":false,"fields":[{"name":"city","type":"text",` +
				`"index":true}]}},"default_analyzer":"keyword"},"default_analyzer":"standard"}`,
			doc:        map[string]interface{}{"city": "San Francisco"},
			queryField: "city",
			queryTerm:  "San Francisco",
		},
		{
			/*
				index_mapping: standard
				default_mapping: keyword (dynamic)
				    -> search over field to (should inherit keyword)
			*/
			name: "Child field to inherit default mapping's default analyzer",
			mappingStr: `{"default_mapping":{"enabled":true,"dynamic":true,"default_analyzer":"keyword"}` +
				`,"default_analyzer":"standard"}`,
			doc:        map[string]interface{}{"city": "San Francisco"},
			queryField: "city",
			queryTerm:  "San Francisco",
		},
		{
			/*
				index_mapping: standard
				default_mapping: keyword
				    -> child mapping: ""
					    -> child field: (should inherit keyword)
			*/
			name: "Nested child field to inherit default mapping's default analyzer",
			mappingStr: `{"default_mapping":{"enabled":true,"dynamic":false,"default_analyzer":` +
				`"keyword","properties":{"address":{"enabled":true,"dynamic":false,"properties":` +
				`{"city":{"enabled":true,"dynamic":false,"fields":[{"name":"city","type":"text",` +
				`"index":true}]}}}}},"default_analyzer":"standard"}`,
			doc: map[string]interface{}{
				"address": map[string]interface{}{"city": "San Francisco"},
			},
			queryField: "address.city",
			queryTerm:  "San Francisco",
		},
		{
			/*
				index_mapping: standard
				default_mapping: ""
				    -> child mapping: "keyword"
					    -> child mapping: ""
						    -> child field: (should inherit keyword)
			*/
			name: "Nested child field to inherit first child mapping's default analyzer",
			mappingStr: `{"default_mapping":{"enabled":true,"dynamic":false,"properties":` +
				`{"address":{"enabled":true,"dynamic":false,"default_analyzer":"keyword",` +
				`"properties":{"state":{"enabled":true,"dynamic":false,"properties":{"city":` +
				`{"enabled":true,"dynamic":false,"fields":[{"name":"city","type":"text",` +
				`"store":false,"index":true}]}}}}}}},"default_analyer":"standard"}`,
			doc: map[string]interface{}{
				"address": map[string]interface{}{
					"state": map[string]interface{}{"city": "San Francisco"},
				},
			},
			queryField: "address.state.city",
			queryTerm:  "San Francisco",
		},
	}

	for i := range tests {
		t.Run(tests[i].name, func(t *testing.T) {
			idxMapping := NewIndexMapping()
			if err := idxMapping.UnmarshalJSON([]byte(tests[i].mappingStr)); err != nil {
				t.Fatal(err)
			}

			tmpIndexPath := createTmpIndexPath(t)
			idx, err := New(tmpIndexPath, idxMapping)
			if err != nil {
				t.Fatal(err)
			}

			defer func() {
				if err := idx.Close(); err != nil {
					t.Fatal(err)
				}
			}()

			if err = idx.Index("doc", tests[i].doc); err != nil {
				t.Fatal(err)
			}

			q := NewTermQuery(tests[i].queryTerm)
			q.SetField(tests[i].queryField)

			res, err := idx.Search(NewSearchRequest(q))
			if err != nil {
				t.Fatal(err)
			}

			if len(res.Hits) != 1 {
				t.Errorf("Unexpected number of hits: %v", len(res.Hits))
			}
		})
	}
}

func TestHightlightingWithHTMLCharacterFilter(t *testing.T) {
	idxMapping := NewIndexMapping()
	if err := idxMapping.AddCustomAnalyzer("custom-html", map[string]interface{}{
		"type":         custom.Name,
		"tokenizer":    "unicode",
		"char_filters": []interface{}{html_char_filter.Name},
	}); err != nil {
		t.Fatal(err)
	}

	fm := mapping.NewTextFieldMapping()
	fm.Analyzer = "custom-html"

	dmap := mapping.NewDocumentMapping()
	dmap.AddFieldMappingsAt("content", fm)

	idxMapping.DefaultMapping = dmap

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	content := "<div> Welcome to blevesearch. </div>"
	if err = idx.Index("doc", map[string]string{
		"content": content,
	}); err != nil {
		t.Fatal(err)
	}

	searchStr := "blevesearch"
	q := query.NewMatchQuery(searchStr)
	q.SetField("content")
	sr := NewSearchRequest(q)
	sr.IncludeLocations = true
	sr.Fields = []string{"*"}
	sr.Highlight = NewHighlightWithStyle(html.Name)
	searchResults, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if len(searchResults.Hits) != 1 ||
		len(searchResults.Hits[0].Locations["content"][searchStr]) != 1 {
		t.Fatalf("Expected 1 hit with 1 location")
	}

	expectedLocation := &search.Location{
		Pos:   3,
		Start: uint64(strings.Index(content, searchStr)),
		End:   uint64(strings.Index(content, searchStr) + len(searchStr)),
	}
	expectedFragment := "&lt;div&gt; Welcome to <mark>blevesearch</mark>. &lt;/div&gt;"

	gotLocation := searchResults.Hits[0].Locations["content"]["blevesearch"][0]
	gotFragment := searchResults.Hits[0].Fragments["content"][0]

	if !reflect.DeepEqual(expectedLocation, gotLocation) {
		t.Fatalf("Mismatch in locations, got: %v, expected: %v",
			gotLocation, expectedLocation)
	}

	if expectedFragment != gotFragment {
		t.Fatalf("Mismatch in fragment, got: %v, expected: %v",
			gotFragment, expectedFragment)
	}
}

func TestIPRangeQuery(t *testing.T) {
	idxMapping := NewIndexMapping()
	im := NewIPFieldMapping()
	dmap := mapping.NewDocumentMapping()
	dmap.AddFieldMappingsAt("ip_content", im)
	idxMapping.DefaultMapping = dmap

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	ipContent := "192.168.10.11"
	if err = idx.Index("doc", map[string]string{
		"ip_content": ipContent,
	}); err != nil {
		t.Fatal(err)
	}

	q := query.NewIPRangeQuery("192.168.10.0/24")
	q.SetField("ip_content")
	sr := NewSearchRequest(q)

	searchResults, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if len(searchResults.Hits) != 1 ||
		searchResults.Hits[0].ID != "doc" {
		t.Fatal("Expected the 1 result - doc")
	}
}

func TestGeoShapePolygonContainsPoint(t *testing.T) {
	fm := mapping.NewGeoShapeFieldMapping()
	dmap := mapping.NewDocumentMapping()
	dmap.AddFieldMappingsAt("geometry", fm)

	idxMapping := NewIndexMapping()
	idxMapping.DefaultMapping = dmap

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	// Polygon coordinates to be ordered in counter-clock-wise order
	// for the outer loop, and holes to follow clock-wise order.
	// See: https://www.rfc-editor.org/rfc/rfc7946.html#section-3.1.6

	one := []byte(`{
		"geometry":{
			"type":"Polygon",
			"coordinates":[[
				[4.8089,46.9307],
				[4.8223,46.8915],
				[4.8149,46.886],
				[4.8252,46.8647],
				[4.8305,46.8531],
				[4.8506,46.8509],
				[4.8574,46.8621],
				[4.8576,46.8769],
				[4.8753,46.8774],
				[4.8909,46.8519],
				[4.8837,46.8485],
				[4.9014,46.8318],
				[4.9067,46.8179],
				[4.8986,46.8122],
				[4.9081,46.7969],
				[4.9535,46.8254],
				[4.9577,46.8053],
				[5.0201,46.821],
				[5.0357,46.8207],
				[5.0656,46.8434],
				[5.0955,46.8411],
				[5.1149,46.8435],
				[5.1259,46.8395],
				[5.1433,46.8463],
				[5.1415,46.8589],
				[5.1533,46.873],
				[5.138,46.8843],
				[5.1525,46.9012],
				[5.1485,46.9165],
				[5.1582,46.926],
				[5.1882,46.9251],
				[5.2039,46.9129],
				[5.2223,46.9175],
				[5.2168,46.926],
				[5.2338,46.9316],
				[5.228,46.9505],
				[5.2078,46.9722],
				[5.2117,46.98],
				[5.1961,46.9783],
				[5.1663,46.9638],
				[5.1213,46.9634],
				[5.1086,46.9596],
				[5.0729,46.9604],
				[5.0731,46.9668],
				[5.0493,46.9817],
				[5.0034,46.9722],
				[4.9852,46.9585],
				[4.9479,46.9664],
				[4.8943,46.9663],
				[4.8937,46.951],
				[4.8534,46.9458],
				[4.8089,46.9307]
			]]
		}
	}`)

	two := []byte(`{
		"geometry":{
			"type":"Polygon",
			"coordinates":[[
				[2.2266,48.7816],
				[2.2266,48.7761],
				[2.2288,48.7745],
				[2.2717,48.7905],
				[2.2799,48.8109],
				[2.3013,48.8251],
				[2.2894,48.8283],
				[2.2726,48.8144],
				[2.2518,48.8164],
				[2.255,48.8101],
				[2.2348,48.7954],
				[2.2266,48.7816]
			]]
		}
	}`)

	var doc1, doc2 map[string]interface{}

	if err = json.Unmarshal(one, &doc1); err != nil {
		t.Fatal(err)
	}
	if err = idx.Index("doc1", doc1); err != nil {
		t.Fatal(err)
	}

	if err = json.Unmarshal(two, &doc2); err != nil {
		t.Fatal(err)
	}
	if err = idx.Index("doc2", doc2); err != nil {
		t.Fatal(err)
	}

	for testi, test := range []struct {
		coordinates []float64
		expectHits  []string
	}{
		{
			coordinates: []float64{5, 46.9},
			expectHits:  []string{"doc1"},
		},
		{
			coordinates: []float64{1.5, 48.2},
		},
	} {
		q, err := NewGeoShapeQuery(
			[][][][]float64{{{test.coordinates}}},
			geo.PointType,
			"contains",
		)
		if err != nil {
			t.Fatalf("test: %d, query err: %v", testi+1, err)
		}
		q.SetField("geometry")

		res, err := idx.Search(NewSearchRequest(q))
		if err != nil {
			t.Fatalf("test: %d, search err: %v", testi+1, err)
		}

		if len(res.Hits) != len(test.expectHits) {
			t.Errorf("test: %d, unexpected hits: %v", testi+1, len(res.Hits))
		}

	OUTER:
		for _, expect := range test.expectHits {
			for _, got := range res.Hits {
				if got.ID == expect {
					continue OUTER
				}
			}
			t.Errorf("test: %d, couldn't get: %v", testi+1, expect)
		}
	}
}

func TestAnalyzerInheritanceForDefaultDynamicMapping(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.DefaultAnalyzer = keyword.Name

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc := map[string]interface{}{
		"fieldX": "AbCdEf",
	}

	if err = idx.Index("doc", doc); err != nil {
		t.Fatal(err)
	}

	// Match query to apply keyword analyzer to fieldX.
	mq := NewMatchQuery("AbCdEf")
	mq.SetField("fieldX")

	sr := NewSearchRequest(mq)
	results, err := idx.Search(sr)
	if err != nil {
		t.Fatal(err)
	}

	if len(results.Hits) != 1 {
		t.Fatalf("expected 1 hit, got %d", len(results.Hits))
	}
}

func TestCustomDateTimeParserLayoutValidation(t *testing.T) {
	flexiblegoName := flexible.Name
	sanitizedgoName := sanitized.Name
	imap := mapping.NewIndexMapping()
	correctConfig := map[string]interface{}{
		"type": sanitizedgoName,
		"layouts": []interface{}{
			// some custom layouts
			"2006-01-02 15:04:05.0000",
			"2006\\01\\02T03:04:05PM",
			"2006/01/02",
			"2006-01-02T15:04:05.999Z0700PMMST",
			"15:04:05.0000Z07:00 Monday",

			// standard layouts
			time.Layout,
			time.ANSIC,
			time.UnixDate,
			time.RubyDate,
			time.RFC822,
			time.RFC822Z,
			time.RFC850,
			time.RFC1123,
			time.RFC1123Z,
			time.RFC3339,
			time.RFC3339Nano,
			time.Kitchen,
			time.Stamp,
			time.StampMilli,
			time.StampMicro,
			time.StampNano,
			"2006-01-02 15:04:05", // time.DateTime
			"2006-01-02",          // time.DateOnly
			"15:04:05",            // time.TimeOnly

			// Corrected layouts to the incorrect ones below.
			"2006-01-02 03:04:05 -0700",
			"2006-01-02 15:04:05 -0700",
			"3:04PM",
			"2006-01-02 15:04:05.000 -0700 MST",
			"January 2 2006 3:04 PM",
			"02/Jan/06 3:04PM",
			"Mon 02 Jan 3:04:05 PM",
		},
	}

	// Correct layouts - sanitizedgo should work without errors.
	err := imap.AddCustomDateTimeParser("custDT", correctConfig)
	if err != nil {
		t.Fatalf("expected no error, got: %v", err)
	}
	// Flexiblego should work without errors as well.
	correctConfig["type"] = flexiblegoName
	err = imap.AddCustomDateTimeParser("custDT_Flexi", correctConfig)
	if err != nil {
		t.Fatalf("expected no error, got: %v", err)
	}

	incorrectLayouts := [][]interface{}{
		{
			"2000-03-31 01:33:51 +0300",
		},
		{
			"2006-01-02 15:04:51 +0300",
		},
		{
			"2000-03-31 01:33:05 +0300",
		},
		{
			"4:45PM",
		},
		{
			"2006-01-02 15:04:05.445 -0700 MST",
		},
		{
			"August 20 2001 8:55 AM",
		},
		{
			"28/Jul/23 12:48PM",
		},
		{
			"Tue 22 Aug 6:37:30 AM",
		},
	}

	// first check sanitizedgo, should throw error for each of the incorrect layouts.
	numExpectedErrors := len(incorrectLayouts)
	numActualErrors := 0
	for idx, badLayout := range incorrectLayouts {
		incorrectConfig := map[string]interface{}{
			"type":    sanitizedgoName,
			"layouts": badLayout,
		}
		err := imap.AddCustomDateTimeParser(fmt.Sprintf("%d_DT", idx), incorrectConfig)
		if err != nil {
			numActualErrors++
		}
	}
	// Expecting all layouts to be incorrect, since sanitizedgo is being used.
	if numActualErrors != numExpectedErrors {
		t.Fatalf("expected %d errors, got: %d", numExpectedErrors, numActualErrors)
	}

	// sanity test - flexiblego should still allow the incorrect layouts, for legacy purposes
	for idx, badLayout := range incorrectLayouts {
		incorrectConfig := map[string]interface{}{
			"type":    flexiblegoName,
			"layouts": badLayout,
		}
		err := imap.AddCustomDateTimeParser(fmt.Sprintf("%d_DT_Flexi", idx), incorrectConfig)
		if err != nil {
			t.Fatalf("expected no error, got: %v", err)
		}
	}
}

func TestDateRangeStringQuery(t *testing.T) {
	idxMapping := NewIndexMapping()

	err := idxMapping.AddCustomDateTimeParser("customDT", map[string]interface{}{
		"type": sanitized.Name,
		"layouts": []interface{}{
			"02/01/2006 15:04:05",
			"2006/01/02 3:04PM",
		},
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idxMapping.AddCustomDateTimeParser("queryDT", map[string]interface{}{
		"type": sanitized.Name,
		"layouts": []interface{}{
			"02/01/2006 3:04PM",
		},
	})
	if err != nil {
		t.Fatal(err)
	}

	dtmap := NewDateTimeFieldMapping()
	dtmap.DateFormat = "customDT"
	idxMapping.DefaultMapping.AddFieldMappingsAt("date", dtmap)

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()
	documents := map[string]map[string]interface{}{
		"doc1": {
			"date": "2001/08/20 6:00PM",
		},
		"doc2": {
			"date": "20/08/2001 18:00:20",
		},
		"doc3": {
			"date": "20/08/2001 18:10:00",
		},
		"doc4": {
			"date": "2001/08/20 6:15PM",
		},
		"doc5": {
			"date": "20/08/2001 18:20:00",
		},
	}

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	type testResult struct {
		docID    string // doc ID of the hit
		hitField string // fields returned as part of the hit
	}

	type testStruct struct {
		start          string
		end            string
		field          string
		dateTimeParser string // name of the custom date time parser to use if nil, use QueryDateTimeParser
		includeStart   bool
		includeEnd     bool
		expectedHits   []testResult
		err            error
	}

	testQueries := []testStruct{
		// test cases with RFC3339 parser and toggling includeStart and includeEnd
		{
			start:        "2001-08-20T18:00:00",
			end:          "2001-08-20T18:10:00",
			field:        "date",
			includeStart: true,
			includeEnd:   true,
			expectedHits: []testResult{
				{
					docID:    "doc1",
					hitField: "2001/08/20 6:00PM",
				},
				{
					docID:    "doc2",
					hitField: "20/08/2001 18:00:20",
				},
				{
					docID:    "doc3",
					hitField: "20/08/2001 18:10:00",
				},
			},
		},
		{
			start:        "2001-08-20T18:00:00",
			end:          "2001-08-20T18:10:00",
			field:        "date",
			includeStart: false,
			includeEnd:   true,
			expectedHits: []testResult{
				{
					docID:    "doc2",
					hitField: "20/08/2001 18:00:20",
				},
				{
					docID:    "doc3",
					hitField: "20/08/2001 18:10:00",
				},
			},
		},
		{
			start:        "2001-08-20T18:00:00",
			end:          "2001-08-20T18:10:00",
			field:        "date",
			includeStart: false,
			includeEnd:   false,
			expectedHits: []testResult{
				{
					docID:    "doc2",
					hitField: "20/08/2001 18:00:20",
				},
			},
		},
		// test cases with custom parser and omitting start and end
		{
			start:          "20/08/2001 18:00:00",
			end:            "2001/08/20 6:10PM",
			field:          "date",
			dateTimeParser: "customDT",
			includeStart:   true,
			includeEnd:     true,
			expectedHits: []testResult{
				{
					docID:    "doc1",
					hitField: "2001/08/20 6:00PM",
				},
				{
					docID:    "doc2",
					hitField: "20/08/2001 18:00:20",
				},
				{
					docID:    "doc3",
					hitField: "20/08/2001 18:10:00",
				},
			},
		},
		{
			end:            "20/08/2001 18:15:00",
			field:          "date",
			dateTimeParser: "customDT",
			includeStart:   true,
			includeEnd:     true,
			expectedHits: []testResult{
				{
					docID:    "doc1",
					hitField: "2001/08/20 6:00PM",
				},
				{
					docID:    "doc2",
					hitField: "20/08/2001 18:00:20",
				},
				{
					docID:    "doc3",
					hitField: "20/08/2001 18:10:00",
				},
				{
					docID:    "doc4",
					hitField: "2001/08/20 6:15PM",
				},
			},
		},
		{
			start:          "2001/08/20 6:15PM",
			field:          "date",
			dateTimeParser: "customDT",
			includeStart:   true,
			includeEnd:     true,
			expectedHits: []testResult{
				{
					docID:    "doc4",
					hitField: "2001/08/20 6:15PM",
				},
				{
					docID:    "doc5",
					hitField: "20/08/2001 18:20:00",
				},
			},
		},
		{
			start:          "20/08/2001 6:15PM",
			field:          "date",
			dateTimeParser: "queryDT",
			includeStart:   true,
			includeEnd:     true,
			expectedHits: []testResult{
				{
					docID:    "doc4",
					hitField: "2001/08/20 6:15PM",
				},
				{
					docID:    "doc5",
					hitField: "20/08/2001 18:20:00",
				},
			},
		},
		// error path test cases
		{
			field:          "date",
			dateTimeParser: "customDT",
			includeStart:   true,
			includeEnd:     true,
			err:            fmt.Errorf("date range query must specify at least one of start/end"),
		},
		{
			field:        "date",
			includeStart: true,
			includeEnd:   true,
			err:          fmt.Errorf("date range query must specify at least one of start/end"),
		},
		{
			start:          "2001-08-20T18:00:00",
			end:            "2001-08-20T18:10:00",
			field:          "date",
			dateTimeParser: "customDT",
			err:            fmt.Errorf("unable to parse datetime with any of the layouts, date time parser name: customDT"),
		},
		{
			start: "3001-08-20T18:00:00",
			end:   "2001-08-20T18:10:00",
			field: "date",
			err:   fmt.Errorf("invalid/unsupported date range, start: 3001-08-20T18:00:00"),
		},
		{
			start:          "2001/08/20 6:00PM",
			end:            "3001/08/20 6:30PM",
			field:          "date",
			dateTimeParser: "customDT",
			err:            fmt.Errorf("invalid/unsupported date range, end: 3001/08/20 6:30PM"),
		},
	}

	for _, dtq := range testQueries {
		var err error
		dateQuery := NewDateRangeInclusiveStringQuery(dtq.start, dtq.end, &dtq.includeStart, &dtq.includeEnd)
		dateQuery.SetDateTimeParser(dtq.dateTimeParser)
		dateQuery.SetField(dtq.field)

		sr := NewSearchRequest(dateQuery)
		sr.SortBy([]string{dtq.field})
		sr.Fields = []string{dtq.field}

		res, err := idx.Search(sr)
		if err != nil {
			if dtq.err == nil {
				t.Fatalf("expected no error, got: %v", err)
			}
			if dtq.err.Error() != err.Error() {
				t.Fatalf("expected error: %v, got: %v", dtq.err, err)
			}
			continue
		}
		if len(res.Hits) != len(dtq.expectedHits) {
			t.Fatalf("expected %d hits, got %d", len(dtq.expectedHits), len(res.Hits))
		}
		for i, hit := range res.Hits {
			if hit.ID != dtq.expectedHits[i].docID {
				t.Fatalf("expected docID %s, got %s", dtq.expectedHits[i].docID, hit.ID)
			}
			if hit.Fields[dtq.field].(string) != dtq.expectedHits[i].hitField {
				t.Fatalf("expected hit field %s, got %s", dtq.expectedHits[i].hitField, hit.Fields[dtq.field])
			}
		}
	}
}

func TestDateRangeFacetQueriesWithCustomDateTimeParser(t *testing.T) {
	idxMapping := NewIndexMapping()

	err := idxMapping.AddCustomDateTimeParser("customDT", map[string]interface{}{
		"type": sanitized.Name,
		"layouts": []interface{}{
			"02/01/2006 15:04:05",
			"2006/01/02 3:04PM",
		},
	})
	if err != nil {
		t.Fatal(err)
	}

	err = idxMapping.AddCustomDateTimeParser("queryDT", map[string]interface{}{
		"type": sanitized.Name,
		"layouts": []interface{}{
			"02/01/2006 3:04PM",
		},
	})
	if err != nil {
		t.Fatal(err)
	}

	dtmap := NewDateTimeFieldMapping()
	dtmap.DateFormat = "customDT"
	idxMapping.DefaultMapping.AddFieldMappingsAt("date", dtmap)

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, idxMapping)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()
	documents := map[string]map[string]interface{}{
		"doc1": {
			"date": "2001/08/20 6:00PM",
		},
		"doc2": {
			"date": "20/08/2001 18:00:20",
		},
		"doc3": {
			"date": "20/08/2001 18:10:00",
		},
		"doc4": {
			"date": "2001/08/20 6:15PM",
		},
		"doc5": {
			"date": "20/08/2001 18:20:00",
		},
	}

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	query := NewMatchAllQuery()

	type testFacetResult struct {
		name  string
		start string
		end   string
		count int
		err   error
	}

	type testFacetRequest struct {
		name   string
		start  string
		end    string
		parser string
		result testFacetResult
	}

	tests := []testFacetRequest{
		{
			// Test without a query time override of the parser (use default parser)
			name:  "test",
			start: "2001-08-20 18:00:00",
			end:   "2001-08-20 18:10:00",
			result: testFacetResult{
				name:  "test",
				start: "2001-08-20T18:00:00Z",
				end:   "2001-08-20T18:10:00Z",
				count: 2,
				err:   nil,
			},
		},
		{
			name:   "test",
			start:  "20/08/2001 6:00PM",
			end:    "20/08/2001 6:10PM",
			parser: "queryDT",
			result: testFacetResult{
				name:  "test",
				start: "2001-08-20T18:00:00Z",
				end:   "2001-08-20T18:10:00Z",
				count: 2,
				err:   nil,
			},
		},
		{
			name:   "test",
			start:  "20/08/2001 15:00:00",
			end:    "2001/08/20 6:10PM",
			parser: "customDT",
			result: testFacetResult{
				name:  "test",
				start: "2001-08-20T15:00:00Z",
				end:   "2001-08-20T18:10:00Z",
				count: 2,
				err:   nil,
			},
		},
		{
			name:   "test",
			end:    "2001/08/20 6:15PM",
			parser: "customDT",
			result: testFacetResult{
				name:  "test",
				end:   "2001-08-20T18:15:00Z",
				count: 3,
				err:   nil,
			},
		},
		{
			name:   "test",
			start:  "20/08/2001 6:15PM",
			parser: "queryDT",
			result: testFacetResult{
				name:  "test",
				start: "2001-08-20T18:15:00Z",
				count: 2,
				err:   nil,
			},
		},
		// some error cases
		{
			name:   "test",
			parser: "queryDT",
			result: testFacetResult{
				name: "test",
				err:  fmt.Errorf("date range query must specify either start, end or both for date range name 'test'"),
			},
		},
		{
			// default parser is used for the query, but the start time is not in the correct format (RFC3339),
			// so it should throw an error
			name:  "test",
			start: "20/08/2001 6:15PM",
			result: testFacetResult{
				name: "test",
				err:  fmt.Errorf("ParseDates err: error parsing start date '20/08/2001 6:15PM' for date range name 'test': unable to parse datetime with any of the layouts, using date time parser named dateTimeOptional"),
			},
		},
	}

	for _, test := range tests {
		searchRequest := NewSearchRequest(query)

		fr := NewFacetRequest("date", 100)
		start := &test.start
		if test.start == "" {
			start = nil
		}
		end := &test.end
		if test.end == "" {
			end = nil
		}

		fr.AddDateTimeRangeStringWithParser(test.name, start, end, test.parser)
		searchRequest.AddFacet("dateFacet", fr)

		searchResults, err := idx.Search(searchRequest)
		if err != nil {
			if test.result.err == nil {
				t.Fatalf("Unexpected error: %v", err)
			}
			if err.Error() != test.result.err.Error() {
				t.Fatalf("Expected error %v, got %v", test.result.err, err)
			}
			continue
		}
		for _, facetResult := range searchResults.Facets {
			if len(facetResult.DateRanges) != 1 {
				t.Fatal("Expected 1 date range facet")
			}
			result := facetResult.DateRanges[0]
			if result.Name != test.result.name {
				t.Fatalf("Expected name %s, got %s", test.result.name, result.Name)
			}
			if result.Start != nil && *result.Start != test.result.start {
				t.Fatalf("Expected start %s, got %s", test.result.start, *result.Start)
			}
			if result.End != nil && *result.End != test.result.end {
				t.Fatalf("Expected end %s, got %s", test.result.end, *result.End)
			}
			if result.Start == nil && test.result.start != "" {
				t.Fatalf("Expected start %s, got nil", test.result.start)
			}
			if result.End == nil && test.result.end != "" {
				t.Fatalf("Expected end %s, got nil", test.result.end)
			}
			if result.Count != test.result.count {
				t.Fatalf("Expected count %d, got %d", test.result.count, result.Count)
			}
		}
	}
}

func TestDateRangeTimestampQueries(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	imap := mapping.NewIndexMapping()

	// add a date field with a valid format to the default mapping
	// for good measure

	dtParserConfig := map[string]interface{}{
		"type":    flexible.Name,
		"layouts": []interface{}{"2006/01/02 15:04:05"},
	}
	err := imap.AddCustomDateTimeParser("custDT", dtParserConfig)
	if err != nil {
		t.Fatal(err)
	}

	dateField := mapping.NewDateTimeFieldMapping()
	dateField.DateFormat = "custDT"

	unixSecField := mapping.NewDateTimeFieldMapping()
	unixSecField.DateFormat = seconds.Name

	unixMilliSecField := mapping.NewDateTimeFieldMapping()
	unixMilliSecField.DateFormat = milliseconds.Name

	unixMicroSecField := mapping.NewDateTimeFieldMapping()
	unixMicroSecField.DateFormat = microseconds.Name

	unixNanoSecField := mapping.NewDateTimeFieldMapping()
	unixNanoSecField.DateFormat = nanoseconds.Name

	imap.DefaultMapping.AddFieldMappingsAt("date", dateField)
	imap.DefaultMapping.AddFieldMappingsAt("seconds", unixSecField)
	imap.DefaultMapping.AddFieldMappingsAt("milliseconds", unixMilliSecField)
	imap.DefaultMapping.AddFieldMappingsAt("microseconds", unixMicroSecField)
	imap.DefaultMapping.AddFieldMappingsAt("nanoseconds", unixNanoSecField)

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	documents := map[string]map[string]string{
		"doc1": {
			"date":         "2001/08/20 03:00:10",
			"seconds":      "998276410",
			"milliseconds": "998276410100",
			"microseconds": "998276410100300",
			"nanoseconds":  "998276410100300400",
		},
		"doc2": {
			"date":         "2001/08/20 03:00:20",
			"seconds":      "998276420",
			"milliseconds": "998276410200",
			"microseconds": "998276410100400",
			"nanoseconds":  "998276410100300500",
		},
		"doc3": {
			"date":         "2001/08/20 03:00:30",
			"seconds":      "998276430",
			"milliseconds": "998276410300",
			"microseconds": "998276410100500",
			"nanoseconds":  "998276410100300600",
		},
		"doc4": {
			"date":         "2001/08/20 03:00:40",
			"seconds":      "998276440",
			"milliseconds": "998276410400",
			"microseconds": "998276410100600",
			"nanoseconds":  "998276410100300700",
		},
		"doc5": {
			"date":         "2001/08/20 03:00:50",
			"seconds":      "998276450",
			"milliseconds": "998276410500",
			"microseconds": "998276410100700",
			"nanoseconds":  "998276410100300800",
		},
	}

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	type testStruct struct {
		start        string
		end          string
		field        string
		expectedHits []string
	}

	testQueries := []testStruct{
		{
			start: "2001-08-20T03:00:05",
			end:   "2001-08-20T03:00:25",
			field: "date",
			expectedHits: []string{
				"doc1",
				"doc2",
			},
		},
		{
			start: "2001-08-20T03:00:15",
			end:   "2001-08-20T03:00:35",
			field: "seconds",
			expectedHits: []string{
				"doc2",
				"doc3",
			},
		},
		{
			start: "2001-08-20T03:00:10.150",
			end:   "2001-08-20T03:00:10.450",
			field: "milliseconds",
			expectedHits: []string{
				"doc2",
				"doc3",
				"doc4",
			},
		},
		{
			start: "2001-08-20T03:00:10.100450",
			end:   "2001-08-20T03:00:10.100650",
			field: "microseconds",
			expectedHits: []string{
				"doc3",
				"doc4",
			},
		},
		{
			start: "2001-08-20T03:00:10.100300550",
			end:   "2001-08-20T03:00:10.100300850",
			field: "nanoseconds",
			expectedHits: []string{
				"doc3",
				"doc4",
				"doc5",
			},
		},
	}
	testLayout := "2006-01-02T15:04:05"
	for _, dtq := range testQueries {
		startTime, err := time.Parse(testLayout, dtq.start)
		if err != nil {
			t.Fatal(err)
		}
		endTime, err := time.Parse(testLayout, dtq.end)
		if err != nil {
			t.Fatal(err)
		}
		drq := NewDateRangeQuery(startTime, endTime)
		drq.SetField(dtq.field)

		sr := NewSearchRequest(drq)
		sr.SortBy([]string{dtq.field})
		sr.Fields = []string{"*"}

		res, err := idx.Search(sr)
		if err != nil {
			t.Fatal(err)
		}
		if len(res.Hits) != len(dtq.expectedHits) {
			t.Fatalf("expected %d hits, got %d", len(dtq.expectedHits), len(res.Hits))
		}
		for i, hit := range res.Hits {
			if hit.ID != dtq.expectedHits[i] {
				t.Fatalf("expected docID %s, got %s", dtq.expectedHits[i], hit.ID)
			}
			if len(hit.Fields) != len(documents[hit.ID]) {
				t.Fatalf("expected hit %s to have %d fields, got %d", hit.ID, len(documents[hit.ID]), len(hit.Fields))
			}
			for k, v := range documents[hit.ID] {
				if hit.Fields[k] != v {
					t.Fatalf("expected field %s to be %s, got %s", k, v, hit.Fields[k])
				}
			}
		}
	}
}

func TestPercentAndIsoStyleDates(t *testing.T) {
	percentName := percent.Name
	isoName := iso.Name

	imap := mapping.NewIndexMapping()
	percentConfig := map[string]interface{}{
		"type": percentName,
		"layouts": []interface{}{
			"%Y/%m/%d %l:%M%p",                // doc 1
			"%d/%m/%Y %H:%M:%S",               // doc 2
			"%Y-%m-%dT%H:%M:%S%z",             // doc 3
			"%d %B %y %l%p %Z",                // doc 4
			"%Y; %b %d (%a) %I:%M:%S.%N%P %z", // doc 5
		},
	}
	isoConfig := map[string]interface{}{
		"type": isoName,
		"layouts": []interface{}{
			"yyyy/MM/dd h:mma",                       // doc 1
			"dd/MM/yyyy HH:mm:ss",                    // doc 2
			"yyyy-MM-dd'T'HH:mm:ssXX",                // doc 3
			"dd MMMM yy ha z",                        // doc 4
			"yyyy; MMM dd (EEE) hh:mm:ss.SSSSSaa xx", // doc 5
		},
	}

	err := imap.AddCustomDateTimeParser("percentDate", percentConfig)
	if err != nil {
		t.Fatal(err)
	}
	err = imap.AddCustomDateTimeParser("isoDate", isoConfig)
	if err != nil {
		t.Fatal(err)
	}

	percentField := mapping.NewDateTimeFieldMapping()
	percentField.DateFormat = "percentDate"

	isoField := mapping.NewDateTimeFieldMapping()
	isoField.DateFormat = "isoDate"

	imap.DefaultMapping.AddFieldMappingsAt("percentDate", percentField)
	imap.DefaultMapping.AddFieldMappingsAt("isoDate", isoField)

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	documents := map[string]map[string]interface{}{
		"doc1": {
			"percentDate": "2001/08/20 6:00PM",
			"isoDate":     "2001/08/20 6:00PM",
		},
		"doc2": {
			"percentDate": "20/08/2001 18:05:00",
			"isoDate":     "20/08/2001 18:05:00",
		},
		"doc3": {
			"percentDate": "2001-08-20T18:10:00Z",
			"isoDate":     "2001-08-20T18:10:00Z",
		},
		"doc4": {
			"percentDate": "20 August 01 6PM UTC",
			"isoDate":     "20 August 01 6PM UTC",
		},
		"doc5": {
			"percentDate": "2001; Aug 20 (Mon) 06:15:15.23456pm +0000",
			"isoDate":     "2001; Aug 20 (Mon) 06:15:15.23456pm +0000",
		},
	}

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	type testStruct struct {
		start string
		end   string
		field string
	}

	for _, field := range []string{"percentDate", "isoDate"} {
		testQueries := []testStruct{
			{
				start: "2001/08/20 6:00PM",
				end:   "2001/08/20 6:20PM",
				field: field,
			},
			{
				start: "20/08/2001 18:00:00",
				end:   "20/08/2001 18:20:00",
				field: field,
			},
			{
				start: "2001-08-20T18:00:00Z",
				end:   "2001-08-20T18:20:00Z",
				field: field,
			},
			{
				start: "20 August 01 6PM UTC",
				end:   "20 August 01 7PM UTC",
				field: field,
			},
			{
				start: "2001; Aug 20 (Mon) 06:00:00.00000pm +0000",
				end:   "2001; Aug 20 (Mon) 06:20:20.00000pm +0000",
				field: field,
			},
		}
		includeStart := true
		includeEnd := true
		for _, dtq := range testQueries {
			drq := NewDateRangeInclusiveStringQuery(dtq.start, dtq.end, &includeStart, &includeEnd)
			drq.SetField(dtq.field)
			drq.SetDateTimeParser(field)
			sr := NewSearchRequest(drq)
			res, err := idx.Search(sr)
			if err != nil {
				t.Fatal(err)
			}
			if len(res.Hits) != 5 {
				t.Fatalf("expected %d hits, got %d", 5, len(res.Hits))
			}
		}
	}
}

func roundToDecimalPlace(num float64, decimalPlaces int) float64 {
	precision := math.Pow(10, float64(decimalPlaces))
	return math.Round(num*precision) / precision
}

func TestScoreBreakdown(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	imap := mapping.NewIndexMapping()
	textField := mapping.NewTextFieldMapping()
	textField.Analyzer = simple.Name
	imap.DefaultMapping.AddFieldMappingsAt("text", textField)

	documents := map[string]map[string]interface{}{
		"doc1": {
			"text": "lorem ipsum dolor sit amet consectetur adipiscing elit do eiusmod tempor",
		},
		"doc2": {
			"text": "lorem dolor amet adipiscing sed eiusmod",
		},
		"doc3": {
			"text": "ipsum sit consectetur elit do tempor",
		},
		"doc4": {
			"text": "lorem ipsum sit amet adipiscing elit do eiusmod",
		},
	}

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	type testResult struct {
		docID          string // doc ID of the hit
		score          float64
		scoreBreakdown map[int]float64
	}
	type testStruct struct {
		query      string
		typ        string
		expectHits []testResult
	}
	testQueries := []testStruct{
		{
			// trigger disjunction heap searcher (>10 searchers)
			// expect score breakdown to have a 0 at BLANK
			query: `{"disjuncts":[{"term":"lorem","field":"text"},{"term":"blank","field":"text"},{"term":"ipsum","field":"text"},{"term":"blank","field":"text"},{"term":"blank","field":"text"},{"term":"dolor","field":"text"},{"term":"sit","field":"text"},{"term":"amet","field":"text"},{"term":"consectetur","field":"text"},{"term":"blank","field":"text"},{"term":"adipiscing","field":"text"},{"term":"blank","field":"text"},{"term":"elit","field":"text"},{"term":"sed","field":"text"},{"term":"do","field":"text"},{"term":"eiusmod","field":"text"},{"term":"tempor","field":"text"},{"term":"blank","field":"text"},{"term":"blank","field":"text"}]}`,
			typ:   "disjunction",
			expectHits: []testResult{
				{
					docID:          "doc1",
					score:          0.3034548543819603,
					scoreBreakdown: map[int]float64{0: 0.040398807605268316, 2: 0.040398807605268316, 5: 0.0669862776967768, 6: 0.040398807605268316, 7: 0.040398807605268316, 8: 0.0669862776967768, 10: 0.040398807605268316, 12: 0.040398807605268316, 14: 0.040398807605268316, 15: 0.040398807605268316, 16: 0.0669862776967768},
				},
				{
					docID:          "doc2",
					score:          0.14725661652397853,
					scoreBreakdown: map[int]float64{0: 0.05470024557900147, 5: 0.09069985124905133, 7: 0.05470024557900147, 10: 0.05470024557900147, 13: 0.15681178542754148, 15: 0.05470024557900147},
				},
				{
					docID:          "doc3",
					score:          0.12637916362550797,
					scoreBreakdown: map[int]float64{2: 0.05470024557900147, 6: 0.05470024557900147, 8: 0.09069985124905133, 12: 0.05470024557900147, 14: 0.05470024557900147, 16: 0.09069985124905133},
				},
				{
					docID:          "doc4",
					score:          0.15956816751152955,
					scoreBreakdown: map[int]float64{0: 0.04737179972998534, 2: 0.04737179972998534, 6: 0.04737179972998534, 7: 0.04737179972998534, 10: 0.04737179972998534, 12: 0.04737179972998534, 14: 0.04737179972998534, 15: 0.04737179972998534},
				},
			},
		},
		{
			// trigger disjunction slice searcher (< 10 searchers)
			// expect BLANK to give a 0 in score breakdown
			query: `{"disjuncts":[{"term":"blank","field":"text"},{"term":"lorem","field":"text"},{"term":"ipsum","field":"text"},{"term":"blank","field":"text"},{"term":"blank","field":"text"},{"term":"dolor","field":"text"},{"term":"sit","field":"text"},{"term":"blank","field":"text"}]}`,
			typ:   "disjunction",
			expectHits: []testResult{
				{
					docID:          "doc1",
					score:          0.1340684440934241,
					scoreBreakdown: map[int]float64{1: 0.05756326446708409, 2: 0.05756326446708409, 5: 0.09544709478559595, 6: 0.05756326446708409},
				},
				{
					docID:          "doc2",
					score:          0.05179425287147191,
					scoreBreakdown: map[int]float64{1: 0.0779410306721006, 5: 0.129235980813787},
				},
				{
					docID:          "doc3",
					score:          0.0389705153360503,
					scoreBreakdown: map[int]float64{2: 0.0779410306721006, 6: 0.0779410306721006},
				},
				{
					docID:          "doc4",
					score:          0.07593627256602972,
					scoreBreakdown: map[int]float64{1: 0.06749890894758198, 2: 0.06749890894758198, 6: 0.06749890894758198},
				},
			},
		},
	}
	for _, dtq := range testQueries {
		var q query.Query
		var rv query.DisjunctionQuery
		err := json.Unmarshal([]byte(dtq.query), &rv)
		if err != nil {
			t.Fatal(err)
		}
		rv.RetrieveScoreBreakdown(true)
		q = &rv
		sr := NewSearchRequest(q)
		sr.SortBy([]string{"_id"})
		sr.Explain = true
		res, err := idx.Search(sr)
		if err != nil {
			t.Fatal(err)
		}
		if len(res.Hits) != len(dtq.expectHits) {
			t.Fatalf("expected %d hits, got %d", len(dtq.expectHits), len(res.Hits))
		}
		for i, hit := range res.Hits {
			if hit.ID != dtq.expectHits[i].docID {
				t.Fatalf("expected docID %s, got %s", dtq.expectHits[i].docID, hit.ID)
			}
			if len(hit.ScoreBreakdown) != len(dtq.expectHits[i].scoreBreakdown) {
				t.Fatalf("expected %d score breakdown, got %d", len(dtq.expectHits[i].scoreBreakdown), len(hit.ScoreBreakdown))
			}
			for j, score := range hit.ScoreBreakdown {
				actualScore := roundToDecimalPlace(score, 3)
				expectScore := roundToDecimalPlace(dtq.expectHits[i].scoreBreakdown[j], 3)
				if actualScore != expectScore {
					t.Fatalf("expected score breakdown %f, got %f", dtq.expectHits[i].scoreBreakdown[j], score)
				}
			}
		}
	}
}

func TestAutoFuzzy(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	imap := mapping.NewIndexMapping()

	if err := imap.AddCustomAnalyzer("splitter", map[string]interface{}{
		"type":          custom.Name,
		"tokenizer":     whitespace.Name,
		"token_filters": []interface{}{lowercase.Name},
	}); err != nil {
		t.Fatal(err)
	}

	textField := mapping.NewTextFieldMapping()
	textField.Analyzer = "splitter"
	textField.Store = true
	textField.IncludeTermVectors = true
	textField.IncludeInAll = true

	imap.DefaultMapping.Dynamic = false
	imap.DefaultMapping.AddFieldMappingsAt("model", textField)

	documents := map[string]map[string]interface{}{
		"product1": {
			"model": "apple iphone 12",
		},
		"product2": {
			"model": "apple iphone 13",
		},
		"product3": {
			"model": "samsung galaxy s22",
		},
		"product4": {
			"model": "samsung galaxy note",
		},
		"product5": {
			"model": "google pixel 5",
		},
		"product6": {
			"model": "oneplus 9 pro",
		},
		"product7": {
			"model": "xiaomi mi 11",
		},
		"product8": {
			"model": "oppo find x3",
		},
		"product9": {
			"model": "vivo x60 pro",
		},
		"product10": {
			"model": "oneplus 8t pro",
		},
		"product11": {
			"model": "nokia xr20",
		},
		"product12": {
			"model": "poco f1",
		},
		"product13": {
			"model": "asus rog 5",
		},
		"product14": {
			"model": "samsung galaxy a15 5g",
		},
		"product15": {
			"model": "tecno camon 17",
		},
	}
	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	type testStruct struct {
		query      string
		expectHits []string
	}
	testQueries := []testStruct{
		{
			// match query with fuzziness set to 2
			query: `{
					"match" : "applle iphone 12",
					"fuzziness": 2,
					"field" : "model"
				}`,
			expectHits: []string{"product1", "product2", "product7", "product14", "product15", "product12", "product10", "product3", "product6", "product8"},
		},
		{
			// match query with fuzziness set to "auto"
			query: `{
					"match" : "applle iphone 12",
					"fuzziness": "auto",
					"field" : "model"
				}`,
			expectHits: []string{"product1", "product2"},
		},
		{
			// match query with fuzziness set to 2 with `and` operator
			query: `{
					"match" : "applle iphone 12",
					"fuzziness": 2,
					"field" : "model",
					"operator": "and"
				}`,
			expectHits: []string{"product1", "product2"},
		},
		{
			// match query with fuzziness set to "auto" with `and`` operator
			query: `{
					"match" : "applle iphone 12",
					"fuzziness": "auto",
					"field" : "model",
					"operator": "and"
				}`,
			expectHits: []string{"product1"},
		},
		// match phrase query with fuzziness set to 2
		{
			query: `{
					"match_phrase" : "onplus 9 pro",
					"fuzziness": 2,
					"field" : "model"
				}`,
			expectHits: []string{"product6", "product10"},
		},
		// match phrase query with fuzziness set to "auto"
		{
			query: `{
				"match_phrase" : "onplus 9 pro",
				"fuzziness": "auto",
				"field" : "model"
			}`,
			expectHits: []string{"product6"},
		},
	}

	for _, dtq := range testQueries {
		q, err := query.ParseQuery([]byte(dtq.query))
		if err != nil {
			t.Fatal(err)
		}

		sr := NewSearchRequest(q)
		sr.Highlight = NewHighlightWithStyle(ansi.Name)
		sr.SortBy([]string{"-_score", "_id"})
		sr.Fields = []string{"*"}
		sr.Explain = true

		res, err := idx.Search(sr)
		if err != nil {
			t.Fatal(err)
		}
		if len(res.Hits) != len(dtq.expectHits) {
			t.Fatalf("expected %d hits, got %d", len(dtq.expectHits), len(res.Hits))
		}
		for i, hit := range res.Hits {
			if hit.ID != dtq.expectHits[i] {
				t.Fatalf("expected docID %s, got %s", dtq.expectHits[i], hit.ID)
			}
		}
	}
}

func TestThesaurusTermReader(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	synonymCollection := "collection1"

	synonymSourceName := "english"

	analyzer := simple.Name

	synonymSourceConfig := map[string]interface{}{
		"collection": synonymCollection,
		"analyzer":   analyzer,
	}

	textField := mapping.NewTextFieldMapping()
	textField.Analyzer = analyzer
	textField.SynonymSource = synonymSourceName

	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.AddFieldMappingsAt("text", textField)
	err := imap.AddSynonymSource(synonymSourceName, synonymSourceConfig)
	if err != nil {
		t.Fatal(err)
	}
	err = imap.Validate()
	if err != nil {
		t.Fatal(err)
	}

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	documents := map[string]map[string]interface{}{
		"doc1": {
			"text": "quick brown fox eats",
		},
		"doc2": {
			"text": "fast red wolf jumps",
		},
		"doc3": {
			"text": "quick red cat runs",
		},
		"doc4": {
			"text": "speedy brown dog barks",
		},
		"doc5": {
			"text": "fast green rabbit hops",
		},
	}

	batch := idx.NewBatch()
	for docID, doc := range documents {
		err := batch.Index(docID, doc)
		if err != nil {
			t.Fatal(err)
		}
	}

	synonymDocuments := map[string]*SynonymDefinition{
		"synDoc1": {
			Synonyms: []string{"quick", "fast", "speedy"},
		},
		"synDoc2": {
			Input:    []string{"color", "colour"},
			Synonyms: []string{"red", "green", "blue", "yellow", "brown"},
		},
		"synDoc3": {
			Input:    []string{"animal", "creature"},
			Synonyms: []string{"fox", "wolf", "cat", "dog", "rabbit"},
		},
		"synDoc4": {
			Synonyms: []string{"eats", "jumps", "runs", "barks", "hops"},
		},
	}

	for synName, synDef := range synonymDocuments {
		err := batch.IndexSynonym(synName, synonymCollection, synDef)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = idx.Batch(batch)
	if err != nil {
		t.Fatal(err)
	}

	sco, err := idx.Advanced()
	if err != nil {
		t.Fatal(err)
	}

	reader, err := sco.Reader()
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = reader.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	thesReader, ok := reader.(index.ThesaurusReader)
	if !ok {
		t.Fatal("expected thesaurus reader")
	}

	type testStruct struct {
		queryTerm        string
		expectedSynonyms []string
	}

	testQueries := []testStruct{
		{
			queryTerm:        "quick",
			expectedSynonyms: []string{"fast", "speedy"},
		},
		{
			queryTerm:        "red",
			expectedSynonyms: []string{},
		},
		{
			queryTerm:        "color",
			expectedSynonyms: []string{"red", "green", "blue", "yellow", "brown"},
		},
		{
			queryTerm:        "colour",
			expectedSynonyms: []string{"red", "green", "blue", "yellow", "brown"},
		},
		{
			queryTerm:        "animal",
			expectedSynonyms: []string{"fox", "wolf", "cat", "dog", "rabbit"},
		},
		{
			queryTerm:        "creature",
			expectedSynonyms: []string{"fox", "wolf", "cat", "dog", "rabbit"},
		},
		{
			queryTerm:        "fox",
			expectedSynonyms: []string{},
		},
		{
			queryTerm:        "eats",
			expectedSynonyms: []string{"jumps", "runs", "barks", "hops"},
		},
		{
			queryTerm:        "jumps",
			expectedSynonyms: []string{"eats", "runs", "barks", "hops"},
		},
	}

	for _, test := range testQueries {
		str, err := thesReader.ThesaurusTermReader(context.Background(), synonymSourceName, []byte(test.queryTerm))
		if err != nil {
			t.Fatal(err)
		}
		var gotSynonyms []string
		for {
			synonym, err := str.Next()
			if err != nil {
				t.Fatal(err)
			}
			if synonym == "" {
				break
			}
			gotSynonyms = append(gotSynonyms, string(synonym))
		}
		if len(gotSynonyms) != len(test.expectedSynonyms) {
			t.Fatalf("expected %d synonyms, got %d", len(test.expectedSynonyms), len(gotSynonyms))
		}
		sort.Strings(gotSynonyms)
		sort.Strings(test.expectedSynonyms)
		for i, syn := range gotSynonyms {
			if syn != test.expectedSynonyms[i] {
				t.Fatalf("expected synonym %s, got %s", test.expectedSynonyms[i], syn)
			}
		}
	}
}

func TestSynonymSearchQueries(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	synonymCollection := "collection1"

	synonymSourceName := "english"

	analyzer := en.AnalyzerName

	synonymSourceConfig := map[string]interface{}{
		"collection": synonymCollection,
		"analyzer":   analyzer,
	}

	textField := mapping.NewTextFieldMapping()
	textField.Analyzer = analyzer
	textField.SynonymSource = synonymSourceName

	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.AddFieldMappingsAt("text", textField)
	err := imap.AddSynonymSource(synonymSourceName, synonymSourceConfig)
	if err != nil {
		t.Fatal(err)
	}
	err = imap.Validate()
	if err != nil {
		t.Fatal(err)
	}

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	documents := map[string]map[string]interface{}{
		"doc1": {
			"text": `The hardworking employee consistently strives to exceed expectations.
					His industrious nature makes him a valuable asset to any team.
					His conscientious attention to detail ensures that projects are completed efficiently and accurately.
					He remains persistent even in the face of challenges.`,
		},
		"doc2": {
			"text": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life.
					Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation.
					The calm environment offers the ideal place to meditate and connect with nature.
					Even the most stressed individuals find themselves feeling relaxed and at ease.`,
		},
		"doc3": {
			"text": `The house was burned down, leaving only a charred shell behind.
					The intense heat of the flames caused the walls to warp and the roof to cave in.
					The seared remains of the furniture told the story of the blaze.
					The incinerated remains left little more than ashes to remember what once was.`,
		},
		"doc4": {
			"text": `The faithful dog followed its owner everywhere, always loyal and steadfast.
					It was devoted to protecting its family, and its reliable nature meant it could always be trusted.
					In the face of danger, the dog remained calm, knowing its role was to stay vigilant.
					Its trustworthy companionship provided comfort and security.`,
		},
		"doc5": {
			"text": `The lively market is bustling with activity from morning to night.
					The dynamic energy of the crowd fills the air as vendors sell their wares.
					Shoppers wander from stall to stall, captivated by the vibrant colors and energetic atmosphere.
					This place is alive with movement and life.`,
		},
		"doc6": {
			"text": `In moments of crisis, bravery shines through.
					It takes valor to step forward when others are afraid to act.
					Heroes are defined by their guts and nerve, taking risks to protect others.
					Boldness in the face of danger is what sets them apart.`,
		},
		"doc7": {
			"text": `Innovation is the driving force behind progress in every industry.
					The company fosters an environment of invention, encouraging creativity at every level.
					The focus on novelty and improvement means that ideas are always evolving.
					The development of new solutions is at the core of the company's mission.`,
		},
		"doc8": {
			"text": `The blazing sunset cast a radiant glow over the horizon, painting the sky with hues of red and orange.
					The intense heat of the day gave way to a fiery display of color.
					As the sun set, the glowing light illuminated the landscape, creating a breathtaking scene.
					The fiery sky was a sight to behold.`,
		},
		"doc9": {
			"text": `The fertile soil of the valley makes it perfect for farming.
					The productive land yields abundant crops year after year.
					Farmers rely on the rich, fruitful ground to sustain their livelihoods.
					The area is known for its plentiful harvests, supporting both local communities and export markets.`,
		},
		"doc10": {
			"text": `The arid desert is a vast, dry expanse with little water or vegetation.
					The barren landscape stretches as far as the eye can see, offering little respite from the scorching sun.
					The desolate environment is unforgiving to those who venture too far without preparation.
					The parched earth cracks under the heat, creating a harsh, unyielding terrain.`,
		},
		"doc11": {
			"text": `The fox is known for its cunning and intelligence.
					As a predator, it relies on its sharp instincts to outwit its prey.
					Its vulpine nature makes it both mysterious and fascinating.
					The fox's ability to hunt with precision and stealth is what makes it such a formidable hunter.`,
		},
		"doc12": {
			"text": `The dog is often considered man's best friend due to its loyal nature.
					As a companion, the hound provides both protection and affection.
					The puppy quickly becomes a member of the family, always by your side.
					Its playful energy and unshakable loyalty make it a beloved pet.`,
		},
		"doc13": {
			"text": `He worked tirelessly through the night, always persistent in his efforts.
					His industrious approach to problem-solving kept the project moving forward.
					No matter how difficult the task, he remained focused, always giving his best.
					His dedication paid off when the project was completed ahead of schedule.`,
		},
		"doc14": {
			"text": `The river flowed calmly through the valley, its peaceful current offering a sense of tranquility.
					Fishermen relaxed by the banks, enjoying the calm waters that reflected the sky above.
					The tranquil nature of the river made it a perfect spot for meditation.
					As the day ended, the river's quiet flow brought a sense of peace.`,
		},
		"doc15": {
			"text": `After the fire, all that was left was the charred remains of what once was.
					The seared walls of the house told a tragic story.
					The intensity of the blaze had burned everything in its path, leaving only the smoldering wreckage behind.
					The incinerated objects could not be salvaged, and the damage was beyond repair.`,
		},
		"doc16": {
			"text": `The devoted employee always went above and beyond to complete his tasks.
					His steadfast commitment to the company made him a valuable team member.
					He was reliable, never failing to meet deadlines.
					His trustworthiness earned him the respect of his colleagues, and was considered an
					ingenious expert in his field.`,
		},
		"doc17": {
			"text": `The city is vibrant, full of life and energy.
					The dynamic pace of the streets reflects the diverse culture of its inhabitants.
					People from all walks of life contribute to the energetic atmosphere.
					The city's lively spirit can be felt in every corner, from the bustling markets to the lively festivals.`,
		},
		"doc18": {
			"text": `In a moment of uncertainty, he made a bold decision that would change his life forever.
					It took courage and nerve to take the leap, but his bravery paid off.
					The guts to face the unknown allowed him to achieve something remarkable.
					Being an bright scholar, the skill he demonstrated inspired those around him.`,
		},
		"doc19": {
			"text": `Innovation is often born from necessity, and the lightbulb is a prime example.
					Thomas Edison's invention changed the world, offering a new way to see the night.
					The creativity involved in developing such a groundbreaking product sparked a wave of
					novelty in the scientific community. This improvement in technology continues to shape the modern world.
					He was a clever academic and a smart researcher.`,
		},
		"doc20": {
			"text": `The fiery volcano erupted with a force that shook the earth. Its radiant lava flowed down the sides,
					illuminating the night sky. The intense heat from the eruption could be felt miles away, as the
					glowing lava burned everything in its path. The fiery display was both terrifying and mesmerizing.`,
		},
	}

	synonymDocuments := map[string]*SynonymDefinition{
		"synDoc1": {
			Synonyms: []string{"hardworking", "industrious", "conscientious", "persistent", "focused", "devoted"},
		},
		"synDoc2": {
			Synonyms: []string{"tranquil", "peaceful", "calm", "relaxed", "unruffled"},
		},
		"synDoc3": {
			Synonyms: []string{"burned", "charred", "seared", "incinerated", "singed"},
		},
		"synDoc4": {
			Synonyms: []string{"faithful", "steadfast", "devoted", "reliable", "trustworthy"},
		},
		"synDoc5": {
			Synonyms: []string{"lively", "dynamic", "energetic", "vivid", "vibrating"},
		},
		"synDoc6": {
			Synonyms: []string{"bravery", "valor", "guts", "nerve", "boldness"},
		},
		"synDoc7": {
			Input:    []string{"innovation"},
			Synonyms: []string{"invention", "creativity", "novelty", "improvement", "development"},
		},
		"synDoc8": {
			Input:    []string{"blazing"},
			Synonyms: []string{"intense", "radiant", "burning", "fiery", "glowing"},
		},
		"synDoc9": {
			Input:    []string{"fertile"},
			Synonyms: []string{"productive", "fruitful", "rich", "abundant", "plentiful"},
		},
		"synDoc10": {
			Input:    []string{"arid"},
			Synonyms: []string{"dry", "barren", "desolate", "parched", "unfertile"},
		},
		"synDoc11": {
			Input:    []string{"fox"},
			Synonyms: []string{"vulpine", "canine", "predator", "hunter", "pursuer"},
		},
		"synDoc12": {
			Input:    []string{"dog"},
			Synonyms: []string{"canine", "hound", "puppy", "pup", "companion"},
		},
		"synDoc13": {
			Synonyms: []string{"researcher", "scientist", "scholar", "academic", "expert"},
		},
		"synDoc14": {
			Synonyms: []string{"bright", "clever", "ingenious", "sharp", "astute", "smart"},
		},
	}

	// Combine both maps into a slice of map entries (as they both have similar structure)
	var combinedDocIDs []string
	for id := range synonymDocuments {
		combinedDocIDs = append(combinedDocIDs, id)
	}
	for id := range documents {
		combinedDocIDs = append(combinedDocIDs, id)
	}
	rand.Shuffle(len(combinedDocIDs), func(i, j int) {
		combinedDocIDs[i], combinedDocIDs[j] = combinedDocIDs[j], combinedDocIDs[i]
	})

	// Function to create batches of 5
	createDocBatches := func(docs []string, batchSize int) [][]string {
		var batches [][]string
		for i := 0; i < len(docs); i += batchSize {
			end := i + batchSize
			if end > len(docs) {
				end = len(docs)
			}
			batches = append(batches, docs[i:end])
		}
		return batches
	}
	// Create batches of 5 documents
	batchSize := 5
	docBatches := createDocBatches(combinedDocIDs, batchSize)
	if len(docBatches) == 0 {
		t.Fatal("expected batches")
	}
	totalDocs := 0
	for _, batch := range docBatches {
		totalDocs += len(batch)
	}
	if totalDocs != len(combinedDocIDs) {
		t.Fatalf("expected %d documents, got %d", len(combinedDocIDs), totalDocs)
	}

	var batches []*Batch
	for _, docBatch := range docBatches {
		batch := idx.NewBatch()
		for _, docID := range docBatch {
			if synDef, ok := synonymDocuments[docID]; ok {
				err := batch.IndexSynonym(docID, synonymCollection, synDef)
				if err != nil {
					t.Fatal(err)
				}
			} else {
				err := batch.Index(docID, documents[docID])
				if err != nil {
					t.Fatal(err)
				}
			}
		}
		batches = append(batches, batch)
	}
	for _, batch := range batches {
		err = idx.Batch(batch)
		if err != nil {
			t.Fatal(err)
		}
	}

	type testStruct struct {
		query      string
		expectHits []string
	}

	testQueries := []testStruct{
		{
			query: `{
				"match": "hardworking employee",
				"field": "text"
			}`,
			expectHits: []string{"doc1", "doc13", "doc16", "doc4", "doc7"},
		},
		{
			query: `{
				"match": "Hardwork and industrius efforts bring lovely and tranqual moments, with a glazing blow of valour.",
				"field": "text",
				"fuzziness": "auto"
			}`,
			expectHits: []string{
				"doc1", "doc13", "doc14", "doc15", "doc16",
				"doc17", "doc18", "doc2", "doc20", "doc3",
				"doc4", "doc5", "doc6", "doc7", "doc8", "doc9",
			},
		},
		{
			query: `{
				"prefix": "in",
				"field": "text"
			}`,
			expectHits: []string{
				"doc1", "doc11", "doc13", "doc15", "doc16",
				"doc17", "doc18", "doc19", "doc2", "doc20",
				"doc3", "doc4", "doc7", "doc8",
			},
		},
		{
			query: `{
				"prefix": "vivid",
				"field": "text"
			}`,
			expectHits: []string{
				"doc17", "doc5",
			},
		},
		{
			query: `{
				"match_phrase": "smart academic",
				"field": "text"
			}`,
			expectHits: []string{"doc16", "doc18", "doc19"},
		},
		{
			query: `{
				"match_phrase": "smrat acedemic",
				"field": "text",
				"fuzziness": "auto"
			}`,
			expectHits: []string{"doc16", "doc18", "doc19"},
		},
		{
			query: `{
				"wildcard": "br*",
				"field": "text"
			}`,
			expectHits: []string{"doc11", "doc14", "doc16", "doc18", "doc19", "doc6", "doc8"},
		},
	}

	getTotalSynonymSearchStat := func(idx Index) int {
		ir, err := idx.Advanced()
		if err != nil {
			t.Fatal(err)
		}
		stat := ir.StatsMap()["synonym_searches"].(uint64)
		return int(stat)
	}

	runTestQueries := func(idx Index) error {
		for _, dtq := range testQueries {
			q, err := query.ParseQuery([]byte(dtq.query))
			if err != nil {
				return err
			}
			sr := NewSearchRequest(q)
			sr.Highlight = NewHighlightWithStyle(ansi.Name)
			sr.SortBy([]string{"_id"})
			sr.Fields = []string{"*"}
			sr.Size = 30
			sr.Explain = true
			res, err := idx.Search(sr)
			if err != nil {
				return err
			}
			if len(res.Hits) != len(dtq.expectHits) {
				return fmt.Errorf("expected %d hits, got %d", len(dtq.expectHits), len(res.Hits))
			}
			// sort the expected hits to match the order of the search results
			sort.Strings(dtq.expectHits)
			for i, hit := range res.Hits {
				if hit.ID != dtq.expectHits[i] {
					return fmt.Errorf("expected docID %s, got %s", dtq.expectHits[i], hit.ID)
				}
			}
		}
		return nil
	}
	err = runTestQueries(idx)
	if err != nil {
		t.Fatal(err)
	}
	// now verify that the stat for number of synonym enabled queries is correct
	totalSynonymSearchStat := getTotalSynonymSearchStat(idx)
	if totalSynonymSearchStat != len(testQueries) {
		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
	}

	// test with index alias - with 1 batch per index
	numIndexes := len(batches)
	indexes := make([]Index, numIndexes)
	indexesPath := make([]string, numIndexes)
	for i := 0; i < numIndexes; i++ {
		tmpIndexPath := createTmpIndexPath(t)
		idx, err := New(tmpIndexPath, imap)
		if err != nil {
			t.Fatal(err)
		}
		err = idx.Batch(batches[i])
		if err != nil {
			t.Fatal(err)
		}
		indexes[i] = idx
		indexesPath[i] = tmpIndexPath
	}
	defer func() {
		for i := 0; i < numIndexes; i++ {
			err = indexes[i].Close()
			if err != nil {
				t.Fatal(err)
			}

			cleanupTmpIndexPath(t, indexesPath[i])
		}
	}()
	alias := NewIndexAlias(indexes...)

	if err := alias.SetIndexMapping(imap); err != nil {
		t.Fatal(err)
	}

	err = runTestQueries(alias)
	if err != nil {
		t.Fatal(err)
	}
	// verify the synonym search stat for the alias
	totalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])
	if totalSynonymSearchStat != len(testQueries) {
		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
	}
	for i := 1; i < numIndexes; i++ {
		idxStat := getTotalSynonymSearchStat(indexes[i])
		if idxStat != totalSynonymSearchStat {
			t.Fatalf("expected %d synonym searches, got %d", totalSynonymSearchStat, idxStat)
		}
	}
	if totalSynonymSearchStat != len(testQueries) {
		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
	}
	// test with multi-level alias now with two index per alias
	// and having any extra index being in the final alias
	numAliases := numIndexes / 2
	extraIndex := numIndexes % 2
	aliases := make([]IndexAlias, numAliases)
	for i := 0; i < numAliases; i++ {
		alias := NewIndexAlias(indexes[i*2], indexes[i*2+1])
		aliases[i] = alias
	}
	if extraIndex > 0 {
		aliases[numAliases-1].Add(indexes[numIndexes-1])
	}
	alias = NewIndexAlias()

	if err := alias.SetIndexMapping(imap); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < numAliases; i++ {
		alias.Add(aliases[i])
	}
	err = runTestQueries(alias)
	if err != nil {
		t.Fatal(err)
	}
	// verify the synonym searches stat for the alias
	totalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])
	if totalSynonymSearchStat != 2*len(testQueries) {
		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
	}
	for i := 1; i < numIndexes; i++ {
		idxStat := getTotalSynonymSearchStat(indexes[i])
		if idxStat != totalSynonymSearchStat {
			t.Fatalf("expected %d synonym searches, got %d", totalSynonymSearchStat, idxStat)
		}
	}
}

func TestGeoDistanceInSort(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	fm := mapping.NewGeoPointFieldMapping()
	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.AddFieldMappingsAt("geo", fm)

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	qp := []float64{0, 0}

	docs := []struct {
		id       string
		point    []float64
		distance float64
	}{
		{
			id:       "1",
			point:    []float64{1, 1},
			distance: geo.Haversin(1, 1, qp[0], qp[1]) * 1000,
		},
		{
			id:       "2",
			point:    []float64{2, 2},
			distance: geo.Haversin(2, 2, qp[0], qp[1]) * 1000,
		},
		{
			id:       "3",
			point:    []float64{3, 3},
			distance: geo.Haversin(3, 3, qp[0], qp[1]) * 1000,
		},
	}

	for _, doc := range docs {
		if err := idx.Index(doc.id, map[string]interface{}{"geo": doc.point}); err != nil {
			t.Fatal(err)
		}
	}

	q := NewGeoDistanceQuery(qp[0], qp[1], "1000000m")
	q.SetField("geo")
	req := NewSearchRequest(q)
	req.Sort = make(search.SortOrder, 0)
	req.Sort = append(req.Sort, &search.SortGeoDistance{
		Field: "geo",
		Desc:  false,
		Unit:  "m",
		Lon:   qp[0],
		Lat:   qp[1],
	})
	res, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	for i, doc := range res.Hits {
		hitDist, err := strconv.ParseFloat(doc.DecodedSort[0], 64)
		if err != nil {
			t.Fatal(err)
		}
		if math.Abs(hitDist-docs[i].distance) > 1 {
			t.Fatalf("distance error greater than 1 meter, expected distance - %v, got - %v", docs[i].distance, hitDist)
		}
	}
}

func TestFilteredBooleanQuery(t *testing.T) {
	tmpIndexPath := createTmpIndexPath(t)

	imap := mapping.NewIndexMapping()

	genreMapping := mapping.NewTextFieldMapping()
	genreMapping.Analyzer = keyword.Name

	authorMapping := mapping.NewTextFieldMapping()
	authorMapping.Analyzer = keyword.Name

	titleMapping := mapping.NewTextFieldMapping()
	titleMapping.Analyzer = en.AnalyzerName

	tagsMapping := mapping.NewNumericFieldMapping()
	tagsMapping.Store = false
	tagsMapping.IncludeInAll = false

	priceMapping := mapping.NewNumericFieldMapping()
	imap.DefaultMapping.AddFieldMappingsAt("genre", genreMapping)
	imap.DefaultMapping.AddFieldMappingsAt("author", authorMapping)
	imap.DefaultMapping.AddFieldMappingsAt("title", titleMapping)
	imap.DefaultMapping.AddFieldMappingsAt("price", priceMapping)
	imap.DefaultMapping.AddFieldMappingsAt("tags", tagsMapping)

	idx, err := New(tmpIndexPath, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx.Close()
		if err != nil {
			t.Fatal(err)
		}
		err := os.RemoveAll(tmpIndexPath)
		if err != nil {
			t.Fatal(err)
		}
	}()

	// Book dataset
	var docs = []map[string]interface{}{
		{
			"title":  "The Catcher in the Rye",
			"author": "J.D. Salinger",
			"genre":  "fiction",
			"price":  9.99,
			"tags":   []int{1, 2, 3},
		},
		{
			"title":  "Sapiens",
			"author": "Yuval Noah Harari",
			"genre":  "non-fiction",
			"price":  14.29,
			"tags":   []int{2},
		},
		{
			"title":  "To Kill a Mockingbird",
			"author": "Harper Lee",
			"genre":  "fiction",
			"price":  12,
			"tags":   []int{},
		},
		{
			"title":  "The Power of Habit",
			"author": "Charles Duhigg",
			"genre":  "self-help",
			"price":  26,
			"tags":   []int{1, 2},
		},
		{
			"title":  "The Great Gatsby",
			"author": "F. Scott Fitzgerald",
			"genre":  "fiction",
			"price":  22,
			"tags":   []int{1, 2},
		},
		{
			"title":  "Atomic Habits",
			"author": "James Clear",
			"genre":  "self-help",
			"price":  15,
			"tags":   []int{3},
		},
		{
			"title":  "Educated",
			"author": "Tara Westover",
			"genre":  "non-fiction",
			"price":  18,
		},
		{
			"title":  "1984",
			"author": "George Orwell",
			"genre":  "fiction",
			"price":  20,
		},
	}

	b := idx.NewBatch()
	for i, doc := range docs {
		err := b.Index(strconv.Itoa(i), doc)
		if err != nil {
			t.Fatal(err)
		}
	}
	// execute the batch
	err = idx.Batch(b)
	if err != nil {
		t.Fatal(err)
	}

	// Suppose the user is interested in books in the fiction genre
	// and is only interested in books within their budget of 20
	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	// A numeric range query for books with a price less than or equal to 20
	max := float64(20)
	maxInclusive := true
	priceFilterQuery := NewNumericRangeQuery(nil, &max)
	priceFilterQuery.InclusiveMax = &maxInclusive
	priceFilterQuery.SetField("price")

	// An unfiltered boolean query requesting all books in the fiction genre
	// All 4 books in the fiction genre should be returned with the
	// same score as they are all in the same genre
	q := NewBooleanQuery()
	q.AddMust(fictionQuery)

	req := NewSearchRequest(q)
	req.Explain = true
	req.Fields = []string{"title"}
	// sort by book titles in ascending order
	req.Sort = make(search.SortOrder, 0)
	titleSort := &search.SortField{
		Field: "price",
		Desc:  false,
	}
	req.Sort = append(req.Sort, titleSort)

	res, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) != 4 {
		t.Fatalf("expected 4 hits, got %d", len(res.Hits))
	}
	// Verify the results are in the correct order
	expectedTitleOrder := []string{
		"The Catcher in the Rye",
		"To Kill a Mockingbird",
		"1984",
		"The Great Gatsby",
	}
	for i, doc := range res.Hits {
		if doc.Fields["title"] != expectedTitleOrder[i] {
			t.Fatalf("expected title %s, got %s", expectedTitleOrder[i], doc.Fields["title"])
		}
	}
	// Ensure that the scores are the same for all documents
	unfilteredScore := res.Hits[0].Score
	for i := 1; i < len(res.Hits); i++ {
		if res.Hits[i].Score != unfilteredScore {
			t.Fatalf("expected score %f, got %f", unfilteredScore, res.Hits[i].Score)
		}
	}
	// A filtered boolean query requesting all books satisfying the
	// filterQuery and the priceFilterQuery
	// But the filter query is in the Must clause
	q = NewBooleanQuery()
	q.AddMust(fictionQuery)
	q.AddMust(priceFilterQuery)
	req = NewSearchRequest(q)
	req.Explain = true
	req.Fields = []string{"title"}
	// sort by book titles in ascending order
	req.Sort = make(search.SortOrder, 0)
	req.Sort = append(req.Sort, titleSort)

	res, err = idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}
	// here the score must not be the same for all documents
	// as the price filter is applied in the Must clause
	// and the score is different compared to the previous unfiltered boolean query
	if len(res.Hits) != 3 {
		t.Fatalf("expected 3 hits, got %d", len(res.Hits))
	}
	// Verify the results are in the correct order
	expectedTitleOrder = []string{
		"The Catcher in the Rye",
		"To Kill a Mockingbird",
		"1984",
	}
	for i, doc := range res.Hits {
		if doc.Fields["title"] != expectedTitleOrder[i] {
			t.Fatalf("expected title %s, got %s", expectedTitleOrder[i], doc.Fields["title"])
		}
	}
	// Ensure that the scores are different for all documents
	for i := 0; i < len(res.Hits); i++ {
		if res.Hits[i].Score == unfilteredScore {
			t.Fatalf("expected different score, got %f", res.Hits[i].Score)
		}
	}
	// A filtered boolean query requesting all books satisfying the
	// filterQuery and the priceFilterQuery
	// But the filter query is in the Filter clause
	q = NewBooleanQuery()
	q.AddMust(fictionQuery)
	q.AddFilter(priceFilterQuery)

	req = NewSearchRequest(q)
	req.Explain = true
	req.Fields = []string{"title"}
	req.Sort = make(search.SortOrder, 0)
	req.Sort = append(req.Sort, titleSort)
	res, err = idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}
	if len(res.Hits) != 3 {
		t.Fatalf("expected 3 hits, got %d", len(res.Hits))
	}
	// Verify the results are in the correct order
	for i, doc := range res.Hits {
		if doc.Fields["title"] != expectedTitleOrder[i] {
			t.Fatalf("expected title %s, got %s", expectedTitleOrder[i], doc.Fields["title"])
		}
	}
	// Ensure that the scores are the same for all documents
	for i := 0; i < len(res.Hits); i++ {
		if res.Hits[i].Score != unfilteredScore {
			t.Fatalf("expected score %f, got %f", unfilteredScore, res.Hits[i].Score)
		}
	}
	// A filtered boolean query requesting all books with tag value 3
	// The filter is in the Filter clause
	// Two books have tag value 3
	p := float64(3)
	incl := true
	eqlFilter := NewNumericRangeInclusiveQuery(&p, &p, &incl, &incl)
	eqlFilter.SetField("tags")
	q = NewBooleanQuery()
	q.AddFilter(eqlFilter)

	req = NewSearchRequest(q)
	req.Fields = []string{"title"}
	res, err = idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}
	if res.Hits.Len() != 2 {
		t.Fatalf("expected two hits, found '%d'", res.Hits.Len())
	}
	if res.Total != 2 {
		t.Fatalf("expected two total, found '%d'", res.Total)
	}
}

func TestCustomFilterQuery(t *testing.T) {
	idx := newCustomQueryTestIndex(t)
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	allowedIDs := map[string]struct{}{
		"0": {},
		"2": {},
		"7": {},
	}

	q := query.NewCustomFilterQueryWithFilter(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (bool, error) {
		_, ok := allowedIDs[d.ID]
		return ok, nil
	}, nil, nil)

	req := NewSearchRequest(q)
	req.Fields = []string{"title"}
	req.Size = 10

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 3 {
		t.Fatalf("expected 3 hits, got %d", res.Total)
	}

	hitIDs := map[string]struct{}{}
	for _, hit := range res.Hits {
		hitIDs[hit.ID] = struct{}{}
	}
	for id := range allowedIDs {
		if _, ok := hitIDs[id]; !ok {
			t.Fatalf("missing expected hit id %s", id)
		}
	}
}

func TestCustomScoreQuery(t *testing.T) {
	idx := newCustomQueryTestIndex(t)
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	boosts := map[string]float64{
		"7": 100.0,
		"2": 10.0,
		"0": 1.0,
	}

	q := query.NewCustomScoreQueryWithScorer(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (float64, error) {
		return d.Score + boosts[d.ID], nil
	}, nil, nil)

	req := NewSearchRequest(q)
	req.Fields = []string{"title"}
	req.Size = 4

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) != 4 {
		t.Fatalf("expected 4 hits, got %d", len(res.Hits))
	}

	expectedOrder := []string{"7", "2", "0", "4"}
	for i, hit := range res.Hits {
		if hit.ID != expectedOrder[i] {
			t.Fatalf("expected hit %d to be %s, got %s", i, expectedOrder[i], hit.ID)
		}
	}
}

func TestCustomFilterQueryDocumentMatchIDWithoutFields(t *testing.T) {
	idx := newCustomQueryTestIndex(t)
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	allowedIDs := map[string]struct{}{
		"0": {},
		"2": {},
		"7": {},
	}

	q := query.NewCustomFilterQueryWithFilter(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (bool, error) {
		_, ok := allowedIDs[d.ID]
		return ok, nil
	}, nil, nil)

	req := NewSearchRequest(q)
	req.Size = 10

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	if res.Total != 3 {
		t.Fatalf("expected 3 hits, got %d", res.Total)
	}

	hitIDs := map[string]struct{}{}
	for _, hit := range res.Hits {
		hitIDs[hit.ID] = struct{}{}
	}
	for id := range allowedIDs {
		if _, ok := hitIDs[id]; !ok {
			t.Fatalf("missing expected hit id %s", id)
		}
	}
}

func TestCustomScoreQueryWithDocValues(t *testing.T) {
	idx := newCustomQueryDocValueTestIndex(t)
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	q := query.NewCustomScoreQueryWithScorer(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (float64, error) {
		rating, ok := d.Fields["rating"].(float64)
		if ok && rating >= 9 {
			return d.Score + 100, nil
		}
		return d.Score, nil
	}, []string{"rating"}, nil)

	req := NewSearchRequest(q)
	req.Size = 4

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) != 2 {
		t.Fatalf("expected 2 hits, got %d", len(res.Hits))
	}
	if res.Hits[0].ID != "1" {
		t.Fatalf("expected boosted hit 1 first, got %s", res.Hits[0].ID)
	}
}

func TestCustomScoreQueryExplain(t *testing.T) {
	idx := newCustomQueryTestIndex(t)
	defer func() {
		if err := idx.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	titleQuery := NewMatchQuery("habit")
	titleQuery.SetField("title")

	q := query.NewCustomScoreQueryWithScorer(titleQuery, func(ctx context.Context, d *search.DocumentMatch) (float64, error) {
		return d.Score * 2, nil
	}, nil, nil)

	req := NewSearchRequest(q)
	req.Explain = true
	req.Size = 10

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) == 0 {
		t.Fatal("expected at least one hit")
	}

	for _, hit := range res.Hits {
		if hit.Expl == nil {
			t.Fatalf("expected explanation on hit %s but got nil", hit.ID)
		}
		if hit.Expl.Message != "custom_score function result" {
			t.Errorf("hit %s: expected wrap message %q, got %q",
				hit.ID, "custom_score function result", hit.Expl.Message)
		}
		if len(hit.Expl.Children) != 0 {
			t.Errorf("hit %s: expected no children (inner score dropped), got %d",
				hit.ID, len(hit.Expl.Children))
		}
		t.Logf("hit %s (score=%f):\n%s", hit.ID, hit.Score, hit.Expl)
	}
}

func newCustomQueryTestIndex(t *testing.T) Index {
	t.Helper()

	imap := mapping.NewIndexMapping()

	genreMapping := mapping.NewTextFieldMapping()
	genreMapping.Analyzer = keyword.Name
	imap.DefaultMapping.AddFieldMappingsAt("genre", genreMapping)

	titleMapping := mapping.NewTextFieldMapping()
	titleMapping.Analyzer = en.AnalyzerName
	imap.DefaultMapping.AddFieldMappingsAt("title", titleMapping)

	tmpIndexPath := createTmpIndexPath(t)
	t.Cleanup(func() { cleanupTmpIndexPath(t, tmpIndexPath) })

	idx, err := NewUsing(tmpIndexPath, imap, scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	docs := []map[string]interface{}{
		{"title": "The Catcher in the Rye", "genre": "fiction"},
		{"title": "Sapiens", "genre": "non-fiction"},
		{"title": "To Kill a Mockingbird", "genre": "fiction"},
		{"title": "The Power of Habit", "genre": "self-help"},
		{"title": "The Great Gatsby", "genre": "fiction"},
		{"title": "Atomic Habits", "genre": "self-help"},
		{"title": "Educated", "genre": "non-fiction"},
		{"title": "1984", "genre": "fiction"},
	}

	b := idx.NewBatch()
	for i, doc := range docs {
		err := b.Index(strconv.Itoa(i), doc)
		if err != nil {
			t.Fatal(err)
		}
	}

	err = idx.Batch(b)
	if err != nil {
		t.Fatal(err)
	}

	return idx
}

func newCustomQueryDocValueTestIndex(t *testing.T) Index {
	t.Helper()

	imap := mapping.NewIndexMapping()

	genreMapping := mapping.NewTextFieldMapping()
	genreMapping.Analyzer = keyword.Name
	imap.DefaultMapping.AddFieldMappingsAt("genre", genreMapping)

	ratingMapping := mapping.NewNumericFieldMapping()
	imap.DefaultMapping.AddFieldMappingsAt("rating", ratingMapping)

	tmpIndexPath := createTmpIndexPath(t)
	t.Cleanup(func() { cleanupTmpIndexPath(t, tmpIndexPath) })

	idx, err := NewUsing(tmpIndexPath, imap, scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	docs := []map[string]interface{}{
		{"genre": "fiction", "rating": 5},
		{"genre": "fiction", "rating": 9},
	}

	b := idx.NewBatch()
	for i, doc := range docs {
		err := b.Index(strconv.Itoa(i), doc)
		if err != nil {
			t.Fatal(err)
		}
	}

	err = idx.Batch(b)
	if err != nil {
		t.Fatal(err)
	}

	return idx
}

func TestCustomFilterQueryDateTimeDocValues(t *testing.T) {
	imap := mapping.NewIndexMapping()

	genreMapping := mapping.NewTextFieldMapping()
	genreMapping.Analyzer = keyword.Name
	imap.DefaultMapping.AddFieldMappingsAt("genre", genreMapping)

	dateMapping := mapping.NewDateTimeFieldMapping()
	imap.DefaultMapping.AddFieldMappingsAt("published", dateMapping)

	tmpIndexPath := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath)

	idx, err := NewUsing(tmpIndexPath, imap, scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	cutoff := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)

	docs := []map[string]interface{}{
		{"genre": "fiction", "published": "2019-06-15T00:00:00Z"},
		{"genre": "fiction", "published": "2022-03-10T00:00:00Z"},
		{"genre": "fiction", "published": "2018-01-01T00:00:00Z"},
	}

	b := idx.NewBatch()
	for i, doc := range docs {
		if err := b.Index(strconv.Itoa(i), doc); err != nil {
			t.Fatal(err)
		}
	}
	if err := idx.Batch(b); err != nil {
		t.Fatal(err)
	}

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	// Datetime doc values are decoded as RFC 3339 strings.
	cutoffStr := cutoff.UTC().Format(time.RFC3339Nano)

	q := query.NewCustomFilterQueryWithFilter(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (bool, error) {
		pubStr, ok := d.Fields["published"].(string)
		if !ok {
			return false, nil
		}
		return pubStr >= cutoffStr, nil
	}, []string{"published"}, nil)

	req := NewSearchRequest(q)
	req.Size = 10

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	// Only doc "1" (2022-03-10) is >= 2020-01-01.
	if res.Total != 1 {
		t.Fatalf("expected 1 hit, got %d", res.Total)
	}
	if res.Hits[0].ID != "1" {
		t.Fatalf("expected hit id 1, got %s", res.Hits[0].ID)
	}
}

// TestCustomFilterQueryWildcardFields verifies that fields:["*"] inside a
// custom_filter query exposes all doc-value fields to the callback. The
// doc-value reader matches field names literally, so before the wildcard was
// expanded "*" matched nothing, d.Fields stayed empty, and every candidate was
// filtered out (total_hits == 0).
func TestCustomFilterQueryWildcardFields(t *testing.T) {
	idx := newCustomQueryDocValueTestIndex(t)
	defer func() {
		if err := idx.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	// The callback inspects multiple fields and keeps the hit only if all are
	// present, so with "*" the reader must surface both the numeric "rating"
	// and the keyword "genre" (and resolve their types correctly).
	q := query.NewCustomFilterQueryWithFilter(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (bool, error) {
		rating, hasRating := d.Fields["rating"].(float64)
		genre, hasGenre := d.Fields["genre"].(string)
		return hasRating && hasGenre && genre == "fiction" && rating > 0, nil
	}, []string{"*"}, nil)

	req := NewSearchRequest(q)
	req.Size = 10

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	// Both fiction docs carry rating + genre, so both must pass the filter.
	if res.Total != 2 {
		t.Fatalf("expected 2 hits with fields:[\"*\"], got %d (doc.Fields likely empty)", res.Total)
	}
}

// TestCustomScoreQueryWildcardFields is the custom_score counterpart: fields:["*"]
// must expose doc-value fields to the scorer just like an explicit field list.
func TestCustomScoreQueryWildcardFields(t *testing.T) {
	idx := newCustomQueryDocValueTestIndex(t)
	defer func() {
		if err := idx.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	fictionQuery := NewTermQuery("fiction")
	fictionQuery.SetField("genre")

	q := query.NewCustomScoreQueryWithScorer(fictionQuery, func(ctx context.Context, d *search.DocumentMatch) (float64, error) {
		if rating, ok := d.Fields["rating"].(float64); ok && rating >= 9 {
			return d.Score + 100, nil
		}
		return d.Score, nil
	}, []string{"*"}, nil)

	req := NewSearchRequest(q)
	req.Size = 4

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}

	if len(res.Hits) != 2 {
		t.Fatalf("expected 2 hits, got %d", len(res.Hits))
	}
	// Doc "1" (rating 9) is boosted by +100; the top hit must be doc "1" AND
	// carry that boost in its score. Without "*" expansion d.Fields["rating"]
	// is absent, the boost never applies, and the score stays below 1 — so the
	// magnitude check (not just ordering) is what proves the field was read.
	if res.Hits[0].ID != "1" {
		t.Fatalf("expected boosted hit 1 first, got %s", res.Hits[0].ID)
	}
	if res.Hits[0].Score < 100 {
		t.Fatalf("expected top hit score >= 100 from boost, got %f (doc.Fields likely empty)", res.Hits[0].Score)
	}
}

// newCustomQueryFieldLeakTestIndex builds an index where "name" is stored (so
// it can be returned via SearchRequest.Fields) and "price" is a doc-value-only
// field used as UDF input. It is the fixture for the field-leak regression
// tests below.
func newCustomQueryFieldLeakTestIndex(t *testing.T) Index {
	t.Helper()

	imap := mapping.NewIndexMapping()

	nameMapping := mapping.NewTextFieldMapping()
	nameMapping.Analyzer = keyword.Name
	nameMapping.Store = true
	imap.DefaultMapping.AddFieldMappingsAt("name", nameMapping)

	priceMapping := mapping.NewNumericFieldMapping()
	imap.DefaultMapping.AddFieldMappingsAt("price", priceMapping)

	tmpIndexPath := createTmpIndexPath(t)
	t.Cleanup(func() { cleanupTmpIndexPath(t, tmpIndexPath) })

	idx, err := NewUsing(tmpIndexPath, imap, scorch.Name, Config.DefaultKVStore, nil)
	if err != nil {
		t.Fatal(err)
	}

	b := idx.NewBatch()
	if err := b.Index("h1", map[string]interface{}{"name": "hotel paris", "price": 150.0}); err != nil {
		t.Fatal(err)
	}
	if err := idx.Batch(b); err != nil {
		t.Fatal(err)
	}

	return idx
}

// TestCustomFilterQueryFieldsDoNotLeakIntoResponse is a regression test: a
// custom_filter's fields list is UDF input only and must not override
// SearchRequest.Fields. Before the fix the UDF-internal field ("price") leaked
// into the response and the client's requested field ("name") went missing.
func TestCustomFilterQueryFieldsDoNotLeakIntoResponse(t *testing.T) {
	idx := newCustomQueryFieldLeakTestIndex(t)
	defer func() {
		if err := idx.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	nameQuery := NewTermQuery("hotel paris")
	nameQuery.SetField("name")

	q := query.NewCustomFilterQueryWithFilter(nameQuery, func(ctx context.Context, d *search.DocumentMatch) (bool, error) {
		price, ok := d.Fields["price"].(float64)
		return ok && price >= 100, nil
	}, []string{"price"}, nil)

	req := NewSearchRequest(q)
	req.Size = 10
	req.Fields = []string{"name"} // client asks only for "name"

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}
	if len(res.Hits) != 1 {
		t.Fatalf("expected 1 hit, got %d", len(res.Hits))
	}
	hit := res.Hits[0]
	if _, ok := hit.Fields["name"]; !ok {
		t.Fatalf("expected requested field \"name\" in response, got fields: %v", hit.Fields)
	}
	if _, ok := hit.Fields["price"]; ok {
		t.Fatalf("UDF-internal field \"price\" leaked into response fields: %v", hit.Fields)
	}
}

// TestCustomScoreQueryFieldsDoNotLeakIntoResponse is the custom_score counterpart.
func TestCustomScoreQueryFieldsDoNotLeakIntoResponse(t *testing.T) {
	idx := newCustomQueryFieldLeakTestIndex(t)
	defer func() {
		if err := idx.Close(); err != nil {
			t.Fatal(err)
		}
	}()

	nameQuery := NewTermQuery("hotel paris")
	nameQuery.SetField("name")

	q := query.NewCustomScoreQueryWithScorer(nameQuery, func(ctx context.Context, d *search.DocumentMatch) (float64, error) {
		if price, ok := d.Fields["price"].(float64); ok && price >= 100 {
			return d.Score + 1, nil
		}
		return d.Score, nil
	}, []string{"price"}, nil)

	req := NewSearchRequest(q)
	req.Size = 10
	req.Fields = []string{"name"}

	res, err := idx.SearchInContext(context.Background(), req)
	if err != nil {
		t.Fatal(err)
	}
	if len(res.Hits) != 1 {
		t.Fatalf("expected 1 hit, got %d", len(res.Hits))
	}
	hit := res.Hits[0]
	if _, ok := hit.Fields["name"]; !ok {
		t.Fatalf("expected requested field \"name\" in response, got fields: %v", hit.Fields)
	}
	if _, ok := hit.Fields["price"]; ok {
		t.Fatalf("UDF-internal field \"price\" leaked into response fields: %v", hit.Fields)
	}
}

func TestGeoDistanceInSortAlias(t *testing.T) {
	tmpIndexPath1 := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath1)

	tmpIndexPath2 := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath2)

	fm := mapping.NewGeoPointFieldMapping()
	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.AddFieldMappingsAt("geo", fm)

	idx1, err := New(tmpIndexPath1, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx1.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	idx2, err := New(tmpIndexPath2, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx2.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	qp := []float64{0, 0}

	docs := []struct {
		id       string
		point    []float64
		distance float64
	}{
		{
			id:       "1",
			point:    []float64{1, 1},
			distance: geo.Haversin(1, 1, qp[0], qp[1]) * 1000,
		},
		{
			id:       "2",
			point:    []float64{2, 2},
			distance: geo.Haversin(2, 2, qp[0], qp[1]) * 1000,
		},
		{
			id:       "3",
			point:    []float64{3, 3},
			distance: geo.Haversin(3, 3, qp[0], qp[1]) * 1000,
		},
	}
	if err := idx1.Index(docs[0].id, map[string]interface{}{"geo": docs[0].point}); err != nil {
		t.Fatal(err)
	}
	if err := idx2.Index(docs[1].id, map[string]interface{}{"geo": docs[1].point}); err != nil {
		t.Fatal(err)
	}
	if err := idx1.Index(docs[2].id, map[string]interface{}{"geo": docs[2].point}); err != nil {
		t.Fatal(err)
	}

	idx := NewIndexAlias(idx1, idx2)

	q := NewGeoDistanceQuery(qp[0], qp[1], "1000000m")
	q.SetField("geo")
	req := NewSearchRequest(q)
	req.Sort = make(search.SortOrder, 0)
	req.Sort = append(req.Sort, &search.SortGeoDistance{
		Field: "geo",
		Lon:   qp[0],
		Lat:   qp[1],
	})
	res, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	for i, doc := range res.Hits {
		hitDist, err := strconv.ParseFloat(doc.DecodedSort[0], 64)
		if err != nil {
			t.Fatal(err)
		}
		if math.Abs(hitDist-docs[i].distance) > 1 {
			t.Fatalf("distance error greater than 1 meter, expected distance - %v, got - %v", docs[i].distance, hitDist)
		}
	}
}

func TestDateSortAlias(t *testing.T) {
	tmpIndexPath1 := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath1)

	tmpIndexPath2 := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath2)

	fm := mapping.NewDateTimeFieldMapping()
	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.AddFieldMappingsAt("date", fm)

	idx1, err := New(tmpIndexPath1, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx1.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	idx2, err := New(tmpIndexPath2, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx2.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	docs := []struct {
		id   string
		date string
	}{
		{
			id:   "1",
			date: "2023-01-01",
		},
		{
			id:   "2",
			date: "2023-02-01",
		},
		{
			id:   "3",
			date: "2023-03-01",
		},
	}

	if err := idx1.Index(docs[0].id, map[string]interface{}{"date": docs[0].date}); err != nil {
		t.Fatal(err)
	}
	if err := idx2.Index(docs[1].id, map[string]interface{}{"date": docs[1].date}); err != nil {
		t.Fatal(err)
	}
	if err := idx1.Index(docs[2].id, map[string]interface{}{"date": docs[2].date}); err != nil {
		t.Fatal(err)
	}

	idx := NewIndexAlias(idx1, idx2)

	q := query.NewMatchAllQuery()
	req := NewSearchRequest(q)
	req.Sort = make(search.SortOrder, 0)
	req.Sort = append(req.Sort, &search.SortField{
		Field: "date",
		Type:  search.SortFieldAsDate,
	})
	res, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	for i, doc := range res.Hits {
		expectedDate, err := time.Parse("2006-01-02", docs[i].date)
		if err != nil {
			t.Fatal(err)
		}
		expectedDateStr := expectedDate.UTC().Format(time.RFC3339Nano)
		if doc.DecodedSort[0] != expectedDateStr {
			t.Fatalf("expected date %s, got %s", doc.DecodedSort[0], expectedDateStr)
		}
	}
}

func TestNumericSortAlias(t *testing.T) {
	tmpIndexPath1 := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath1)

	tmpIndexPath2 := createTmpIndexPath(t)
	defer cleanupTmpIndexPath(t, tmpIndexPath2)

	fm := mapping.NewNumericFieldMapping()
	imap := mapping.NewIndexMapping()
	imap.DefaultMapping.AddFieldMappingsAt("num", fm)

	idx1, err := New(tmpIndexPath1, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx1.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	idx2, err := New(tmpIndexPath2, imap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = idx2.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	docs := []struct {
		id  string
		num int
	}{
		{
			id:  "1",
			num: 10,
		},
		{
			id:  "2",
			num: 20,
		},
		{
			id:  "3",
			num: 30,
		},
	}

	if err := idx1.Index(docs[0].id, map[string]interface{}{"num": docs[0].num}); err != nil {
		t.Fatal(err)
	}
	if err := idx2.Index(docs[1].id, map[string]interface{}{"num": docs[1].num}); err != nil {
		t.Fatal(err)
	}
	if err := idx1.Index(docs[2].id, map[string]interface{}{"num": docs[2].num}); err != nil {
		t.Fatal(err)
	}

	idx := NewIndexAlias(idx1, idx2)

	q := query.NewMatchAllQuery()
	req := NewSearchRequest(q)
	req.Sort = make(search.SortOrder, 0)
	req.Sort = append(req.Sort, &search.SortField{
		Field: "num",
		Type:  search.SortFieldAsNumber,
	})
	res, err := idx.Search(req)
	if err != nil {
		t.Fatal(err)
	}

	for i, doc := range res.Hits {
		hitNum, err := strconv.Atoi(doc.DecodedSort[0])
		if err != nil {
			t.Fatal(err)
		}
		if hitNum != docs[i].num {
			t.Fatalf("expected num %d, got %d", docs[i].num, hitNum)
		}
	}
}

func TestSearchRequestValidatePagination(t *testing.T) {
	tests := []struct {
		name      string
		req       *SearchRequest
		expectErr error
	}{
		{
			name: "invalid search after with numeric sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "num", Type: search.SortFieldAsNumber},
				},
				SearchAfter: []string{"not-a-number"},
			},
			expectErr: fmt.Errorf("invalid search after value for sort field 'num': 'not-a-number'. strconv.ParseFloat: parsing \"not-a-number\": invalid syntax"),
		},
		{
			name: "invalid search before with numeric sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "num", Type: search.SortFieldAsNumber},
				},
				SearchBefore: []string{"not-a-number"},
			},
			expectErr: fmt.Errorf("invalid search before value for sort field 'num': 'not-a-number'. strconv.ParseFloat: parsing \"not-a-number\": invalid syntax"),
		},
		{
			name: "invalid search after with date sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "date", Type: search.SortFieldAsDate},
				},
				SearchAfter: []string{"1 March 2023"},
			},
			expectErr: fmt.Errorf("invalid search after value for sort field 'date': '1 March 2023'. parsing time \"1 March 2023\" as \"2006-01-02T15:04:05.999999999Z07:00\": cannot parse \"1 March 2023\" as \"2006\""),
		},
		{
			name: "invalid search before with date sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "date", Type: search.SortFieldAsDate},
				},
				SearchBefore: []string{"1 March 2023"},
			},
			expectErr: fmt.Errorf("invalid search before value for sort field 'date': '1 March 2023'. parsing time \"1 March 2023\" as \"2006-01-02T15:04:05.999999999Z07:00\": cannot parse \"1 March 2023\" as \"2006\""),
		},
		{
			name: "invalid search after with geo distance sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortGeoDistance{Field: "geo"},
				},
				SearchAfter: []string{"not-a-number"},
			},
			expectErr: fmt.Errorf("invalid search after value for sort field 'geo': 'not-a-number'. strconv.ParseFloat: parsing \"not-a-number\": invalid syntax"),
		},
		{
			name: "invalid search before with geo distance sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortGeoDistance{Field: "geo"},
				},
				SearchBefore: []string{"not-a-number"},
			},
			expectErr: fmt.Errorf("invalid search before value for sort field 'geo': 'not-a-number'. strconv.ParseFloat: parsing \"not-a-number\": invalid syntax"),
		},
		{
			name: "valid search after with text sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "text", Type: search.SortFieldAsString},
				},
				SearchAfter: []string{"anything"},
			},
			expectErr: nil,
		},
		{
			name: "valid search after with numeric sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "num", Type: search.SortFieldAsNumber},
				},
				SearchAfter: []string{"50.5"},
			},
			expectErr: nil,
		},
		{
			name: "valid search after with date sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortField{Field: "date", Type: search.SortFieldAsDate},
				},
				SearchAfter: []string{time.Now().UTC().Format(time.RFC3339Nano)},
			},
			expectErr: nil,
		},
		{
			name: "valid search after with geo distance sort",
			req: &SearchRequest{
				Query: NewMatchAllQuery(),
				Sort: search.SortOrder{
					&search.SortGeoDistance{Field: "geo"},
				},
				SearchAfter: []string{"1.234"},
			},
			expectErr: nil,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			err := test.req.Validate()
			if test.expectErr != nil {
				if err == nil {
					t.Fatalf("expected error: %v, got nil", test.expectErr)
				}
				if err.Error() != test.expectErr.Error() {
					t.Fatalf("expected error: %v, got: %v", test.expectErr, err)
				}
			} else if err != nil {
				t.Fatalf("expected no error, got: %v", err)
			}
		})
	}
}