From fe80a9ce1197ddef9c2758cd9a474f1fbf023f63 Mon Sep 17 00:00:00 2001 From: Simon Pither Date: Tue, 7 Jun 2011 19:56:06 +0100 Subject: [PATCH 1/5] Attempt to force a pause to avoid the first mapping creation from throwing --- .../mapping/SearchableClassMappingConfigurator.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/java/org/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.java b/src/java/org/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.java index a2bdf777..18a8396f 100644 --- a/src/java/org/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.java +++ b/src/java/org/grails/plugins/elasticsearch/mapping/SearchableClassMappingConfigurator.java @@ -95,6 +95,11 @@ public void installMappings(Collection mappings) { installedIndices.add(scm.getIndexName()); LOG.debug(elasticMapping.toString()); + // Wait for the index to be ready + elasticSearchClient.admin().cluster().prepareHealth(scm.getIndexName()) + .setWaitForYellowStatus() + .execute().actionGet(); + // If the index already exists, ignore the exception } catch (IndexAlreadyExistsException iaee) { LOG.debug("Index " + scm.getIndexName() + " already exists, skip index creation."); From a2b1bc59015e81af31a3f0d30ae6ea81ccf0eaaf Mon Sep 17 00:00:00 2001 From: Simon Pither Date: Wed, 8 Jun 2011 15:43:43 +0100 Subject: [PATCH 2/5] Add some useful ignores --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f63a24e6..1a1ffb0e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea *.iml out +target +plugin.xml From 0332606d05ac5d600e19cde5b5efa254b223b2fc Mon Sep 17 00:00:00 2001 From: Simon Pither Date: Thu, 9 Jun 2011 11:10:51 +0100 Subject: [PATCH 3/5] Add support for cluster.name config option --- .../elasticsearch/ClientNodeFactoryBean.groovy | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/groovy/org/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy b/src/groovy/org/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy index 13fb5a55..719accf6 100644 --- a/src/groovy/org/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy +++ b/src/groovy/org/grails/plugins/elasticsearch/ClientNodeFactoryBean.groovy @@ -47,6 +47,12 @@ class ClientNodeFactoryBean implements FactoryBean { LOG.info "Using ElasticSearch data path: ${dataPath}" } + def clusterName = elasticSearchContextHolder.config.cluster.name + if (clusterName) { + nb.settings.put('cluster.name', clusterName as String) + LOG.info "Using ElasticSearch cluster name: ${clusterName}" + } + // Configure the client based on the client mode switch (clientMode) { case 'local': @@ -69,12 +75,15 @@ class ClientNodeFactoryBean implements FactoryBean { break; case 'transport': + def transportSettings = ImmutableSettings.settingsBuilder() // Use the "sniff" feature of transport client ? if(elasticSearchContextHolder.config.client.transport.sniff) { - transportClient = new TransportClient(ImmutableSettings.settingsBuilder().put("client.transport.sniff", true)) - } else { - transportClient = new TransportClient() + transportSettings.put("client.transport.sniff", true) + } + if (clusterName) { + transportSettings.put('cluster.name', clusterName as String) } + transportClient = new TransportClient(transportSettings) // Configure transport addresses if (!elasticSearchContextHolder.config.client.hosts) { From a8c594eec25dcc5e3206fcc4f08ff5c7ea432d51 Mon Sep 17 00:00:00 2001 From: Simon Pither Date: Thu, 9 Jun 2011 17:22:40 +0100 Subject: [PATCH 4/5] Recycle the Hibernate session and limit the size of the bulk request during bulk operations (eg import) --- .../elasticsearch/ElasticSearchService.groovy | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/grails-app/services/org/grails/plugins/elasticsearch/ElasticSearchService.groovy b/grails-app/services/org/grails/plugins/elasticsearch/ElasticSearchService.groovy index ebc0dcef..b4c7c904 100644 --- a/grails-app/services/org/grails/plugins/elasticsearch/ElasticSearchService.groovy +++ b/grails-app/services/org/grails/plugins/elasticsearch/ElasticSearchService.groovy @@ -220,6 +220,7 @@ public class ElasticSearchService implements GrailsApplicationAware { mappings = elasticSearchContextHolder.mapping.values() } + int perLoop = elasticSearchContextHolder.config.maxBulkRequest ?: 500 mappings.each { scm -> if (scm.root) { if (operationType == INDEX_REQUEST) { @@ -227,12 +228,27 @@ public class ElasticSearchService implements GrailsApplicationAware { } else if (operationType == DELETE_REQUEST) { LOG.debug("Deleting all instances of ${scm.domainClass}") } - scm.domainClass.metaClass.invokeStaticMethod(scm.domainClass.clazz, "getAll", null).each { - if (operationType == INDEX_REQUEST) { - indexRequestQueue.addIndexRequest(it) - } else if (operationType == DELETE_REQUEST) { - indexRequestQueue.addDeleteRequest(it) + long lastId = 0 + long maxId = scm.domainClass.metaClass.invokeStaticMethod(scm.domainClass.clazz, "withCriteria", { + projections { + max('id') } + })[0] + while(lastId < maxId) { + scm.domainClass.metaClass.invokeStaticMethod(scm.domainClass.clazz, "withTransaction", { status -> + scm.domainClass.metaClass.invokeStaticMethod(scm.domainClass.clazz, "withNewSession", { session -> + scm.domainClass.metaClass.invokeStaticMethod(scm.domainClass.clazz, "findAllByIdGreaterThan", lastId, [ max: perLoop, sort: 'id', order: 'asc' ]).each { + if (operationType == INDEX_REQUEST) { + indexRequestQueue.addIndexRequest(it) + } else if (operationType == DELETE_REQUEST) { + indexRequestQueue.addDeleteRequest(it) + } + lastId = it.id + } + indexRequestQueue.executeRequests() + session.clear() + }) + }) } } else { LOG.debug("${scm.domainClass.clazz} is not a root searchable class and has been ignored.") From 2a6bf6b20292c894e6fcb14914447d570ecd6f5b Mon Sep 17 00:00:00 2001 From: Simon Pither Date: Sat, 11 Jun 2011 20:34:17 +0100 Subject: [PATCH 5/5] Force per-domain methods to always set domain based filters, not just when no extra parameters are provided --- .../util/DomainDynamicMethodsUtils.groovy | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/groovy/org/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy b/src/groovy/org/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy index 1f116c6e..bbf0adf8 100644 --- a/src/groovy/org/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy +++ b/src/groovy/org/grails/plugins/elasticsearch/util/DomainDynamicMethodsUtils.groovy @@ -45,24 +45,36 @@ class DomainDynamicMethodsUtils { // Only inject the methods if the domain is mapped as "root" if (elasticSearchContextHolder.getMappingContext(domainCopy)?.root) { // Inject the search method - domain.metaClass.'static'.search << { String q, Map params = [indices: domainCopy.packageName ?: domainCopy.propertyName, types: domainCopy.clazz] -> + domain.metaClass.'static'.search << { String q, Map params = [:] -> + params.indices = domainCopy.packageName ?: domainCopy.propertyName + params.types = domainCopy.clazz elasticSearchService.search(q, params) } - domain.metaClass.'static'.search << { Map params = [indices: domainCopy.packageName ?: domainCopy.propertyName, types: domainCopy.clazz], Closure q -> + domain.metaClass.'static'.search << { Map params = [:], Closure q -> + params.indices = domainCopy.packageName ?: domainCopy.propertyName + params.types = domainCopy.clazz elasticSearchService.search(params, q) } - domain.metaClass.'static'.search << { Closure q, Map params = [indices: domainCopy.packageName ?: domainCopy.propertyName, types: domainCopy.clazz] -> + domain.metaClass.'static'.search << { Closure q, Map params = [:] -> + params.indices = domainCopy.packageName ?: domainCopy.propertyName + params.types = domainCopy.clazz elasticSearchService.search(params, q) } // Inject the countHits method - domain.metaClass.'static'.countHits << { String q, Map params = [indices: domainCopy.packageName ?: domainCopy.propertyName, types: domainCopy.clazz] -> + domain.metaClass.'static'.countHits << { String q, Map params = [:] -> + params.indices = domainCopy.packageName ?: domainCopy.propertyName + params.types = domainCopy.clazz elasticSearchService.countHits(q, params) } - domain.metaClass.'static'.countHits << { Map params = [indices: domainCopy.packageName ?: domainCopy.propertyName, types: domainCopy.clazz], Closure q -> + domain.metaClass.'static'.countHits << { Map params = [:], Closure q -> + params.indices = domainCopy.packageName ?: domainCopy.propertyName + params.types = domainCopy.clazz elasticSearchService.countHits(params, q) } - domain.metaClass.'static'.countHits << { Closure q, Map params = [indices: domainCopy.packageName ?: domainCopy.propertyName, types: domainCopy.clazz] -> + domain.metaClass.'static'.countHits << { Closure q, Map params = [:] -> + params.indices = domainCopy.packageName ?: domainCopy.propertyName + params.types = domainCopy.clazz elasticSearchService.countHits(params, q) }