Merge pull request #53 from wyhw/master

support for Elasticsearch 7.3.1

Author: bells

pom.xml

@@ -4,15 +4,15 @@
 
     <groupId>com.bellszhu.elasticsearch</groupId>
     <artifactId>elasticsearch-analysis-dynamic-synonym</artifactId>
-    <version>7.0.0</version>
+    <version>7.3.1</version>
     <packaging>jar</packaging>
     <name>elasticsearch-dynamic-synonym</name>
     <description>Analysis-plugin for synonym</description>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <elasticsearch.version>${project.version}</elasticsearch.version>
-        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.target>12</maven.compiler.target>
         <elasticsearch.plugin.name>analysis-dynamic-synonym</elasticsearch.plugin.name>
         <elasticsearch.assembly.descriptor>${project.basedir}/src/main/assemblies/plugin.xml
         </elasticsearch.assembly.descriptor>

src/main/java/com/bellszhu/elasticsearch/plugin/DynamicSynonymPlugin.java

@@ -4,7 +4,6 @@
 package com.bellszhu.elasticsearch.plugin;
 
 import com.bellszhu.elasticsearch.plugin.synonym.analysis.DynamicSynonymTokenFilterFactory;
-import com.bellszhu.elasticsearch.plugin.synonym.service.DynamicSynonymAnalysisService;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
@@ -14,7 +13,6 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.AnalysisPlugin;
@@ -30,65 +28,18 @@
 import java.util.Map;
 
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.plugins.AnalysisPlugin.requiresAnalysisSettings;
+
 
 /**
  * @author bellszhu
  */
 public class DynamicSynonymPlugin extends Plugin implements AnalysisPlugin {
-    private PluginComponent pluginComponent = new PluginComponent();
-
-    @Override
-    public Collection<Object> createComponents(Client client,
-                                               ClusterService clusterService,
-                                               ThreadPool threadPool,
-                                               ResourceWatcherService resourceWatcherService,
-                                               ScriptService scriptService,
-                                               NamedXContentRegistry xContentRegistry,
-                                               Environment environment,
-                                               NodeEnvironment nodeEnvironment,
-                                               NamedWriteableRegistry namedWriteableRegistry) {
-        Collection<Object> components = new ArrayList<>();
-        components.add(pluginComponent);
-        return components;
-    }
-
-    @Override
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        return singletonList(DynamicSynonymAnalysisService.class);
-    }
 
     @Override
     public Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
-        Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenFilterFactory>> extra = new HashMap<>();
-
-        extra.put("dynamic_synonym", new AnalysisModule.AnalysisProvider<TokenFilterFactory>() {
-
-            @Override
-            public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings)
-                    throws IOException {
-                return new DynamicSynonymTokenFilterFactory(indexSettings, environment, name, settings, pluginComponent.getAnalysisRegistry());
-            }
-
-            @Override
-            public boolean requiresAnalysisSettings() {
-                return true;
-            }
-        });
+        Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
+        extra.put("dynamic_synonym", requiresAnalysisSettings(DynamicSynonymTokenFilterFactory::new));
         return extra;
     }
-
-
-    public static class PluginComponent {
-
-        private AnalysisRegistry analysisRegistry;
-
-        AnalysisRegistry getAnalysisRegistry() {
-            return analysisRegistry;
-        }
-
-        public void setAnalysisRegistry(AnalysisRegistry analysisRegistry) {
-            this.analysisRegistry = analysisRegistry;
-        }
-
-    }
 }

src/main/java/com/bellszhu/elasticsearch/plugin/synonym/analysis/DynamicSynonymTokenFilterFactory.java

@@ -9,13 +9,17 @@
 import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.synonym.SynonymMap;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.Analysis;
+import org.elasticsearch.index.analysis.AnalysisMode;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 
 import java.io.IOException;
 import java.util.Map;
@@ -26,12 +30,18 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import java.util.List;
+import java.util.function.Function;
+
 /**
  * @author bellszhu
  */
 public class DynamicSynonymTokenFilterFactory extends
         AbstractTokenFilterFactory {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER
+            = new DeprecationLogger(LogManager.getLogger(DynamicSynonymTokenFilterFactory.class));
+
     /**
      * Static id generator
      */
@@ -43,20 +53,21 @@ public class DynamicSynonymTokenFilterFactory extends
         return thread;
     });
     private final String location;
-    private final boolean ignoreCase;
     private final boolean expand;
+    private final boolean lenient;
     private final String format;
     private final int interval;
     private volatile ScheduledFuture<?> scheduledFuture;
     private SynonymMap synonymMap;
     private Map<DynamicSynonymFilter, Integer> dynamicSynonymFilters = new WeakHashMap<>();
+    protected final Environment environment;
+    protected final AnalysisMode analysisMode;
 
     public DynamicSynonymTokenFilterFactory(
             IndexSettings indexSettings,
             Environment env,
             String name,
-            Settings settings,
-            AnalysisRegistry analysisRegistry
+            Settings settings
     ) throws IOException {
 
         super(indexSettings, name, settings);
@@ -68,58 +79,103 @@ public DynamicSynonymTokenFilterFactory(
         }
 
         this.interval = settings.getAsInt("interval", 60);
-        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
+        if (settings.get("ignore_case") != null) {
+            DEPRECATION_LOGGER.deprecated(
+                "The ignore_case option on the synonym_graph filter is deprecated. " +
+                    "Instead, insert a lowercase filter in the filter chain before the synonym_graph filter.");
+        }
         this.expand = settings.getAsBoolean("expand", true);
+        this.lenient = settings.getAsBoolean("lenient", false);
         this.format = settings.get("format", "");
+        boolean updateable = settings.getAsBoolean("updateable", false);
+        this.analysisMode = updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
+        this.environment = env;
+    }
 
-        String tokenizerName = settings.get("tokenizer", "whitespace");
-
-        AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory =
-                analysisRegistry.getTokenizerProvider(tokenizerName, indexSettings);
-        if (tokenizerFactoryFactory == null) {
-            throw new IllegalArgumentException("failed to find tokenizer [" + tokenizerName + "] for synonym token filter");
-        }
-        final TokenizerFactory tokenizerFactory = tokenizerFactoryFactory.get(indexSettings, env, tokenizerName,
+    @Override
+    public AnalysisMode getAnalysisMode() {
+        return this.analysisMode;
+    }
 
-                AnalysisRegistry.getSettingsFromIndexSettings(indexSettings, AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER + "." + tokenizerName));
 
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        throw new IllegalStateException("Call getChainAwareTokenFilterFactory to specialize this factory for an analysis chain first");
+    }
 
-        Analyzer analyzer = new Analyzer() {
+    public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                                              List<TokenFilterFactory> previousTokenFilters,
+                                                              Function<String, TokenFilterFactory> allFilters) {
+        final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters, allFilters);
+        synonymMap = buildSynonyms(analyzer);
+        final String name = name();
+        return new TokenFilterFactory() {
             @Override
-            protected TokenStreamComponents createComponents(String fieldName) {
-                Tokenizer tokenizer = tokenizerFactory == null ? new WhitespaceTokenizer() : tokenizerFactory.create();
-                TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer;
-                return new TokenStreamComponents(tokenizer, stream);
+            public String name() {
+                return name;
             }
-        };
 
-        SynonymFile synonymFile;
-        if (location.startsWith("http://") || location.startsWith("https://")) {
-            synonymFile = new RemoteSynonymFile(env, analyzer, expand, format,
-                    location);
-        } else {
-            synonymFile = new LocalSynonymFile(env, analyzer, expand, format,
-                    location);
-        }
-        synonymMap = synonymFile.reloadSynonymMap();
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                // fst is null means no synonyms
+                if (synonymMap.fst == null) {
+                    return tokenStream;
+                }
+                DynamicSynonymFilter dynamicSynonymFilter = new DynamicSynonymFilter(tokenStream, synonymMap, false);
+                dynamicSynonymFilters.put(dynamicSynonymFilter, 1);
 
-        scheduledFuture = pool.scheduleAtFixedRate(new Monitor(synonymFile),
-                interval, interval, TimeUnit.SECONDS);
+                return dynamicSynonymFilter;
+            }
+
+            @Override
+            public TokenFilterFactory getSynonymFilter() {
+                // In order to allow chained synonym filters, we return IDENTITY here to
+                // ensure that synonyms don't get applied to the synonym map itself,
+                // which doesn't support stacked input tokens
+                return IDENTITY_FILTER;
+            }
 
+            @Override
+            public AnalysisMode getAnalysisMode() {
+                return analysisMode;
+            }
+        };
     }
 
+    Analyzer buildSynonymAnalyzer(TokenizerFactory tokenizer, List<CharFilterFactory> charFilters,
+                                  List<TokenFilterFactory> tokenFilters, Function<String, TokenFilterFactory> allFilters) {
+        return new CustomAnalyzer("dynamic_synonym", tokenizer, charFilters.toArray(new CharFilterFactory[0]),
+            tokenFilters.stream()
+                .map(TokenFilterFactory::getSynonymFilter)
+                .toArray(TokenFilterFactory[]::new));
+    }
 
-    @Override
-    public TokenStream create(TokenStream tokenStream) {
-        // fst is null means no synonyms
-        if (synonymMap == null || synonymMap.fst == null) {
-            return tokenStream;
+    SynonymMap buildSynonyms(Analyzer analyzer) {
+        try {
+            return getSynonymFile(analyzer).reloadSynonymMap();
+        } catch (Exception e) {
+            throw new IllegalArgumentException("failed to build synonyms", e);
         }
+    }
 
-        DynamicSynonymFilter dynamicSynonymFilter = new DynamicSynonymFilter(tokenStream, synonymMap, ignoreCase);
-        dynamicSynonymFilters.put(dynamicSynonymFilter, 1);
-
-        return dynamicSynonymFilter;
+    SynonymFile getSynonymFile(Analyzer analyzer) {
+        try {
+            SynonymFile synonymFile;
+            if (location.startsWith("http://") || location.startsWith("https://")) {
+                synonymFile = new RemoteSynonymFile(environment, analyzer, expand, format,
+                        location);
+            } else {
+                synonymFile = new LocalSynonymFile(environment, analyzer, expand, format,
+                        location);
+            }
+            if (scheduledFuture == null) {
+                scheduledFuture = pool.scheduleAtFixedRate(new Monitor(synonymFile),
+                                interval, interval, TimeUnit.SECONDS);
+            }
+            return synonymFile;
+        } catch (Exception e) {
+            throw new IllegalArgumentException("failed to get synonyms : " + location, e);
+        }
     }
 
     public class Monitor implements Runnable {