Index: lucene/CHANGES.txt
===================================================================
--- lucene/CHANGES.txt	(revision 1660064)
+++ lucene/CHANGES.txt	(working copy)
@@ -132,6 +132,8 @@
 
 New Features
 
+* LUCENE-6254: Dictionary-based lemmatizer (Erlend Garåsen)
+
 * LUCENE-5945: All file handling converted to NIO.2 apis. (Robert Muir)
 
 * LUCENE-5946: SimpleFSDirectory now uses Files.newByteChannel, for 
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizer.java	(working copy)
@@ -0,0 +1,48 @@
+package org.apache.lucene.analysis.lemmatizer;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Map;
+
+/**
+ * Lemmatizer which looks up lemmas from a dictionary
+ */
+public class DictionaryLemmatizer {
+
+  private final Map<String, String[]> wordlist;
+
+  /**
+   * Creates a new DictionaryLemmatizer
+   * 
+   * @param wordlist a Hashmap containing all the words with their lemmas
+   */
+  public DictionaryLemmatizer(final Map<String, String[]> wordlist) {
+    this.wordlist = wordlist;
+  }
+
+  /**
+   * Find the lemma(s) of the provided word.
+   * 
+   * @param word Word to find the lemma(s)
+   * @return a list of lemmas for the word
+   */
+  public String[] lemmatize(final String word) {
+    return wordlist.get(word);
+  }
+
+}

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilter.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilter.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilter.java	(working copy)
@@ -0,0 +1,106 @@
+package org.apache.lucene.analysis.lemmatizer;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Queue;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.AttributeSource;
+
+/**
+ * A {@link TokenFilter} that applies {@link DictionaryLemmatizer} to lemmatize
+ * words.
+ * <p>
+ * To prevent terms from being lemmatized, use an instance of
+ * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets the
+ * {@link KeywordAttribute} before this {@link TokenStream}.
+ * </p>
+ */
+public final class DictionaryLemmatizerFilter extends TokenFilter {
+  private final DictionaryLemmatizer lemmatizer;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
+  private final PositionIncrementAttribute positionAttr = addAttribute(PositionIncrementAttribute.class);
+  private final Queue<String> terms = new LinkedList<String>();
+
+  private AttributeSource.State current = null;
+
+  /**
+   * Creates a DictionaryLemmatizerFilter outputting possible lemmas.
+   * 
+   * @param input TokenStream whose tokens will be lemmatized
+   * @param wordlist a Hashmap containing all the words with their lemmas
+   */
+  public DictionaryLemmatizerFilter(final TokenStream input, final Map<String, String[]> wordlist) {
+    super(input);
+    lemmatizer = new DictionaryLemmatizer(wordlist);
+  }
+
+  @Override
+  public boolean incrementToken() throws IOException {
+    if (!terms.isEmpty()) {
+      if (createToken(terms.poll(), current)) {
+        return true;
+      }
+    }
+    if (input.incrementToken()) {
+      if (!keywordAttr.isKeyword()) {
+        char[] buffer = termAtt.buffer();
+        final String tokenTerm = new String(buffer, 0, termAtt.length());
+
+        final String[] values = lemmatizer.lemmatize(tokenTerm);
+        if (values != null) {
+          // Replace first token with the lemma:
+          termAtt.setEmpty().append(values[0]);
+          if (values.length > 1) {
+            // Queue remaining lemmas for later processing
+            for (int i = 1; i < values.length; i++) {
+              terms.add(values[i]);
+            }
+          }
+        }
+        current = captureState();
+      }
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  protected boolean createToken(final String synonym, final AttributeSource.State current) {
+    restoreState(current);
+    termAtt.setEmpty().append(synonym);
+    positionAttr.setPositionIncrement(0);
+    return true;
+  }
+
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    current = null;
+  }
+
+}

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilter.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilterFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilterFactory.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilterFactory.java	(working copy)
@@ -0,0 +1,317 @@
+package org.apache.lucene.analysis.lemmatizer;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Factory for {@link DictionaryLemmatizerFilter}. Minimal configuration
+ * example:
+ * 
+ * <pre class="prettyprint">
+ * &lt;filter class=&quot;solr.DictionaryLemmatizerFilterFactory&quot;
+ *         dictionaries=&quot;dictionary.zip,my_custom_dic.txt&quot;
+ *         wordPos=&quot;1&quot;
+ *         lemmaPos=&quot;2&quot;
+ * </pre>
+ */
+public class DictionaryLemmatizerFilterFactory extends TokenFilterFactory implements
+    ResourceLoaderAware {
+
+  private final Map<String, Set<String>> unnormalizedWordlist = new HashMap<String, Set<String>>();
+  private final Map<String, String[]> normalizedWordlist = new HashMap<String, String[]>();
+
+  private Reader reader = null;
+  private BufferedReader br = null;
+
+  private static final String PARAM_WORD_CLASSES = "wordClasses";
+  private static final String PARAM_CHARSET = "charset";
+  private static final String PARAM_MIN_LENGTH = "minLength";
+  private static final String PARAM_LEMMA_POS = "lemmaPos";
+  private static final String PARAM_WORD_POS = "wordPos";
+  private static final String PARAM_WORD_CLASS_POS = "wordClassPos";
+  private static final String PARAM_REDUCE_TO = "reduceTo";
+  private static final String PARAM_STORE_POS_TAG = "storePosTag";
+  private static final String PARAM_DICTIONARIES = "dictionaries";
+
+  private int minLength;
+  private String dictionaries;
+  private int lemmaPos;
+  private int wordPos;
+  private int wordClassPos;
+  private String[] wordClasses;
+  private String charset;
+  private String[] reduceTo;
+  private boolean storePosTag;
+
+  /** Creates a new DictionaryLemmatizerFilterFactory */
+  public DictionaryLemmatizerFilterFactory(final Map<String, String> args) {
+    super(args);
+
+    final String wordClassList = get(args, PARAM_WORD_CLASSES);
+    wordClasses = (wordClassList != null) ? wordClassList.split(",") : null;
+    charset = get(args, PARAM_CHARSET, "UTF-8");
+    minLength = getInt(args, PARAM_MIN_LENGTH, 3);
+    dictionaries = require(args, PARAM_DICTIONARIES);
+    lemmaPos = getInt(args, PARAM_LEMMA_POS, -1);
+    wordPos = getInt(args, PARAM_WORD_POS, -1);
+    wordClassPos = getInt(args, PARAM_WORD_CLASS_POS, -1);
+    final String reduceToList = get(args, PARAM_REDUCE_TO);
+    reduceTo = (reduceToList != null) ? reduceToList.split(",") : null;
+    storePosTag = getBoolean(args, PARAM_STORE_POS_TAG, false);
+
+    if (lemmaPos < 0) {
+      throw new IllegalArgumentException("Parameter " + PARAM_LEMMA_POS + " not properly set");
+    }
+    if (wordPos < 0) {
+      throw new IllegalArgumentException("Parameter " + PARAM_WORD_POS + " not properly set");
+    }
+
+    if (storePosTag && wordClassPos < 0) {
+      throw new IllegalArgumentException("Parameter " + PARAM_STORE_POS_TAG + " requires that "
+          + PARAM_WORD_POS + " is properly set");
+    }
+
+    if (storePosTag && wordClasses.length == 0) {
+      throw new IllegalArgumentException("Parameter " + PARAM_STORE_POS_TAG + " requires that "
+          + PARAM_WORD_CLASSES + " is properly set");
+    }
+
+    if (!args.isEmpty()) {
+      throw new IllegalArgumentException("Unknown parameters: " + args);
+    }
+
+  }
+
+  private void handleStream(final List<InputStream> inputStreams) throws IOException {
+    for (InputStream inputStream : inputStreams) {
+      if (inputStream instanceof ZipInputStream) {
+        ZipEntry entry;
+        while ((entry = ((ZipInputStream) inputStream).getNextEntry()) != null) {
+          final String entryName = entry.getName();
+          final Path currentPath = Paths.get(entryName);
+          final Path parentPath = currentPath.getParent();
+          if (parentPath == null && Files.isDirectory(currentPath)) {
+            break;
+          }
+          addDictionary(inputStream);
+        }
+      } else {
+        addDictionary(inputStream);
+      }
+    }
+    addEntries();
+    unnormalizedWordlist.clear();
+  }
+
+  /*
+   * Adds entries in a dictionary to a temporary map where the key is the word
+   * and the value is a comma-separated list of lemmas for that word. A word can
+   * have several lemmas: German: (wir/viele) fragen => (zu) fragen (verb),
+   * (eine) Frage (noun). Norwegian: (vi/mange) sykler => (å) sykle (verb), (en)
+   * sykkel (noun)
+   */
+  private void addDictionary(final InputStream inputStream) throws IOException {
+    reader = new InputStreamReader(inputStream, charset);
+    br = new BufferedReader(reader);
+    String line = null;
+    while ((line = br.readLine()) != null) {
+      final String[] parts = line.split("\t");
+
+      // Skip comments etc. in file:
+      if (parts.length < 2 || line.trim().startsWith("#") || line.trim().startsWith("*")) {
+        continue;
+      }
+
+      final String word = parts[wordPos];
+
+      // Skip words which do not meet the threshold:
+      if (word.length() <= minLength) {
+        continue;
+      }
+
+      // Skip splitting words and those with a hyphen — they
+      // interfere with tokenizers.
+      if (word.contains(" ") || word.contains("-")) {
+        continue;
+      }
+
+      // Only include words which belong to the defined word classes
+      String lemma = null;
+      if (wordClasses != null) {
+        for (int i = 0; i < wordClasses.length; i++) {
+          final String wordClass = wordClasses[i];
+          if (parts[wordClassPos].contains(wordClass)) {
+            // add POS-tag for the given word class:
+            lemma = parts[lemmaPos] + "$" + i;
+            break;
+          }
+        }
+      } else {
+        lemma = parts[lemmaPos];
+      }
+      if (lemma == null) {
+        continue;
+      }
+
+      Set<String> entry = unnormalizedWordlist.get(word);
+      if (entry == null) {
+        entry = new LinkedHashSet<String>();
+      }
+      entry.add(lemma);
+      unnormalizedWordlist.put(word, entry);
+    }
+  }
+
+  private void addEntries() {
+    for (Iterator<Map.Entry<String, Set<String>>> entries = unnormalizedWordlist.entrySet()
+        .iterator(); entries.hasNext();) {
+      final Map.Entry<String, Set<String>> entry = entries.next();
+      final Set<String> lemmas = entry.getValue();
+      final String word = entry.getKey();
+
+      // If reduce is defined, make sure that at least one lemma from a defined
+      // word class is added:
+      if (reduceTo != null) {
+        if (lemmas.size() > 1) {
+          // If several lemmas for the same word class are found, use
+          // the shortest:
+          String lemmaToUse = null;
+          for (String wordClass : reduceTo) {
+            final int posTag = Arrays.asList(wordClasses).indexOf(wordClass);
+            for (String lemma : lemmas) {
+              if (lemma.contains("$" + posTag) && !lemma.equals(word + "$" + posTag)) {
+                lemmaToUse = (lemmaToUse != null) ? (lemmaToUse.length() > lemma.length()) ? lemma
+                    : lemmaToUse : lemma;
+              }
+            }
+            if (lemmaToUse != null) {
+              break;
+            }
+          }
+          if (lemmaToUse == null) { // Did not find any matched word classes,
+            // just use the shortest lemma:
+            for (String lemma : lemmas) {
+              lemmaToUse = (lemmaToUse != null) ? (lemmaToUse.length() > lemma.length()) ? lemma
+                  : lemmaToUse : lemma;
+            }
+          }
+          if (lemmaToUse != null) {
+            final String newLemma = (storePosTag) ? lemmaToUse : lemmaToUse.replaceAll("\\$\\d+",
+                "");
+            final String[] newLemmas = { newLemma };
+            normalizedWordlist.put(word, newLemmas);
+            continue;
+          } else {
+            continue;
+          }
+        } else {
+          storeLemmas(lemmas, word);
+        }
+      } else {
+        storeLemmas(lemmas, word);
+      }
+    }
+  }
+
+  private void storeLemmas(final Set<String> lemmas, final String word) {
+    if (storePosTag) {
+      final int size = (reduceTo != null) ? 1 : lemmas.size();
+      final String[] newLemmas = lemmas.toArray(new String[size]);
+      normalizedWordlist.put(word, newLemmas);
+    } else {
+      // Exclude entries where the lemma equals the word as long as we
+      // only have one lemma:
+      if (lemmas.size() == 1) {
+        final String newLemma = lemmas.iterator().next().replaceAll("\\$\\d+", "");
+        if (!newLemma.equals(word)) {
+          final String[] newLemmas = { newLemma };
+          normalizedWordlist.put(word, newLemmas);
+        }
+      } else {
+        final Set<String> lemmaList = new HashSet<String>();
+        for (String lemma : lemmas) {
+          final String newLemma = lemma.replaceAll("\\$\\d+", "");
+          if (!newLemma.equals(word)) {
+            lemmaList.add(newLemma);
+          }
+        }
+        if (lemmaList.size() > 0) {
+          final int size = (reduceTo != null) ? 1 : lemmaList.size();
+          final String[] newLemmas = lemmaList.toArray(new String[size]);
+          normalizedWordlist.put(word, newLemmas);
+        }
+      }
+    }
+  }
+
+  @Override
+  public TokenStream create(TokenStream input) {
+    return new DictionaryLemmatizerFilter(input, normalizedWordlist);
+  }
+
+  @Override
+  public void inform(final ResourceLoader resourceLoader) throws IOException {
+    final String[] files = dictionaries.split(",");
+    final List<InputStream> dictionaries = new ArrayList<InputStream>();
+    try {
+      for (String file : files) {
+        InputStream inputStream = resourceLoader.openResource(file);
+        if (file.endsWith(".gz")) {
+          inputStream = new GZIPInputStream(inputStream);
+        } else if (file.endsWith(".zip")) {
+          inputStream = new ZipInputStream(inputStream);
+        }
+        dictionaries.add(inputStream);
+      }
+      handleStream(dictionaries);
+    } catch (Exception e) {
+      throw new IOException("Unable to load dictionary", e);
+    } finally {
+      IOUtils.closeWhileHandlingException(dictionaries);
+      IOUtils.closeWhileHandlingException(reader);
+      IOUtils.closeWhileHandlingException(br);
+    }
+  }
+
+}

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/DictionaryLemmatizerFilterFactory.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/package.html
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/package.html	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/package.html	(working copy)
@@ -0,0 +1,22 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html><head></head>
+<body>
+A dictionary-based lemmatizer.
+</body>
+</html>

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/lemmatizer/package.html
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
===================================================================
--- lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory	(revision 1660064)
+++ lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory	(working copy)
@@ -56,6 +56,7 @@
 org.apache.lucene.analysis.id.IndonesianStemFilterFactory
 org.apache.lucene.analysis.in.IndicNormalizationFilterFactory
 org.apache.lucene.analysis.it.ItalianLightStemFilterFactory
+org.apache.lucene.analysis.lemmatizer.DictionaryLemmatizerFilterFactory
 org.apache.lucene.analysis.lv.LatvianStemFilterFactory
 org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory
 org.apache.lucene.analysis.miscellaneous.CapitalizationFilterFactory
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java	(revision 1660064)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java	(working copy)
@@ -429,6 +429,16 @@
         }
       }
     });
+    put(Map.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        Map<String, String> map = new HashMap<String, String>();
+        int num = random.nextInt(5);
+        for (int i = 0; i < num; i++) {
+         map.put(TestUtil.randomSimpleString(random), TestUtil.randomSimpleString(random));
+        }
+        return map;
+      }
+    });
     put(HyphenationTree.class, new ArgProducer() {
       @Override public Object create(Random random) {
         // TODO: make nastier
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilter.java	(revision 0)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilter.java	(working copy)
@@ -0,0 +1,83 @@
+package org.apache.lucene.analysis.lemmatizer;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
+
+public class TestDictionaryLemmatizerFilter extends BaseTokenStreamTestCase {
+
+  private Analyzer analyzer = new Analyzer() {
+    @Override
+    protected TokenStreamComponents createComponents(final String fieldName) {
+      Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+      return new TokenStreamComponents(source, new DictionaryLemmatizerFilter(source,
+          getMockedWordlist()));
+    }
+  };
+
+  public void testKeyword() throws IOException {
+    final CharArraySet exclusionSet = new CharArraySet(asSet("katze"), false);
+    Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(final String fieldName) {
+        Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
+        return new TokenStreamComponents(source, new DictionaryLemmatizerFilter(sink,
+            getMockedWordlist()));
+      }
+    };
+    checkOneTerm(a, "katze", "katze");
+  }
+
+  /** blast some random strings through the analyzer */
+  public void testRandomStrings() throws Exception {
+    checkRandomData(random(), analyzer, 1000 * RANDOM_MULTIPLIER);
+  }
+
+  public void testEmptyTerm() throws IOException {
+    Analyzer a = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(final String fieldName) {
+        Tokenizer tokenizer = new KeywordTokenizer();
+        return new TokenStreamComponents(tokenizer, new DictionaryLemmatizerFilter(tokenizer,
+            getMockedWordlist()));
+      }
+    };
+    checkOneTerm(a, "", "");
+  }
+
+  private Map<String, String[]> getMockedWordlist() {
+    final Map<String, String[]> wordList = new HashMap<String, String[]>();
+    wordList.put("bücher", new String[] { "buch" });
+    wordList.put("fragen", new String[] { "frage, fragen" });
+    wordList.put("katzen", new String[] { "katze" });
+    return wordList;
+  }
+
+}

Property changes on: lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilter.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilterFactory.java	(revision 0)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilterFactory.java	(working copy)
@@ -0,0 +1,73 @@
+package org.apache.lucene.analysis.lemmatizer;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+public class TestDictionaryLemmatizerFilterFactory extends BaseTokenStreamFactoryTestCase {
+
+  public void testLemmatizerWithIrregularLemmas() throws Exception {
+    Reader reader = new StringReader("bücher eldre");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("DictionaryLemmatizer", "dictionaries", "dictionary.txt",
+        "lemmaPos", "0", "wordPos", "1").create(stream);
+    assertTokenStreamContents(stream, new String[] { "buch", "gammel" });
+  }
+
+  public void testLemmatizerWithMultipleLemmas() throws Exception {
+    Reader reader = new StringReader("sykler");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("DictionaryLemmatizer", "dictionaries", "dictionary.txt",
+        "lemmaPos", "0", "wordPos", "1").create(stream);
+    assertTokenStreamContents(stream, new String[] { "sykle", "sykkel" }, new int[] { 1, 0 });
+  }
+
+  public void testLemmatizerUsingPOSTags() throws Exception {
+    Reader reader = new StringReader("sykler");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("DictionaryLemmatizer", "dictionaries", "dictionary.txt",
+        "lemmaPos", "0", "wordPos", "1", "wordClassPos", "2", "storePosTag", "true", "wordClasses",
+        "noun,verb").create(stream);
+    assertTokenStreamContents(stream, new String[] { "sykkel$0", "sykle$1" }, new int[] { 1, 0 });
+  }
+
+  public void testLemmatizerUsingReduction() throws Exception {
+    Reader reader = new StringReader("sykler");
+    TokenStream stream = whitespaceMockTokenizer(reader);
+    stream = tokenFilterFactory("DictionaryLemmatizer", "dictionaries", "dictionary.txt",
+        "lemmaPos", "0", "wordPos", "1", "wordClassPos", "2", "reduceTo", "noun", "wordClasses",
+        "noun,verb").create(stream);
+    assertTokenStreamContents(stream, new String[] { "sykkel" });
+  }
+
+  /** Test that bogus arguments result in exception */
+  public void testBogusArguments() throws Exception {
+    try {
+      tokenFilterFactory("DictionaryLemmatizer", "dictionaries", "dictionary.txt", "lemmaPos", "0",
+          "wordPos", "1", "bogusArg", "bogusValue");
+      fail();
+    } catch (IllegalArgumentException expected) {
+      assertTrue(expected.getMessage().contains("Unknown parameters"));
+    }
+  }
+
+}
\ No newline at end of file

Property changes on: lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/TestDictionaryLemmatizerFilterFactory.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/dictionary.txt
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/dictionary.txt	(revision 0)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/dictionary.txt	(working copy)
@@ -0,0 +1,27 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+
+# Lemma	word	word class
+sykkel	sykler	noun
+sykkel	sykkelen	noun
+sykle	sykler	verb
+sykle	syklet	verb
+
+frage	fragen	noun
+frage	frage noun
+fragen	fragte	verb
+fragen	fragt	verb
+
+buch	bücher	noun
+gammel	eldre	noun
\ No newline at end of file

Property changes on: lucene/analysis/common/src/test/org/apache/lucene/analysis/lemmatizer/dictionary.txt
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
