diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterTokenizer.java
new file mode 100644
index 0000000000..2483537cf9
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterTokenizer.java
@@ -0,0 +1,559 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.InPlaceMergeSorter;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/**
+ *
+ */
+public class WordDelimiterTokenizer extends Tokenizer {
+
+  /**
+   * Causes parts of words to be generated:
+   * <p>
+   * "PowerShot" =&gt; "Power" "Shot"
+   */
+  public static final int GENERATE_WORD_PARTS = 1;
+
+  /**
+   * Causes number subwords to be generated:
+   * <p>
+   * "500-42" =&gt; "500" "42"
+   */
+  public static final int GENERATE_NUMBER_PARTS = 2;
+
+  /**
+   * Causes maximum runs of word parts to be catenated:
+   * <p>
+   * "wi-fi" =&gt; "wifi"
+   */
+  public static final int CATENATE_WORDS = 4;
+
+  /**
+   * Causes maximum runs of number parts to be catenated:
+   * <p>
+   * "500-42" =&gt; "50042"
+   */
+  public static final int CATENATE_NUMBERS = 8;
+
+  /**
+   * Causes all subword parts to be catenated:
+   * <p>
+   * "wi-fi-4000" =&gt; "wifi4000"
+   */
+  public static final int CATENATE_ALL = 16;
+
+  /**
+   * Causes original words are preserved and added to the subword list (Defaults to false)
+   * <p>
+   * "500-42" =&gt; "500" "42" "500-42"
+   */
+  public static final int PRESERVE_ORIGINAL = 32;
+
+  /**
+   * Causes lowercase -&gt; uppercase transition to start a new subword.
+   */
+  public static final int SPLIT_ON_CASE_CHANGE = 64;
+
+  /**
+   * If not set, causes numeric changes to be ignored (subwords will only be generated
+   * given SUBWORD_DELIM tokens).
+   */
+  public static final int SPLIT_ON_NUMERICS = 128;
+
+  private final Tokenizer inputTokenizer;
+  private final int flags;
+
+  // packs start pos, end pos, start part, end part (= slice of the term text) for each buffered part:
+  private int[] bufferedParts = new int[16];
+  private int bufferedLen;
+  private int bufferedPos;
+
+  // holds text for each buffered part, or null if it's a simple slice of the original term
+  private char[][] bufferedTermParts = new char[4][];
+
+  private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
+  private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
+  private final PositionLengthAttribute posLenAttribute = addAttribute(PositionLengthAttribute.class);
+
+  private final Predicate<CharTermAttribute> protectTerm;
+
+  // used for iterating word delimiter breaks
+  private final WordDelimiterIterator iterator;
+
+  // used for concatenating runs of similar typed subwords (word,number)
+  private final WordDelimiterConcatenation concat = new WordDelimiterConcatenation();
+
+  // number of subwords last output by concat.
+  private int lastConcatCount;
+
+  // used for catenate all
+  private final WordDelimiterConcatenation concatAll = new WordDelimiterConcatenation();
+
+  // used for accumulating position increment gaps so that we preserve incoming holes:
+  private int accumPosInc;
+
+  private char[] savedTermBuffer = new char[16];
+  private int savedTermLength;
+  private int savedStartOffset;
+  private int savedEndOffset;
+  private AttributeSource.State savedState;
+  private int lastStartOffset;
+
+  // if length by start + end offsets doesn't match the term text then assume
+  // this is a synonym and don't adjust the offsets.
+  private boolean hasIllegalOffsets;
+
+  private int wordPos;
+
+  public WordDelimiterTokenizer(Tokenizer input, int flags, byte[] delimiterTable, Predicate<CharTermAttribute> protectTerm) {
+    super(input.getAttributeFactory());
+    this.inputTokenizer = input;
+    this.flags = flags;
+    this.protectTerm = protectTerm;
+    this.iterator = new WordDelimiterIterator(delimiterTable,
+        has(SPLIT_ON_CASE_CHANGE), has(SPLIT_ON_NUMERICS), false);
+  }
+
+  public WordDelimiterTokenizer(Tokenizer input, int flags) {
+    this(input, flags, WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, a -> false);
+  }
+
+  public WordDelimiterTokenizer(Tokenizer input, int flags, CharArraySet protectedWords) {
+    this(input, flags, WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, c -> protectedWords.contains(c.buffer(), 0, c.length()));
+  }
+
+  /** Iterates all words parts and concatenations, buffering up the term parts we should return. */
+  private void bufferWordParts() throws IOException {
+
+    saveState();
+
+    bufferedLen = 0;
+    lastConcatCount = 0;
+    wordPos = 0;
+
+    // if length by start + end offsets doesn't match the term's text then set offsets for all our word parts/concats to the incoming
+    // offsets.  this can happen if WDGF is applied to an injected synonym, or to a stem'd form, etc:
+    hasIllegalOffsets = (savedEndOffset - savedStartOffset != savedTermLength);
+
+    if (iterator.isSingleWord()) {
+      buffer(wordPos, wordPos+1, iterator.current, iterator.end);
+      wordPos++;
+      iterator.next();
+    } else {
+
+      // iterate all words parts, possibly buffering them, building up concatenations and possibly buffering them too:
+      while (iterator.end != WordDelimiterIterator.DONE) {
+        int wordType = iterator.type();
+
+        // do we already have queued up incompatible concatenations?
+        if (concat.isNotEmpty() && (concat.type & wordType) == 0) {
+          flushConcatenation(concat);
+        }
+
+        // add subwords depending upon options
+        if (shouldConcatenate(wordType)) {
+          concatenate(concat);
+        }
+
+        // add all subwords (catenateAll)
+        if (has(CATENATE_ALL)) {
+          concatenate(concatAll);
+        }
+
+        // if we should output the word or number part
+        if (shouldGenerateParts(wordType)) {
+          buffer(wordPos, wordPos+1, iterator.current, iterator.end);
+          wordPos++;
+        }
+        iterator.next();
+      }
+
+      if (concat.isNotEmpty()) {
+        // flush final concatenation
+        flushConcatenation(concat);
+      }
+
+      if (concatAll.isNotEmpty()) {
+        // only if we haven't output this same combo above, e.g. PowerShot with CATENATE_WORDS:
+        if (concatAll.subwordCount > lastConcatCount) {
+          if (wordPos == concatAll.startPos) {
+            // we are not generating parts, so we must advance wordPos now
+            wordPos++;
+          }
+          concatAll.write();
+        }
+        concatAll.clear();
+      }
+    }
+
+    if (has(PRESERVE_ORIGINAL)) {
+      if (wordPos == 0) {
+        // can happen w/ strange flag combos and inputs :)
+        wordPos++;
+      }
+      // add the original token now so that we can set the correct end position
+      buffer(0, wordPos, 0, savedTermLength);
+    }
+
+    sorter.sort(0, bufferedLen);
+    wordPos = 0;
+
+    // set back to 0 for iterating from the buffer
+    bufferedPos = 0;
+  }
+
+  @Override
+  public final boolean incrementToken() throws IOException {
+    while (true) {
+      if (savedState == null) {
+
+        // process a new input token
+        if (inputTokenizer.incrementToken() == false) {
+          return false;
+        }
+        clearAttributes();
+        inputTokenizer.copyTo(this);
+        int termLength = termAttribute.length();
+        char[] termBuffer = termAttribute.buffer();
+
+        accumPosInc += posIncAttribute.getPositionIncrement();
+
+        // iterate & cache all word parts up front:
+        iterator.setText(termBuffer, termLength);
+        iterator.next();
+
+        // word of no delimiters, or protected word: just return it
+        if ((iterator.current == 0 && iterator.end == termLength) ||
+            (protectTerm.test(termAttribute))) {
+          posIncAttribute.setPositionIncrement(accumPosInc);
+          accumPosInc = 0;
+          return true;
+        }
+
+        // word of simply delimiters: swallow this token, creating a hole, and move on to next token
+        if (iterator.end == WordDelimiterIterator.DONE) {
+          if (has(PRESERVE_ORIGINAL) == false) {
+            continue;
+          } else {
+            accumPosInc = 0;
+            return true;
+          }
+        }
+
+        // otherwise, we have delimiters, process & buffer all parts:
+        bufferWordParts();
+      }
+
+      if (bufferedPos < bufferedLen) {
+        clearAttributes();
+        restoreState(savedState);
+
+        char[] termPart = bufferedTermParts[bufferedPos];
+        int startPos = bufferedParts[4*bufferedPos];
+        int endPos = bufferedParts[4*bufferedPos+1];
+        int startPart = bufferedParts[4*bufferedPos+2];
+        int endPart = bufferedParts[4*bufferedPos+3];
+        bufferedPos++;
+
+        int startOffset;
+        int endOffset;
+
+        if (hasIllegalOffsets) {
+          startOffset = savedStartOffset;
+          endOffset = savedEndOffset;
+        } else {
+          startOffset = savedStartOffset + startPart;
+          endOffset = savedStartOffset + endPart;
+        }
+
+        // never let offsets go backwards:
+        startOffset = Math.max(startOffset, lastStartOffset);
+        endOffset = Math.max(endOffset, lastStartOffset);
+
+        offsetAttribute.setOffset(startOffset, endOffset);
+        lastStartOffset = startOffset;
+
+        if (termPart == null) {
+          termAttribute.copyBuffer(savedTermBuffer, startPart, endPart - startPart);
+        } else {
+          termAttribute.copyBuffer(termPart, 0, termPart.length);
+        }
+
+        posIncAttribute.setPositionIncrement(accumPosInc + startPos - wordPos);
+        accumPosInc = 0;
+        posLenAttribute.setPositionLength(endPos - startPos);
+        wordPos = startPos;
+        return true;
+      }
+
+      // no saved concatenations, on to the next input word
+      savedState = null;
+    }
+  }
+
+  // ================================================= Helper Methods ================================================
+
+  private class PositionSorter extends InPlaceMergeSorter {
+    @Override
+    protected int compare(int i, int j) {
+      // sort by smaller start position
+      int iPosStart = bufferedParts[4*i];
+      int jPosStart = bufferedParts[4*j];
+      int cmp = Integer.compare(iPosStart, jPosStart);
+      if (cmp != 0) {
+        return cmp;
+      }
+
+      // tie break by longest pos length:
+      int iPosEnd = bufferedParts[4*i+1];
+      int jPosEnd = bufferedParts[4*j+1];
+      return Integer.compare(jPosEnd, iPosEnd);
+    }
+
+    @Override
+    protected void swap(int i, int j) {
+      int iOffset = 4*i;
+      int jOffset = 4*j;
+      for(int x=0;x<4;x++) {
+        int tmp = bufferedParts[iOffset+x];
+        bufferedParts[iOffset+x] = bufferedParts[jOffset+x];
+        bufferedParts[jOffset+x] = tmp;
+      }
+
+      char[] tmp2 = bufferedTermParts[i];
+      bufferedTermParts[i] = bufferedTermParts[j];
+      bufferedTermParts[j] = tmp2;
+    }
+  }
+
+  final PositionSorter sorter = new PositionSorter();
+
+  /**
+   * startPos, endPos -> graph start/end position
+   * startPart, endPart -> slice of the original term for this part
+   */
+
+  void buffer(int startPos, int endPos, int startPart, int endPart) {
+    buffer(null, startPos, endPos, startPart, endPart);
+  }
+
+  /**
+   * a null termPart means it's a simple slice of the original term
+   */
+  void buffer(char[] termPart, int startPos, int endPos, int startPart, int endPart) {
+    /*
+    System.out.println("buffer: pos=" + startPos + "-" + endPos + " part=" + startPart + "-" + endPart);
+    if (termPart != null) {
+      System.out.println("  termIn=" + new String(termPart));
+    } else {
+      System.out.println("  term=" + new String(savedTermBuffer, startPart, endPart-startPart));
+    }
+    */
+    assert endPos > startPos: "startPos=" + startPos + " endPos=" + endPos;
+    assert endPart > startPart || (endPart == 0 && startPart == 0 && savedTermLength == 0): "startPart=" + startPart + " endPart=" + endPart;
+    if ((bufferedLen+1)*4 > bufferedParts.length) {
+      bufferedParts = ArrayUtil.grow(bufferedParts, (bufferedLen+1)*4);
+    }
+    if (bufferedTermParts.length == bufferedLen) {
+      int newSize = ArrayUtil.oversize(bufferedLen+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
+      char[][] newArray = new char[newSize][];
+      System.arraycopy(bufferedTermParts, 0, newArray, 0, bufferedTermParts.length);
+      bufferedTermParts = newArray;
+    }
+    bufferedTermParts[bufferedLen] = termPart;
+    bufferedParts[bufferedLen*4] = startPos;
+    bufferedParts[bufferedLen*4+1] = endPos;
+    bufferedParts[bufferedLen*4+2] = startPart;
+    bufferedParts[bufferedLen*4+3] = endPart;
+    bufferedLen++;
+  }
+
+  /**
+   * Saves the existing attribute states
+   */
+  private void saveState() {
+    savedTermLength = termAttribute.length();
+    savedStartOffset = offsetAttribute.startOffset();
+    savedEndOffset = offsetAttribute.endOffset();
+    savedState = captureState();
+
+    if (savedTermBuffer.length < savedTermLength) {
+      savedTermBuffer = new char[ArrayUtil.oversize(savedTermLength, Character.BYTES)];
+    }
+
+    System.arraycopy(termAttribute.buffer(), 0, savedTermBuffer, 0, savedTermLength);
+  }
+
+  /**
+   * Flushes the given WordDelimiterConcatenation by either writing its concat and then clearing, or just clearing.
+   *
+   * @param concat WordDelimiterConcatenation that will be flushed
+   */
+  private void flushConcatenation(WordDelimiterConcatenation concat) {
+    if (wordPos == concat.startPos) {
+      // we are not generating parts, so we must advance wordPos now
+      wordPos++;
+    }
+    lastConcatCount = concat.subwordCount;
+    if (concat.subwordCount != 1 || shouldGenerateParts(concat.type) == false) {
+      concat.write();
+    }
+    concat.clear();
+  }
+
+  /**
+   * Determines whether to concatenate a word or number if the current word is the given type
+   *
+   * @param wordType Type of the current word used to determine if it should be concatenated
+   * @return {@code true} if concatenation should occur, {@code false} otherwise
+   */
+  private boolean shouldConcatenate(int wordType) {
+    return (has(CATENATE_WORDS) && WordDelimiterIterator.isAlpha(wordType)) || (has(CATENATE_NUMBERS) && WordDelimiterIterator.isDigit(wordType));
+  }
+
+  /**
+   * Determines whether a word/number part should be generated for a word of the given type
+   *
+   * @param wordType Type of the word used to determine if a word/number part should be generated
+   * @return {@code true} if a word/number part should be generated, {@code false} otherwise
+   */
+  private boolean shouldGenerateParts(int wordType) {
+    return (has(GENERATE_WORD_PARTS) && WordDelimiterIterator.isAlpha(wordType)) || (has(GENERATE_NUMBER_PARTS) && WordDelimiterIterator.isDigit(wordType));
+  }
+
+  /**
+   * Concatenates the saved buffer to the given WordDelimiterConcatenation
+   *
+   * @param concatenation WordDelimiterConcatenation to concatenate the buffer to
+   */
+  private void concatenate(WordDelimiterConcatenation concatenation) {
+    if (concatenation.isEmpty()) {
+      concatenation.type = iterator.type();
+      concatenation.startPart = iterator.current;
+      concatenation.startPos = wordPos;
+    }
+    concatenation.append(savedTermBuffer, iterator.current, iterator.end - iterator.current);
+    concatenation.endPart = iterator.end;
+  }
+
+
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    inputTokenizer.setReader(input);
+    inputTokenizer.reset();
+    accumPosInc = 0;
+    savedState = null;
+    lastStartOffset = 0;
+    concat.clear();
+    concatAll.clear();
+  }
+
+  @Override
+  public void close() throws IOException {
+    super.close();
+    inputTokenizer.close();
+  }
+
+  @Override
+  public void end() throws IOException {
+    super.end();
+    inputTokenizer.end();
+    endAttributes();
+    inputTokenizer.copyTo(this);
+  }
+
+  /**
+   * Determines whether the given flag is set
+   *
+   * @param flag Flag to see if set
+   * @return {@code true} if flag is set
+   */
+  private boolean has(int flag) {
+    return (flags & flag) != 0;
+  }
+
+  /**
+   * A WDF concatenated 'run'
+   */
+  private final class WordDelimiterConcatenation {
+    final StringBuilder buffer = new StringBuilder();
+    int startPart;
+    int endPart;
+    int startPos;
+    int type;
+    int subwordCount;
+
+    /**
+     * Appends the given text of the given length, to the concetenation at the given offset
+     *
+     * @param text Text to append
+     * @param offset Offset in the concetenation to add the text
+     * @param length Length of the text to append
+     */
+    void append(char text[], int offset, int length) {
+      buffer.append(text, offset, length);
+      subwordCount++;
+    }
+
+    /**
+     * Writes the concatenation to part buffer
+     */
+    void write() {
+      char[] termPart = new char[buffer.length()];
+      buffer.getChars(0, buffer.length(), termPart, 0);
+      buffer(termPart, startPos, wordPos, startPart, endPart);
+    }
+
+    /**
+     * Determines if the concatenation is empty
+     *
+     * @return {@code true} if the concatenation is empty, {@code false} otherwise
+     */
+    boolean isEmpty() {
+      return buffer.length() == 0;
+    }
+
+    boolean isNotEmpty() {
+      return isEmpty() == false;
+    }
+
+    /**
+     * Clears the concatenation and resets its state
+     */
+    void clear() {
+      buffer.setLength(0);
+      startPart = endPart = type = subwordCount = 0;
+    }
+  }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterTokenizer.java
new file mode 100644
index 0000000000..3f9ff4a5cf
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterTokenizer.java
@@ -0,0 +1,876 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.miscellaneous;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
+
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.CATENATE_ALL;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.CATENATE_NUMBERS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.CATENATE_WORDS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.GENERATE_NUMBER_PARTS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.GENERATE_WORD_PARTS;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.PRESERVE_ORIGINAL;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.SPLIT_ON_CASE_CHANGE;
+import static org.apache.lucene.analysis.miscellaneous.WordDelimiterTokenizer.SPLIT_ON_NUMERICS;
+
+/**
+ * New WordDelimiterGraphFilter tests... most of the tests are in ConvertedLegacyTest
+ * TODO: should explicitly test things like protWords and not rely on
+ * the factory tests in Solr.
+ */
+public class TestWordDelimiterTokenizer extends BaseTokenStreamTestCase {
+
+  static Tokenizer tokenizer(Token... tokens) {
+    return new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) {
+      OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+      int upto = -1;
+      @Override
+      public final boolean incrementToken() throws IOException {
+        clearAttributes();
+        if (upto >= tokens.length - 1)
+          return false;
+        upto++;
+        tokens[upto].copyTo((Token)offsetAtt);
+        return true;
+      }
+
+      @Override
+      public void reset() throws IOException {
+        super.reset();
+        upto = -1;
+      }
+    };
+  }
+
+  public void testOffsets() throws IOException {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    // test that subwords and catenated subwords have
+    // the correct offsets.
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(new Token("foo-bar", 5, 12)), flags);
+
+    assertTokenStreamContents(wdf,
+        new String[] { "foobar", "foo", "bar" },
+        new int[] { 5, 5, 9 },
+        new int[] { 12, 8, 12 });
+
+    // with illegal offsets:
+    wdf = new WordDelimiterTokenizer(tokenizer(new Token("foo-bar", 5, 6)), flags);
+    assertTokenStreamContents(wdf,
+        new String[] { "foobar", "foo", "bar" },
+        new int[] { 5, 5, 5 },
+        new int[] { 6, 6, 6 });
+  }
+
+  public void testOffsetChange() throws Exception {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(new Token("übelkeit)", 7, 16)), flags);
+
+    assertTokenStreamContents(wdf,
+        new String[] { "übelkeit" },
+        new int[] { 7 },
+        new int[] { 15 });
+  }
+
+  public void testOffsetChange2() throws Exception {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(new Token("(übelkeit", 7, 17)), flags);
+    // illegal offsets:
+    assertTokenStreamContents(wdf,
+        new String[] { "übelkeit" },
+        new int[] { 7 },
+        new int[] { 17 });
+  }
+
+  public void testOffsetChange3() throws Exception {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(new Token("(übelkeit", 7, 16)), flags);
+    assertTokenStreamContents(wdf,
+        new String[] { "übelkeit" },
+        new int[] { 8 },
+        new int[] { 16 });
+  }
+
+  public void testOffsetChange4() throws Exception {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(new Token("(foo,bar)", 7, 16)), flags);
+
+    assertTokenStreamContents(wdf,
+        new String[] { "foobar", "foo", "bar"},
+        new int[] { 8, 8, 12 },
+        new int[] { 15, 11, 15 });
+  }
+
+  public void doSplit(final String input, String... output) throws Exception {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(new MockTokenizer(MockTokenizer.KEYWORD, false), flags);
+    wdf.setReader(new StringReader(input));
+    assertTokenStreamContents(wdf, output);
+  }
+
+  public void testSplits() throws Exception {
+    doSplit("basic-split","basic","split");
+    doSplit("camelCase","camel","Case");
+
+    // non-space marking symbol shouldn't cause split
+    // this is an example in Thai
+    doSplit("\u0e1a\u0e49\u0e32\u0e19","\u0e1a\u0e49\u0e32\u0e19");
+
+    // some russian upper and lowercase
+    doSplit("Роберт", "Роберт");
+    // now cause a split (russian camelCase)
+    doSplit("РобЕрт", "Роб", "Ерт");
+
+    // a composed titlecase character, don't split
+    doSplit("aǅungla", "aǅungla");
+
+    // a modifier letter, don't split
+    doSplit("ســـــــــــــــــلام", "ســـــــــــــــــلام");
+
+    // enclosing mark, don't split
+    doSplit("test⃝", "test⃝");
+
+    // combining spacing mark (the virama), don't split
+    doSplit("हिन्दी", "हिन्दी");
+
+    // don't split non-ascii digits
+    doSplit("١٢٣٤", "١٢٣٤");
+
+    // don't split supplementaries into unpaired surrogates
+    doSplit("𠀀𠀀", "𠀀𠀀");
+  }
+
+  public void testTokenType() throws Exception {
+    int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    // test that subwords and catenated subwords have
+    // the correct offsets.
+    Token token = new Token("foo-bar", 5, 12);
+    token.setType("mytype");
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(token), flags);
+
+    assertTokenStreamContents(wdf,
+        new String[] {"foobar", "foo", "bar"},
+        new String[] {"mytype", "mytype", "mytype"});
+  }
+
+  /*
+   * Set a large position increment gap of 10 if the token is "largegap"
+   */
+  private static final class LargePosIncTokenFilter extends TokenFilter {
+    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+
+    protected LargePosIncTokenFilter(TokenStream input) {
+      super(input);
+    }
+
+    @Override
+    public boolean incrementToken() throws IOException {
+      if (input.incrementToken()) {
+        if (termAtt.toString().equals("largegap"))
+          posIncAtt.setPositionIncrement(10);
+        return true;
+      } else {
+        return false;
+      }
+    }
+  }
+
+  public void testPositionIncrements() throws Exception {
+
+    Analyzer a4 = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        final int flags = SPLIT_ON_NUMERICS | GENERATE_WORD_PARTS | PRESERVE_ORIGINAL | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE;
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags);
+        return new TokenStreamComponents(tokenizer);
+      }
+    };
+    assertAnalyzesTo(a4, "SAL_S8371 - SAL",
+        new String[]{ "SAL_S8371", "SAL", "S", "8371", "-", "SAL"},
+        new int[]{    1,            0,    1,    1,      1,    1});
+
+    final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+    final CharArraySet protWords = new CharArraySet(Collections.singleton("NUTCH"), false);
+
+    /* analyzer that uses whitespace + wdf */
+    Analyzer a = new Analyzer() {
+      @Override
+      public TokenStreamComponents createComponents(String field) {
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags, protWords);
+        return new TokenStreamComponents(tokenizer);
+      }
+    };
+
+    /* in this case, works as expected. */
+    assertAnalyzesTo(a, "LUCENE / SOLR", new String[] { "LUCENE", "SOLR" },
+        new int[] { 0, 9 },
+        new int[] { 6, 13 },
+        null,
+        new int[] { 1, 2 },
+        null,
+        false);
+
+    /* only in this case, posInc of 2 ?! */
+    assertAnalyzesTo(a, "LUCENE / solR", new String[] { "LUCENE", "solR", "sol", "R" },
+        new int[] { 0, 9, 9, 12 },
+        new int[] { 6, 13, 12, 13 },
+        null,
+        new int[] { 1, 2, 0, 1 },
+        null,
+        false);
+
+    assertAnalyzesTo(a, "LUCENE / NUTCH SOLR", new String[] { "LUCENE", "NUTCH", "SOLR" },
+        new int[] { 0, 9, 15 },
+        new int[] { 6, 14, 19 },
+        null,
+        new int[] { 1, 2, 1 },
+        null,
+        false);
+
+    /* analyzer that will consume tokens with large position increments */
+    Analyzer a2 = new Analyzer() {
+      @Override
+      public TokenStreamComponents createComponents(String field) {
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags, protWords);
+        return new TokenStreamComponents(tokenizer, new LargePosIncTokenFilter(tokenizer));
+      }
+    };
+
+    /* increment of "largegap" is preserved */
+    assertAnalyzesTo(a2, "LUCENE largegap SOLR", new String[] { "LUCENE", "largegap", "SOLR" },
+        new int[] { 0, 7, 16 },
+        new int[] { 6, 15, 20 },
+        null,
+        new int[] { 1, 10, 1 },
+        null,
+        false);
+
+    Analyzer a3 = new Analyzer() {
+      @Override
+      public TokenStreamComponents createComponents(String field) {
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags, protWords);
+        StopFilter filter = new StopFilter(tokenizer, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
+        return new TokenStreamComponents(tokenizer, filter);
+      }
+    };
+
+    assertAnalyzesTo(a3, "lucene.solr",
+        new String[] { "lucenesolr", "lucene", "solr" },
+        new int[] { 0, 0, 7 },
+        new int[] { 11, 6, 11 },
+        null,
+        new int[] { 1, 0, 1 },
+        null,
+        false);
+
+    /* the stopword should add a gap here */
+    assertAnalyzesTo(a3, "the lucene.solr",
+        new String[] { "lucenesolr", "lucene", "solr" },
+        new int[] { 4, 4, 11 },
+        new int[] { 15, 10, 15 },
+        null,
+        new int[] { 2, 0, 1 },
+        null,
+        false);
+
+    IOUtils.close(a, a2, a3, a4);
+  }
+
+  /** concat numbers + words + all */
+  public void testLotsOfConcatenating() throws Exception {
+    final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+
+    /* analyzer that uses whitespace + wdf */
+    Analyzer a = new Analyzer() {
+      @Override
+      public TokenStreamComponents createComponents(String field) {
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags);
+        return new TokenStreamComponents(tokenizer);
+      }
+    };
+
+    assertAnalyzesTo(a, "abc-def-123-456",
+        new String[] { "abcdef123456", "abcdef", "abc", "def", "123456", "123", "456" },
+        new int[] { 0, 0, 0, 4, 8, 8, 12 },
+        new int[] { 15, 7, 3, 7, 15, 11, 15 },
+        null,
+        new int[] { 1, 0, 0, 1, 1, 0, 1 },
+        null,
+        false);
+    a.close();
+  }
+
+  /** concat numbers + words + all + preserve original */
+  public void testLotsOfConcatenating2() throws Exception {
+    final int flags = PRESERVE_ORIGINAL | GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+
+    /* analyzer that uses whitespace + wdf */
+    Analyzer a = new Analyzer() {
+      @Override
+      public TokenStreamComponents createComponents(String field) {
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags);
+        return new TokenStreamComponents(tokenizer);
+      }
+    };
+
+    assertAnalyzesTo(a, "abc-def-123-456",
+        new String[] { "abcdef123456", "abc-def-123-456", "abcdef", "abc", "def", "123456", "123", "456" },
+        new int[] { 0, 0, 0, 0, 4, 8, 8, 12 },
+        new int[] { 15, 15, 7, 3, 7, 15, 11, 15 },
+        null,
+        new int[] { 1, 0, 0, 0, 1, 1, 0, 1 },
+        null,
+        false);
+    a.close();
+  }
+
+  /** blast some random strings through the analyzer */
+  public void testRandomStrings() throws Exception {
+    int numIterations = atLeast(5);
+    for (int i = 0; i < numIterations; i++) {
+      final int flags = random().nextInt(512);
+      final CharArraySet protectedWords;
+      if (random().nextBoolean()) {
+        protectedWords = new CharArraySet(new HashSet<>(Arrays.asList("a", "b", "cd")), false);
+      } else {
+        protectedWords = new CharArraySet(2, false);
+      }
+
+      Analyzer a = new Analyzer() {
+
+        @Override
+        protected TokenStreamComponents createComponents(String fieldName) {
+          Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags, protectedWords);
+          return new TokenStreamComponents(tokenizer);
+        }
+      };
+      // TODO: properly support positionLengthAttribute
+      checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 20, false, false);
+      a.close();
+    }
+  }
+
+  /** blast some enormous random strings through the analyzer */
+  public void testRandomHugeStrings() throws Exception {
+    int numIterations = atLeast(5);
+    for (int i = 0; i < numIterations; i++) {
+      final int flags = random().nextInt(512);
+      final CharArraySet protectedWords;
+      if (random().nextBoolean()) {
+        protectedWords = new CharArraySet(new HashSet<>(Arrays.asList("a", "b", "cd")), false);
+      } else {
+        protectedWords = new CharArraySet(2, false);
+      }
+
+      Analyzer a = new Analyzer() {
+
+        @Override
+        protected TokenStreamComponents createComponents(String fieldName) {
+          Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags, protectedWords);
+          return new TokenStreamComponents(tokenizer);
+        }
+      };
+      // TODO: properly support positionLengthAttribute
+      checkRandomData(random(), a, 20*RANDOM_MULTIPLIER, 8192, false, false);
+      a.close();
+    }
+  }
+
+  public void testEmptyTerm() throws IOException {
+    Random random = random();
+    for (int i = 0; i < 512; i++) {
+      final int flags = i;
+      final CharArraySet protectedWords;
+      if (random.nextBoolean()) {
+        protectedWords = new CharArraySet(new HashSet<>(Arrays.asList("a", "b", "cd")), false);
+      } else {
+        protectedWords = new CharArraySet(2, false);
+      }
+
+      Analyzer a = new Analyzer() {
+        @Override
+        protected TokenStreamComponents createComponents(String fieldName) {
+          Tokenizer tokenizer = new WordDelimiterTokenizer(new WhitespaceTokenizer(), flags, protectedWords);
+          return new TokenStreamComponents(tokenizer);
+        }
+      };
+      // depending upon options, this thing may or may not preserve the empty term
+      checkAnalysisConsistency(random, a, random.nextBoolean(), "");
+      a.close();
+    }
+  }
+
+  private Analyzer getAnalyzer(int flags) {
+    return new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        Tokenizer tokenizer = new WordDelimiterTokenizer(new KeywordTokenizer(), flags);
+        return new TokenStreamComponents(tokenizer);
+      }
+    };
+  }
+
+  private static boolean has(int flags, int flag) {
+    return (flags & flag) != 0;
+  }
+
+  private static class WordPart {
+    final String part;
+    final int startOffset;
+    final int endOffset;
+    final int type;
+
+    public WordPart(String text, int startOffset, int endOffset) {
+      this.part = text.substring(startOffset, endOffset);
+      this.startOffset = startOffset;
+      this.endOffset = endOffset;
+      this.type = toType(part.charAt(0));
+    }
+
+    @Override
+    public String toString() {
+      return "WordPart(" + part + " " + startOffset + "-" + endOffset + ")";
+    }
+  }
+
+  private static final int NUMBER = 0;
+  private static final int LETTER = 1;
+  private static final int DELIM = 2;
+
+  private static int toType(char ch) {
+    if (Character.isDigit(ch)) {
+      // numbers
+      return NUMBER;
+    } else if (Character.isLetter(ch)) {
+      // letters
+      return LETTER;
+    } else {
+      // delimiter
+      return DELIM;
+    }
+  }
+
+  /** Does (hopefully) the same thing as WordDelimiterGraphFilter, according to the flags, but more slowly, returning all string paths combinations. */
+  private Set<String> slowWDF(String text, int flags) {
+
+    // first make word parts:
+    List<WordPart> wordParts = new ArrayList<>();
+    int lastCH = -1;
+    int wordPartStart = 0;
+    boolean inToken = false;
+
+    for(int i=0;i<text.length();i++) {
+      char ch = text.charAt(i);
+      if (toType(ch) == DELIM) {
+        // delimiter
+        if (inToken) {
+          // end current token
+          wordParts.add(new WordPart(text, wordPartStart, i));
+          inToken = false;
+        }
+      } else if (inToken == false) {
+        // start new token
+        inToken = true;
+        wordPartStart = i;
+      } else {
+        boolean newToken = false;
+        if (Character.isLetter(lastCH)) {
+          if (Character.isLetter(ch)) {
+            if (has(flags, SPLIT_ON_CASE_CHANGE) && Character.isLowerCase(lastCH) && Character.isLowerCase(ch) == false) {
+              // start new token on lower -> UPPER case change (but not vice versa!)
+              newToken = true;
+            }
+          } else if (has(flags, SPLIT_ON_NUMERICS) && Character.isDigit(ch)) {
+            // start new token on letter -> number change
+            newToken = true;
+          }
+        } else {
+          assert Character.isDigit(lastCH);
+          if (Character.isLetter(ch) && has(flags, SPLIT_ON_NUMERICS) ) {
+            // start new token on number -> letter change
+            newToken = true;
+          }
+        }
+        if (newToken) {
+          wordParts.add(new WordPart(text, wordPartStart, i));
+          wordPartStart = i;
+        }
+      }
+      lastCH = ch;
+    }
+
+    if (inToken) {
+      // add last token
+      wordParts.add(new WordPart(text, wordPartStart, text.length()));
+    }
+
+    Set<String> paths = new HashSet<>();
+    if (wordParts.isEmpty() == false) {
+      enumerate(flags, 0, wordParts, paths, new StringBuilder());
+    }
+
+    if (has(flags, PRESERVE_ORIGINAL)) {
+      paths.add(text);
+    }
+
+    if (has(flags, CATENATE_ALL) && wordParts.isEmpty() == false) {
+      StringBuilder b = new StringBuilder();
+      for(WordPart wordPart : wordParts) {
+        b.append(wordPart.part);
+      }
+      paths.add(b.toString());
+    }
+
+    return paths;
+  }
+
+  private void add(StringBuilder path, String part) {
+    if (path.length() != 0) {
+      path.append(' ');
+    }
+    path.append(part);
+  }
+
+  private void add(StringBuilder path, List<WordPart> wordParts, int from, int to) {
+    if (path.length() != 0) {
+      path.append(' ');
+    }
+    // no spaces:
+    for(int i=from;i<to;i++) {
+      path.append(wordParts.get(i).part);
+    }
+  }
+
+  private void addWithSpaces(StringBuilder path, List<WordPart> wordParts, int from, int to) {
+    for(int i=from;i<to;i++) {
+      add(path, wordParts.get(i).part);
+    }
+  }
+
+  /** Finds the end (exclusive) of the series of part with the same type */
+  private int endOfRun(List<WordPart> wordParts, int start) {
+    int upto = start+1;
+    while(upto < wordParts.size() && wordParts.get(upto).type == wordParts.get(start).type) {
+      upto++;
+    }
+    return upto;
+  }
+
+  /** Recursively enumerates all paths through the word parts */
+  private void enumerate(int flags, int upto, List<WordPart> wordParts, Set<String> paths, StringBuilder path) {
+    if (upto == wordParts.size()) {
+      if (path.length() > 0) {
+        paths.add(path.toString());
+      }
+    } else {
+      int savLength = path.length();
+      int end = endOfRun(wordParts, upto);
+
+      if (wordParts.get(upto).type == NUMBER) {
+        // always output single word, optionally surrounded by delims:
+        if (has(flags, GENERATE_NUMBER_PARTS) || wordParts.size() == 1) {
+          addWithSpaces(path, wordParts, upto, end);
+          if (has(flags, CATENATE_NUMBERS)) {
+            // recurse first with the parts
+            enumerate(flags, end, wordParts, paths, path);
+            path.setLength(savLength);
+            // .. and second with the concat
+            add(path, wordParts, upto, end);
+          }
+        } else if (has(flags, CATENATE_NUMBERS)) {
+          add(path, wordParts, upto, end);
+        }
+        enumerate(flags, end, wordParts, paths, path);
+        path.setLength(savLength);
+      } else {
+        assert wordParts.get(upto).type == LETTER;
+        // always output single word, optionally surrounded by delims:
+        if (has(flags, GENERATE_WORD_PARTS) || wordParts.size() == 1) {
+          addWithSpaces(path, wordParts, upto, end);
+          if (has(flags, CATENATE_WORDS)) {
+            // recurse first with the parts
+            enumerate(flags, end, wordParts, paths, path);
+            path.setLength(savLength);
+            // .. and second with the concat
+            add(path, wordParts, upto, end);
+          }
+        } else if (has(flags, CATENATE_WORDS)) {
+          add(path, wordParts, upto, end);
+        }
+        enumerate(flags, end, wordParts, paths, path);
+        path.setLength(savLength);
+      }
+    }
+  }
+
+  public void testBasicGraphSplits() throws Exception {
+    assertGraphStrings(getAnalyzer(0),
+        "PowerShotPlus",
+        "PowerShotPlus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS),
+        "PowerShotPlus",
+        "PowerShotPlus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE),
+        "PowerShotPlus",
+        "Power Shot Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE | PRESERVE_ORIGINAL),
+        "PowerShotPlus",
+        "PowerShotPlus",
+        "Power Shot Plus");
+
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS),
+        "Power-Shot-Plus",
+        "Power Shot Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE),
+        "Power-Shot-Plus",
+        "Power Shot Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE | PRESERVE_ORIGINAL),
+        "Power-Shot-Plus",
+        "Power-Shot-Plus",
+        "Power Shot Plus");
+
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE),
+        "PowerShotPlus",
+        "Power Shot Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE),
+        "PowerShot1000Plus",
+        "Power Shot1000Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE),
+        "Power-Shot-Plus",
+        "Power Shot Plus");
+
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE | CATENATE_WORDS),
+        "PowerShotPlus",
+        "Power Shot Plus",
+        "PowerShotPlus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE | CATENATE_WORDS),
+        "PowerShot1000Plus",
+        "Power Shot1000Plus",
+        "PowerShot1000Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | CATENATE_WORDS | CATENATE_NUMBERS),
+        "Power-Shot-1000-17-Plus",
+        "Power Shot 1000 17 Plus",
+        "Power Shot 100017 Plus",
+        "PowerShot 1000 17 Plus",
+        "PowerShot 100017 Plus");
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | CATENATE_WORDS | CATENATE_NUMBERS | PRESERVE_ORIGINAL),
+        "Power-Shot-1000-17-Plus",
+        "Power-Shot-1000-17-Plus",
+        "Power Shot 1000 17 Plus",
+        "Power Shot 100017 Plus",
+        "PowerShot 1000 17 Plus",
+        "PowerShot 100017 Plus");
+  }
+
+  private String randomWDFText() {
+    StringBuilder b = new StringBuilder();
+    int length = TestUtil.nextInt(random(), 1, 50);
+    for(int i=0;i<length;i++) {
+      int surpriseMe = random().nextInt(37);
+      int lower = -1;
+      int upper = -1;
+      if (surpriseMe < 10) {
+        // lowercase letter
+        lower = 'a';
+        upper = 'z';
+      } else if (surpriseMe < 20) {
+        // uppercase letter
+        lower = 'A';
+        upper = 'Z';
+      } else if (surpriseMe < 30) {
+        // digit
+        lower = '0';
+        upper = '9';
+      } else if (surpriseMe < 35) {
+        // punct
+        lower = '-';
+        upper = '-';
+      } else {
+        b.append("'s");
+      }
+
+      if (lower != -1) {
+        b.append((char) TestUtil.nextInt(random(), lower, upper));
+      }
+    }
+
+    return b.toString();
+  }
+
+  public void testInvalidFlag() throws Exception {
+    expectThrows(IllegalArgumentException.class,
+        () -> {
+          new WordDelimiterGraphFilter(new CannedTokenStream(), 1 << 31, null);
+        });
+  }
+
+  public void testRandomPaths() throws Exception {
+    int iters = atLeast(100);
+    for(int iter=0;iter<iters;iter++) {
+      String text = randomWDFText();
+      if (VERBOSE) {
+        System.out.println("\nTEST: text=" + text + " len=" + text.length());
+      }
+
+      int flags = 0;
+      if (random().nextBoolean()) {
+        flags |= GENERATE_WORD_PARTS;
+      }
+      if (random().nextBoolean()) {
+        flags |= GENERATE_NUMBER_PARTS;
+      }
+      if (random().nextBoolean()) {
+        flags |= CATENATE_WORDS;
+      }
+      if (random().nextBoolean()) {
+        flags |= CATENATE_NUMBERS;
+      }
+      if (random().nextBoolean()) {
+        flags |= CATENATE_ALL;
+      }
+      if (random().nextBoolean()) {
+        flags |= PRESERVE_ORIGINAL;
+      }
+      if (random().nextBoolean()) {
+        flags |= SPLIT_ON_CASE_CHANGE;
+      }
+      if (random().nextBoolean()) {
+        flags |= SPLIT_ON_NUMERICS;
+      }
+
+      verify(text, flags);
+    }
+  }
+
+  /** Runs normal and slow WDGF and compares results */
+  private void verify(String text, int flags) throws IOException {
+
+    Set<String> expected = slowWDF(text, flags);
+    if (VERBOSE) {
+      for(String path : expected) {
+        System.out.println("  " + path);
+      }
+    }
+
+    Set<String> actual = getGraphStrings(getAnalyzer(flags), text);
+    if (actual.equals(expected) == false) {
+      StringBuilder b = new StringBuilder();
+      b.append("\n\nFAIL: text=");
+      b.append(text);
+      b.append(" flags=");
+      b.append(WordDelimiterGraphFilter.flagsToString(flags));
+      b.append('\n');
+      b.append("  expected paths:\n");
+      for (String s : expected) {
+        b.append("    ");
+        b.append(s);
+        if (actual.contains(s) == false) {
+          b.append(" [missing!]");
+        }
+        b.append('\n');
+      }
+
+      b.append("  actual paths:\n");
+      for (String s : actual) {
+        b.append("    ");
+        b.append(s);
+        if (expected.contains(s) == false) {
+          b.append(" [unexpected!]");
+        }
+        b.append('\n');
+      }
+
+      fail(b.toString());
+    }
+  }
+
+  public void testOnlyNumbers() throws Exception {
+    // no token should be produced
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS), "7-586");
+  }
+
+  public void testNoCatenate() throws Exception {
+    // no token should be produced
+    assertGraphStrings(getAnalyzer(GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS), "a-b-c-9-d", "a b c 9 d");
+  }
+
+  public void testCuriousCase1() throws Exception {
+    verify("u-0L-4836-ip4Gw--13--q7--L07E1", CATENATE_WORDS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS);
+  }
+
+  public void testCuriousCase2() throws Exception {
+    verify("u-l-p", CATENATE_ALL);
+  }
+
+  public void testOriginalPosLength() throws Exception {
+    verify("Foo-Bar-Baz", CATENATE_WORDS | SPLIT_ON_CASE_CHANGE | PRESERVE_ORIGINAL);
+  }
+
+  public void testCuriousCase3() throws Exception {
+    verify("cQzk4-GL0izl0mKM-J8--4m-'s", GENERATE_NUMBER_PARTS | CATENATE_NUMBERS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS);
+  }
+
+  public void testEmptyString() throws Exception {
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokenizer(new Token("", 0, 0)),
+        GENERATE_WORD_PARTS | CATENATE_ALL | PRESERVE_ORIGINAL);
+    wdf.reset();
+    assertTrue(wdf.incrementToken());
+    assertFalse(wdf.incrementToken());
+    wdf.end();
+    wdf.close();
+  }
+
+  public void testProtectedWords() throws Exception {
+    Tokenizer tokens = tokenizer(new Token("foo17-bar", 0, 9),
+        new Token("foo-bar", 0, 7));
+
+    CharArraySet protectedWords = new CharArraySet(new HashSet<>(Arrays.asList("foo17-BAR")), true);
+    WordDelimiterTokenizer wdf = new WordDelimiterTokenizer(tokens, GENERATE_WORD_PARTS | PRESERVE_ORIGINAL | CATENATE_ALL,
+        protectedWords);
+    assertGraphStrings(wdf,
+        "foo17-bar foo bar",
+        "foo17-bar foo-bar",
+        "foo17-bar foobar");
+  }
+}
