Index: lucene/analysis/common/build.xml
===================================================================
--- lucene/analysis/common/build.xml	(revision 1713947)
+++ lucene/analysis/common/build.xml	(working copy)
@@ -31,6 +31,8 @@
   
   <property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>  
   
+  <property name="unicode-props-file" location="src/java/org/apache/lucene/analysis/util/UnicodeProps.java"/>
+
   <target name="jflex" depends="-install-jflex,clean-jflex,-jflex-StandardAnalyzer,-jflex-UAX29URLEmailTokenizer,
                                 -jflex-wiki-tokenizer,-jflex-HTMLStripCharFilter"/>
 
@@ -114,6 +116,18 @@
     </delete>
   </target>
   
+  <target xmlns:ivy="antlib:org.apache.ivy.ant" name="-resolve-icu4j" unless="icu4j.resolved" depends="ivy-availability-check,ivy-configure">
+    <loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
+    <ivy:cachepath organisation="com.ibm.icu" module="icu4j" revision="${ivyversions./com.ibm.icu/icu4j}"
+      inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
+    <property name="icu4j.resolved" value="true"/>
+  </target>
+  
+  <target name="unicode-data" depends="-resolve-icu4j,resolve-groovy">
+    <groovy classpathref="icu4j.classpath" src="src/tools/groovy/generate-unicode-data.groovy"/>
+    <fixcrlf file="${unicode-props-file}" encoding="UTF-8"/>
+  </target>
+  
   <property name="tld.zones" value="http://www.internic.net/zones/root.zone"/>
   <property name="tld.output" location="src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro"/>
 
@@ -141,7 +155,7 @@
 
   <target name="javadocs" depends="module-build.javadocs"/>
 
-  <target name="regenerate" depends="jflex"/>
+  <target name="regenerate" depends="jflex,unicode-data"/>
   
   <target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
       <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java	(working copy)
@@ -0,0 +1,37 @@
+package org.apache.lucene.analysis.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+
+/**
+ * An Analyzer that uses {@link UnicodeWhitespaceTokenizer}.
+ **/
+public final class UnicodeWhitespaceAnalyzer extends Analyzer {
+  
+  /**
+   * Creates a new {@link UnicodeWhitespaceAnalyzer}
+   */
+  public UnicodeWhitespaceAnalyzer() {
+  }
+  
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName) {
+    return new TokenStreamComponents(new UnicodeWhitespaceTokenizer());
+  }
+}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java	(working copy)

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceAnalyzer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,57 @@
+package org.apache.lucene.analysis.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.analysis.util.UnicodeProps;
+import org.apache.lucene.util.AttributeFactory;
+
+/**
+ * A UnicodeWhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * Adjacent sequences of non-Whitespace characters form tokens (according to
+ * Unicode's WHITESPACE property).
+ * <p>
+ * <em>For Unicode version see: {@link UnicodeProps}</em>
+ */
+public final class UnicodeWhitespaceTokenizer extends CharTokenizer {
+  
+  /**
+   * Construct a new UnicodeWhitespaceTokenizer.
+   */
+  public UnicodeWhitespaceTokenizer() {
+  }
+
+  /**
+   * Construct a new UnicodeWhitespaceTokenizer using a given
+   * {@link org.apache.lucene.util.AttributeFactory}.
+   *
+   * @param factory
+   *          the attribute factory to use for this {@link Tokenizer}
+   */
+  public UnicodeWhitespaceTokenizer(AttributeFactory factory) {
+    super(factory);
+  }
+  
+  /** Collects only characters which do not satisfy Unicode's WHITESPACE property. */
+  @Override
+  protected boolean isTokenChar(int c) {
+    return !UnicodeProps.WHITESPACE.get(c);
+  }
+  
+}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(working copy)

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java	(revision 1713947)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java	(working copy)
@@ -22,8 +22,11 @@
 import org.apache.lucene.util.AttributeFactory;
 
 /**
- * A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * A tokenizer that divides text at whitespace characters as defined by
+ * {@link Character#isWhitespace(int)}.  Note: That definition explicitly excludes the non-breaking space.
  * Adjacent sequences of non-Whitespace characters form tokens.
+ *
+ * @see UnicodeWhitespaceTokenizer
  */
 public final class WhitespaceTokenizer extends CharTokenizer {
   
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java	(revision 1713947)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java	(working copy)
@@ -17,32 +17,56 @@
  * limitations under the License.
  */
 
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
 import org.apache.lucene.util.AttributeFactory;
 
-import java.util.Map;
-
 /**
  * Factory for {@link WhitespaceTokenizer}. 
  * <pre class="prettyprint">
  * &lt;fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"&gt;
  *   &lt;analyzer&gt;
- *     &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
+ *     &lt;tokenizer class="solr.WhitespaceTokenizerFactory" rule="unicode"/&gt;
  *   &lt;/analyzer&gt;
  * &lt;/fieldType&gt;</pre>
+ *
+ * Options:
+ * <ul>
+ *   <li>rule: either "java" for {@link WhitespaceTokenizer}
+ *      or "unicode" for {@link UnicodeWhitespaceTokenizer}</li>
+ * </ul>
  */
 public class WhitespaceTokenizerFactory extends TokenizerFactory {
+  public static final String RULE_JAVA = "java";
+  public static final String RULE_UNICODE = "unicode";
+  private static final Collection<String> RULE_NAMES = Arrays.asList(RULE_JAVA, RULE_UNICODE);
 
+  private final String rule;
+
   /** Creates a new WhitespaceTokenizerFactory */
   public WhitespaceTokenizerFactory(Map<String,String> args) {
     super(args);
+
+    rule = get(args, "rule", RULE_NAMES, RULE_JAVA);
+
     if (!args.isEmpty()) {
       throw new IllegalArgumentException("Unknown parameters: " + args);
     }
   }
 
   @Override
-  public WhitespaceTokenizer create(AttributeFactory factory) {
-    return new WhitespaceTokenizer(factory);
+  public Tokenizer create(AttributeFactory factory) {
+    switch (rule) {
+      case RULE_JAVA:
+        return new WhitespaceTokenizer(factory);
+      case RULE_UNICODE:
+        return new UnicodeWhitespaceTokenizer(factory);
+      default:
+        throw new AssertionError();
+    }
   }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(working copy)
@@ -0,0 +1,58 @@
+// DO NOT EDIT THIS FILE! Use "ant unicode-data" to recreate.
+
+package org.apache.lucene.analysis.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.SparseFixedBitSet;
+
+/**
+ * This file contains unicode properties used by various {@link CharTokenizer}s.
+ * The data was created using ICU4J v54.1.0.0
+ * <p>
+ * Unicode version: 7.0.0.0
+ */
+public final class UnicodeProps {
+  private UnicodeProps() {}
+  
+  /** Unicode version that was used to generate this file: {@value} */
+  public static final String UNICODE_VERSION = "7.0.0.0";
+  
+  /** Bitset with Unicode WHITESPACE code points. */
+  public static final Bits WHITESPACE = createBits(
+    0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x0020, 0x0085, 0x00A0, 0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 
+    0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000);
+  
+  private static Bits createBits(final int... codepoints) {
+    final int len = codepoints[codepoints.length - 1] + 1;
+    final SparseFixedBitSet bitset = new SparseFixedBitSet(len);
+    for (int i : codepoints) bitset.set(i);
+    return new Bits() {
+      @Override
+      public boolean get(int index) {
+        return index < len && bitset.get(index);
+      }
+      
+      @Override
+      public int length() {
+        return 0x10FFFF + 1;
+      }
+    };
+  }
+}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(working copy)

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java	(revision 1713947)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java	(working copy)
@@ -101,7 +101,8 @@
       ReversePathHierarchyTokenizer.class, // this is supported via an option to PathHierarchyTokenizer's factory
       SnowballFilter.class, // this is called SnowballPorterFilterFactory
       PatternKeywordMarkerFilter.class,
-      SetKeywordMarkerFilter.class
+      SetKeywordMarkerFilter.class,
+      UnicodeWhitespaceTokenizer.class // a supported option via WhitespaceTokenizerFactory
     );
   }
 
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java	(revision 1713947)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java	(working copy)
@@ -130,7 +130,7 @@
 
     @Override
     public TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new WhitespaceTokenizer();
+      Tokenizer tokenizer = random().nextBoolean() ? new WhitespaceTokenizer() : new UnicodeWhitespaceTokenizer();
       return new TokenStreamComponents(tokenizer, new LowerCaseFilter(tokenizer));
     }
     
@@ -140,7 +140,7 @@
 
     @Override
     public TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new WhitespaceTokenizer();
+      Tokenizer tokenizer = random().nextBoolean() ? new WhitespaceTokenizer() : new UnicodeWhitespaceTokenizer();
       return new TokenStreamComponents(tokenizer, new UpperCaseFilter(tokenizer));
     }
     
@@ -230,7 +230,7 @@
   
   /** blast some random strings through the analyzer */
   public void testRandomStrings() throws Exception {
-    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
+    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer(), new UnicodeWhitespaceAnalyzer() };
     for (Analyzer analyzer : analyzers) {
       checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
     }
@@ -239,7 +239,7 @@
   
   /** blast some random large strings through the analyzer */
   public void testRandomHugeStrings() throws Exception {
-    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
+    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer(), new UnicodeWhitespaceAnalyzer() };
     for (Analyzer analyzer : analyzers) {
       checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192);
     }
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java	(revision 0)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,57 @@
+package org.apache.lucene.analysis.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.AttributeFactory;
+
+public class TestUnicodeWhitespaceTokenizer extends BaseTokenStreamTestCase {
+  
+  // clone of test from WhitespaceTokenizer
+  public void testSimple() throws IOException {
+    StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+    UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+        "\ud801\udc1ctest" });
+  }
+  
+  public void testNBSP() throws IOException {
+    StringReader reader = new StringReader("Tokenizer\u00A0test");
+    UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+        "test" });
+  }
+
+  public void testFactory() {
+    Map<String, String> args = new HashMap<>();
+    args.put("rule", "unicode");
+    WhitespaceTokenizerFactory factory = new WhitespaceTokenizerFactory(args);
+    AttributeFactory attributeFactory = newAttributeFactory();
+    Tokenizer tokenizer = factory.create(attributeFactory);
+    assertEquals(UnicodeWhitespaceTokenizer.class, tokenizer.getClass());
+  }
+
+}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java	(revision 0)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java	(working copy)

Property changes on: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUnicodeWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy
===================================================================
--- lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(revision 0)
+++ lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(working copy)
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.util.VersionInfo;
+
+def linesep = properties['line.separator'];
+
+def appendChar = { StringBuilder sb, int c ->
+  int len = sb.length();
+  if (len != 0) {
+    sb.append(', ');
+  }
+  if (len == 0 || len - sb.lastIndexOf(linesep) > 100) {
+    sb.append(linesep).append('    ');
+  }
+  sb.append(String.format(Locale.ROOT, "0x%04X", c));
+}
+
+def whitespace = new StringBuilder();
+for (int c = UCharacter.MIN_CODE_POINT; c <= UCharacter.MAX_CODE_POINT; c++) {
+  if (UCharacter.isUWhiteSpace(c)) {
+    appendChar(whitespace, c);
+  }
+}
+
+def icuVersion = VersionInfo.ICU_VERSION.toString();
+def unicodeVersion = UCharacter.getUnicodeVersion().toString();
+
+def code = """
+// DO NOT EDIT THIS FILE! Use "ant unicode-data" to recreate.
+
+package org.apache.lucene.analysis.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.SparseFixedBitSet;
+
+/**
+ * This file contains unicode properties used by various {@link CharTokenizer}s.
+ * The data was created using ICU4J v${icuVersion}
+ * <p>
+ * Unicode version: ${unicodeVersion}
+ */
+public final class UnicodeProps {
+  private UnicodeProps() {}
+  
+  /** Unicode version that was used to generate this file: {@value} */
+  public static final String UNICODE_VERSION = "${unicodeVersion}";
+  
+  /** Bitset with Unicode WHITESPACE code points. */
+  public static final Bits WHITESPACE = createBits(${whitespace});
+  
+  private static Bits createBits(final int... codepoints) {
+    final int len = codepoints[codepoints.length - 1] + 1;
+    final SparseFixedBitSet bitset = new SparseFixedBitSet(len);
+    for (int i : codepoints) bitset.set(i);
+    return new Bits() {
+      @Override
+      public boolean get(int index) {
+        return index < len && bitset.get(index);
+      }
+      
+      @Override
+      public int length() {
+        return ${String.format(Locale.ROOT, "0x%X", UCharacter.MAX_CODE_POINT)} + 1;
+      }
+    };
+  }
+}
+""";
+
+File f = new File(properties['unicode-props-file']);
+f.write(code.trim(), 'UTF-8');
+
+task.log("Unicode data written to: " + f);
Index: lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy
===================================================================
--- lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(revision 0)
+++ lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(working copy)

Property changes on: lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/benchmark/conf/wstok.alg
===================================================================
--- lucene/benchmark/conf/wstok.alg	(revision 0)
+++ lucene/benchmark/conf/wstok.alg	(working copy)
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This alg compares the performance of the original Character.isWhitespace()-based
+
+content.source=org.apache.lucene.benchmark.byTask.feeds.ReutersContentSource
+doc.tokenized=false
+doc.body.tokenized=true
+docs.dir=reuters-out
+
+-AnalyzerFactory(name:WhitespaceTokenizer, WhitespaceTokenizer(rule:java))
+
+-AnalyzerFactory(name:UnicodeWhitespaceTokenizer, WhitespaceTokenizer(rule:unicode))
+
+{ "Rounds"
+
+    -NewAnalyzer(WhitespaceTokenizer)
+    -ResetInputs
+    { "[Character.isWhitespace()] WhitespaceTokenizer" { ReadTokens > : 20000 }
+
+    -NewAnalyzer(UnicodeWhitespaceTokenizer)
+    -ResetInputs
+    { "[UnicodeProps.WHITESPACE.get()] UnicodeWhitespaceTokenizer" { ReadTokens > : 20000 }
+
+    NewRound
+} : 5
+RepSumByNameRound
Index: lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
===================================================================
--- lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java	(revision 1713947)
+++ lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java	(working copy)
@@ -47,6 +47,7 @@
 
   public void extract() throws IOException {
     long count = 0;
+    Files.createDirectories(outputDir);
     try (DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm")) {
       for (Path sgmFile : stream) {
         extractFile(sgmFile);
@@ -70,7 +71,7 @@
    * Override if you wish to change what is extracted
    */
   protected void extractFile(Path sgmFile) {
-    try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.UTF_8)) {
+    try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.ISO_8859_1)) {
       StringBuilder buffer = new StringBuilder(1024);
       StringBuilder outBuffer = new StringBuilder(1024);
 
