Index: lucene/analysis/common/build.xml
===================================================================
--- lucene/analysis/common/build.xml	(revision 1713947)
+++ lucene/analysis/common/build.xml	(working copy)
@@ -31,6 +31,8 @@
   
   <property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>  
   
+  <property name="unicode-props-file" location="src/java/org/apache/lucene/analysis/util/UnicodeProps.java"/>
+
   <target name="jflex" depends="-install-jflex,clean-jflex,-jflex-StandardAnalyzer,-jflex-UAX29URLEmailTokenizer,
                                 -jflex-wiki-tokenizer,-jflex-HTMLStripCharFilter"/>
 
@@ -114,6 +116,18 @@
     </delete>
   </target>
   
+  <target xmlns:ivy="antlib:org.apache.ivy.ant" name="-resolve-icu4j" unless="icu4j.resolved" depends="ivy-availability-check,ivy-configure">
+    <loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
+    <ivy:cachepath organisation="com.ibm.icu" module="icu4j" revision="${ivyversions./com.ibm.icu/icu4j}"
+      inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
+    <property name="icu4j.resolved" value="true"/>
+  </target>
+  
+  <target name="unicode-data" depends="-resolve-icu4j,resolve-groovy">
+    <groovy classpathref="icu4j.classpath" src="src/tools/groovy/generate-unicode-data.groovy"/>
+    <fixcrlf file="${unicode-props-file}" encoding="UTF-8"/>
+  </target>
+  
   <property name="tld.zones" value="http://www.internic.net/zones/root.zone"/>
   <property name="tld.output" location="src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro"/>
 
@@ -141,7 +155,7 @@
 
   <target name="javadocs" depends="module-build.javadocs"/>
 
-  <target name="regenerate" depends="jflex"/>
+  <target name="regenerate" depends="jflex,unicode-data"/>
   
   <target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
       <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,58 @@
+package org.apache.lucene.analysis.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.analysis.util.UnicodeProps;
+import org.apache.lucene.util.AttributeFactory;
+
+/**
+ * A UnicodeWhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * Adjacent sequences of non-Whitespace characters form tokens (according to
+ * Unicode's WHITESPACE property).
+ * <p>
+ * <em>For Unicode version see: {@link UnicodeProps}</em>
+ */
+public final class UnicodeWhitespaceTokenizer extends CharTokenizer {
+  
+  /**
+   * Construct a new UnicodeWhitespaceTokenizer.
+   */
+  public UnicodeWhitespaceTokenizer() {
+  }
+
+  /**
+   * Construct a new UnicodeWhitespaceTokenizer using a given
+   * {@link org.apache.lucene.util.AttributeFactory}.
+   *
+   * @param factory
+   *          the attribute factory to use for this {@link Tokenizer}
+   */
+  public UnicodeWhitespaceTokenizer(AttributeFactory factory) {
+    super(factory);
+  }
+  
+  /** Collects only characters which do not satisfy Unicode's WHITESPACE property. */
+  @Override
+  protected boolean isTokenChar(int c) {
+    return !UnicodeProps.WHITESPACE.get(c);
+  }
+  
+  private static final int BITS_LENGTH = UnicodeProps.WHITESPACE.length();
+}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java	(working copy)

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UnicodeWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(working copy)
@@ -0,0 +1,58 @@
+// DO NOT EDIT THIS FILE! Use "ant unicode-data" to recreate.
+
+package org.apache.lucene.analysis.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.SparseFixedBitSet;
+
+/**
+ * This file contains unicode properties used by various {@link CharTokenizer}s.
+ * The data was created using ICU4J v54.1.0.0
+ * <p>
+ * Unicode version: 7.0.0.0
+ */
+public final class UnicodeProps {
+  private UnicodeProps() {}
+  
+  /** Unicode version that was used to generate this file: {@value} */
+  public static final String UNICODE_VERSION = "7.0.0.0";
+  
+  /** Bitset with Unicode WHITESPACE code points. */
+  public static final Bits WHITESPACE = createBits(
+    0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x0020, 0x0085, 0x00A0, 0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 
+    0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000);
+  
+  private static Bits createBits(final int... codepoints) {
+    final int len = codepoints[codepoints.length - 1] + 1;
+    final SparseFixedBitSet bitset = new SparseFixedBitSet(len);
+    for (int i : codepoints) bitset.set(i);
+    return new Bits() {
+      @Override
+      public boolean get(int index) {
+        return index < len && bitset.get(index);
+      }
+      
+      @Override
+      public int length() {
+        return 0x10FFFF + 1;
+      }
+    };
+  }
+}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java	(working copy)

Property changes on: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy
===================================================================
--- lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(revision 0)
+++ lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(working copy)
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.util.VersionInfo;
+
+def linesep = properties['line.separator'];
+
+def appendChar = { StringBuilder sb, int c ->
+  int len = sb.length();
+  if (len != 0) {
+    sb.append(', ');
+  }
+  if (len == 0 || len - sb.lastIndexOf(linesep) > 100) {
+    sb.append(linesep).append('    ');
+  }
+  sb.append(String.format(Locale.ROOT, "0x%04X", c));
+}
+
+def whitespace = new StringBuilder();
+for (int c = UCharacter.MIN_CODE_POINT; c <= UCharacter.MAX_CODE_POINT; c++) {
+  if (UCharacter.isUWhiteSpace(c)) {
+    appendChar(whitespace, c);
+  }
+}
+
+def icuVersion = VersionInfo.ICU_VERSION.toString();
+def unicodeVersion = UCharacter.getUnicodeVersion().toString();
+
+def code = """
+// DO NOT EDIT THIS FILE! Use "ant unicode-data" to recreate.
+
+package org.apache.lucene.analysis.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.SparseFixedBitSet;
+
+/**
+ * This file contains unicode properties used by various {@link CharTokenizer}s.
+ * The data was created using ICU4J v${icuVersion}
+ * <p>
+ * Unicode version: ${unicodeVersion}
+ */
+public final class UnicodeProps {
+  private UnicodeProps() {}
+  
+  /** Unicode version that was used to generate this file: {@value} */
+  public static final String UNICODE_VERSION = "${unicodeVersion}";
+  
+  /** Bitset with Unicode WHITESPACE code points. */
+  public static final Bits WHITESPACE = createBits(${whitespace});
+  
+  private static Bits createBits(final int... codepoints) {
+    final int len = codepoints[codepoints.length - 1] + 1;
+    final SparseFixedBitSet bitset = new SparseFixedBitSet(len);
+    for (int i : codepoints) bitset.set(i);
+    return new Bits() {
+      @Override
+      public boolean get(int index) {
+        return index < len && bitset.get(index);
+      }
+      
+      @Override
+      public int length() {
+        return ${String.format(Locale.ROOT, "0x%X", UCharacter.MAX_CODE_POINT)} + 1;
+      }
+    };
+  }
+}
+""";
+
+File f = new File(properties['unicode-props-file']);
+f.write(code.trim(), 'UTF-8');
+
+task.log("Unicode data written to: " + f);
Index: lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy
===================================================================
--- lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(revision 0)
+++ lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy	(working copy)

Property changes on: lucene/analysis/common/src/tools/groovy/generate-unicode-data.groovy
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
