Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
===================================================================
--- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java	(revision 1154109)
+++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java	(working copy)
@@ -60,7 +60,7 @@
    * @param analyzer The analyzer to use for creating the TokenStream if the
    *        vector doesn't exist
    * @return The {@link org.apache.lucene.analysis.TokenStream} for the
-   *         {@link org.apache.lucene.document.Fieldable} on the
+   *         {@link org.apache.lucene.index.IndexableField} on the
    *         {@link org.apache.lucene.document.Document}
    * @throws IOException if there was an error loading
    */
Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
===================================================================
--- lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java	(revision 1154109)
+++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java	(working copy)
@@ -21,6 +21,7 @@
 import java.util.LinkedList;
 import java.util.Set;
 
+import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.TermFreqVector;
 import org.apache.lucene.index.TermPositionVector;
@@ -46,8 +47,12 @@
   //  Directory dir = new RAMDirectory();
   //  IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, analyzer));
   //  Document doc = new Document();
-  //  doc.add( new Field( "f", "a a a b b c a b b c d e f", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
-  //  doc.add( new Field( "f", "b a b a f", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+  //  FieldType ft = new FieldType(TextField.TYPE_STORED);
+  //  ft.setStoreTermVectors(true);
+  //  ft.setStoreTermVectorOffsets(true);
+  //  ft.setStoreTermVectorPositions(true);
+  //  doc.add( new Field( "f", ft, "a a a b b c a b b c d e f" ) );
+  //  doc.add( new Field( "f", ft, "b a b a f" ) );
   //  writer.addDocument( doc );
   //  writer.close();
     
@@ -67,7 +72,7 @@
    */
   public FieldTermStack( IndexReader reader, int docId, String fieldName, final FieldQuery fieldQuery ) throws IOException {
     this.fieldName = fieldName;
-
+    
     TermFreqVector tfv = reader.getTermFreqVector( docId, fieldName );
     if( tfv == null ) return; // just return to make null snippets
     TermPositionVector tpv = null;
Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html
===================================================================
--- lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html	(revision 1154109)
+++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html	(working copy)
@@ -25,7 +25,7 @@
 <li>support N-gram fields</li>
 <li>support phrase-unit highlighting with slops</li>
 <li>need Java 1.5</li>
-<li>highlight fields need to be TermVector.WITH_POSITIONS_OFFSETS</li>
+<li>highlight fields need to be stored with term vector positions and offsets</li>
 <li>take into account query boost to score fragments</li>
 <li>support colored highlight tags</li>
 <li>pluggable FragListBuilder</li>
@@ -95,7 +95,7 @@
 
 <h3>Step 2.</h3>
 <p>In Step 2, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldTermStack}. Fast Vector Highlighter uses {@link org.apache.lucene.index.TermFreqVector} data
-(must be stored {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS_OFFSETS})
+(must be stored with term vector positions and offsets)
 to generate it. <code>FieldTermStack</code> keeps the terms in the user query.
 Therefore, in this sample case, Fast Vector Highlighter generates the following <code>FieldTermStack</code>:</p>
 <pre>
Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
===================================================================
--- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java	(revision 1154109)
+++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java	(working copy)
@@ -240,11 +240,8 @@
   /**
    * Convenience method; Tokenizes the given field text and adds the resulting
    * terms to the index; Equivalent to adding an indexed non-keyword Lucene
-   * {@link org.apache.lucene.document.Field} that is
-   * {@link org.apache.lucene.document.Field.Index#ANALYZED tokenized},
-   * {@link org.apache.lucene.document.Field.Store#NO not stored},
-   * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions} (or
-   * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions and offsets}),
+   * {@link org.apache.lucene.document.Field} that is tokenized, not stored,
+   * termVectorStored with positions (or termVectorStored with positions and offsets),
    * 
    * @param fieldName
    *            a name to be associated with the text
Index: lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java
===================================================================
--- lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java	(revision 1153521)
+++ lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java	(working copy)
@@ -20,39 +20,41 @@
  *  Provides information about what should be done with this Field 
  *
  **/
+import org.apache.lucene.index.IndexableField; // for javadocs
+
 public enum FieldSelectorResult {
 
     /**
      * Load this {@link Field} every time the {@link Document} is loaded, reading in the data as it is encountered.
-     *  {@link Document#getField(String)} and {@link Document#getFieldable(String)} should not return null.
+     *  {@link Document#getField(String)} should not return null.
      *<p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LOAD,
 
     /**
      * Lazily load this {@link Field}.  This means the {@link Field} is valid, but it may not actually contain its data until
-     * invoked.  {@link Document#getField(String)} SHOULD NOT BE USED.  {@link Document#getFieldable(String)} is safe to use and should
-     * return a valid instance of a {@link Fieldable}.
+     * invoked.  {@link Document#getField(String)} is safe to use and should
+     * return a valid instance of a {@link IndexableField}.
      *<p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LAZY_LOAD,
 
     /**
-     * Do not load the {@link Field}.  {@link Document#getField(String)} and {@link Document#getFieldable(String)} should return null.
-     * {@link Document#add(Fieldable)} is not called.
+     * Do not load the {@link Field}.  {@link Document#getField(String)} should return null.
+     * {@link Document#add(IndexableField)} is not called.
      * <p/>
-     * {@link Document#add(Fieldable)} should not be called by the Reader.
+     * {@link Document#add(IndexableField)} should not be called by the Reader.
      */
   NO_LOAD,
 
     /**
      * Load this field as in the {@link #LOAD} case, but immediately return from {@link Field} loading for the {@link Document}.  Thus, the
-     * Document may not have its complete set of Fields.  {@link Document#getField(String)} and {@link Document#getFieldable(String)} should
+     * Document may not have its complete set of Fields.  {@link Document#getField(String)} should
      * both be valid for this {@link Field}
      * <p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LOAD_AND_BREAK,
 
@@ -67,10 +69,10 @@
 
   /**
      * Lazily load this {@link Field}, but do not cache the result.  This means the {@link Field} is valid, but it may not actually contain its data until
-     * invoked.  {@link Document#getField(String)} SHOULD NOT BE USED.  {@link Document#getFieldable(String)} is safe to use and should
-     * return a valid instance of a {@link Fieldable}.
+     * invoked.  {@link Document#getField(String)} is safe to use and should
+     * return a valid instance of a {@link IndexableField}.
      *<p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LATENT
 }
Index: lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
===================================================================
--- lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java	(revision 1154109)
+++ lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java	(working copy)
@@ -36,7 +36,7 @@
  * 
  * If Similarity class is specified, uses its computeNorm method to set norms.
  * If -n command line argument is used, removed field norms, as if 
- * {@link org.apache.lucene.document.Field.Index}.NO_NORMS was used.
+ * {@link org.apache.lucene.document.FieldType#setOmitNorms(boolean)} was used.
  *
  * <p>
  * NOTE: This will overwrite any length normalization or field/document boosts.
Index: lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java
===================================================================
--- lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java	(revision 1154109)
+++ lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java	(working copy)
@@ -152,10 +152,10 @@
     lazyFields.add("bin1");
     FieldSelector sel = new SetBasedFieldSelector(new HashSet<String>(), lazyFields);
     doc2 = getDocument(reader, reader.maxDoc() - 1, sel);
-    IndexableField[] fieldables = doc2.getFields("bin1");
-    assertNotNull(fieldables);
-    assertEquals(1, fieldables.length);
-    IndexableField fb1 = fieldables[0];
+    fields = doc2.getFields("bin1");
+    assertNotNull(fields);
+    assertEquals(1, fields.length);
+    IndexableField fb1 = fields[0];
     assertTrue(fb1.binaryValue(null)!=null);
     bytesRef = fb1.binaryValue(null);
     assertEquals(bin.length, bytesRef.bytes.length);
Index: lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
===================================================================
--- lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java	(revision 1154109)
+++ lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java	(working copy)
@@ -56,7 +56,6 @@
  * "word" field, and if present there will be fields named "syn"
  * for every synonym. What's tricky here is that there could be <b>multiple</b>
  * fields with the same name, in the general case for words that have multiple synonyms.
- * That's not a problem with Lucene, you just use {@link org.apache.lucene.document.Document#getValues}
  * </p>
  * <p>
  * While the WordNet file distinguishes groups of synonyms with
Index: lucene/src/java/org/apache/lucene/analysis/package.html
===================================================================
--- lucene/src/java/org/apache/lucene/analysis/package.html	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/analysis/package.html	(working copy)
@@ -172,7 +172,7 @@
 </p>
 <h3>Field Section Boundaries</h3>
 <p>
-  When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)}
+  When {@link org.apache.lucene.document.Document#add(org.apache.lucene.index.IndexableField) document.add(field)}
   is called multiple times for the same field name, we could say that each such call creates a new 
   section for that field in that document. 
   In fact, a separate call to 
Index: lucene/src/java/org/apache/lucene/document/BinaryField.java
===================================================================
--- lucene/src/java/org/apache/lucene/document/BinaryField.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/document/BinaryField.java	(revision 0)
@@ -0,0 +1,46 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public final class BinaryField extends Field {
+
+  public static final FieldType TYPE_STORED = new FieldType();
+  static {
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.freeze();
+  }
+
+  public BinaryField(String name, byte[] value) {
+    super(name, BinaryField.TYPE_STORED, value);
+    this.isBinary = true;
+  }
+  
+  public BinaryField(String name, byte[] value, int offset, int length) {
+    super(name, BinaryField.TYPE_STORED, value, offset, length);
+    this.isBinary = true;
+  }
+  
+  public BinaryField(String name, FieldType custom, byte[] value) {
+	  super(name, custom, value);
+	  this.isBinary = true;
+  }
+    
+  public boolean isNumeric() {
+    return false;
+  }  
+}
Index: lucene/src/java/org/apache/lucene/document/Field.java
===================================================================
--- lucene/src/java/org/apache/lucene/document/Field.java	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/document/Field.java	(working copy)
@@ -241,11 +241,7 @@
    * into the score of all hits on this this field of this document.
    * 
    * <p>
-   * The boost is multiplied by
-   * {@link org.apache.lucene.document.Document#getBoost()} of the document
-   * containing this field. If a document has multiple fields with the same
-   * name, all such values are multiplied together. This product is then used to
-   * compute the norm factor for the field. By default, in the
+   * Boost is used to compute the norm factor for the field. By default, in the
    * {@link org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)}
    * method, the boost value is multiplied by the length normalization factor
    * and then rounded by
@@ -253,7 +249,6 @@
    * it is stored in the index. One should attempt to ensure that this product
    * does not overflow the range of that encoding.
    * 
-   * @see org.apache.lucene.document.Document#setBoost(float)
    * @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
    * @see org.apache.lucene.search.Similarity#encodeNormValue(float)
    */
Index: lucene/src/java/org/apache/lucene/document/FieldType.java
===================================================================
--- lucene/src/java/org/apache/lucene/document/FieldType.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/document/FieldType.java	(revision 0)
@@ -0,0 +1,183 @@
+package org.apache.lucene.document;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class FieldType {
+
+  private boolean indexed;
+  private boolean stored;
+  private boolean tokenized;
+  private boolean storeTermVectors;
+  private boolean storeTermVectorOffsets;
+  private boolean storeTermVectorPositions;
+  private boolean omitNorms;
+  private boolean omitTermFreqsAndPositions;
+  private boolean lazy;
+  private boolean frozen;
+
+  public FieldType(FieldType ref) {
+    this.indexed = ref.indexed();
+    this.stored = ref.stored();
+    this.tokenized = ref.tokenized();
+    this.storeTermVectors = ref.storeTermVectors();
+    this.storeTermVectorOffsets = ref.storeTermVectorOffsets();
+    this.storeTermVectorPositions = ref.storeTermVectorPositions();
+    this.omitNorms = ref.omitNorms();
+    this.omitTermFreqsAndPositions = ref.omitTermFreqAndPositions();
+    this.lazy = ref.lazy();
+  }
+  
+  public FieldType() {
+  }
+
+  private void checkIfFrozen() {
+    if (frozen) {
+      throw new IllegalStateException();
+    }
+  }
+  
+  public void freeze() {
+    this.frozen = true;
+  }
+  
+  public boolean indexed() {
+    return this.indexed;
+  }
+  
+  public void setIndexed(boolean value) {
+    checkIfFrozen();
+    this.indexed = value;
+  }
+
+  public boolean stored() {
+    return this.stored;
+  }
+  
+  public void setStored(boolean value) {
+    checkIfFrozen();
+    this.stored = value;
+  }
+
+  public boolean tokenized() {
+    return this.tokenized;
+  }
+  
+  public void setTokenized(boolean value) {
+    checkIfFrozen();
+    this.tokenized = value;
+  }
+
+  public boolean storeTermVectors() {
+    return this.storeTermVectors;
+  }
+  
+  public void setStoreTermVectors(boolean value) {
+    checkIfFrozen();
+    this.storeTermVectors = value;
+  }
+
+  public boolean storeTermVectorOffsets() {
+    return this.storeTermVectorOffsets;
+  }
+  
+  public void setStoreTermVectorOffsets(boolean value) {
+    checkIfFrozen();
+    this.storeTermVectorOffsets = value;
+  }
+
+  public boolean storeTermVectorPositions() {
+    return this.storeTermVectorPositions;
+  }
+  
+  public void setStoreTermVectorPositions(boolean value) {
+    checkIfFrozen();
+    this.storeTermVectorPositions = value;
+  }
+  
+  public boolean omitNorms() {
+    return this.omitNorms;
+  }
+  
+  public void setOmitNorms(boolean value) {
+    checkIfFrozen();
+    this.omitNorms = value;
+  }
+
+  public boolean omitTermFreqAndPositions() {
+    return this.omitTermFreqsAndPositions;
+  }
+  
+  public void setOmitTermFreqAndPositions(boolean value) {
+    checkIfFrozen();
+    this.omitTermFreqsAndPositions = value;
+  }
+
+  public boolean lazy() {
+    return this.lazy;
+  }
+  
+  public void setLazy(boolean value) {
+    checkIfFrozen();
+    this.lazy = value;
+  }
+
+  /** Prints a Field for human consumption. */
+  @Override
+  public final String toString() {
+    StringBuilder result = new StringBuilder();
+    if (stored()) {
+      result.append("stored");
+    }
+    if (indexed()) {
+      if (result.length() > 0)
+        result.append(",");
+      result.append("indexed");
+    }
+    if (tokenized()) {
+      if (result.length() > 0)
+        result.append(",");
+      result.append("tokenized");
+    }
+    if (storeTermVectors()) {
+      if (result.length() > 0)
+        result.append(",");
+      result.append("termVector");
+    }
+    if (storeTermVectorOffsets()) {
+      if (result.length() > 0)
+        result.append(",");
+      result.append("termVectorOffsets");
+    }
+    if (storeTermVectorPositions()) {
+      if (result.length() > 0)
+        result.append(",");
+      result.append("termVectorPosition");
+    }
+    if (omitNorms()) {
+      result.append(",omitNorms");
+    }
+    if (omitTermFreqAndPositions()) {
+      result.append(",omitTermFreqAndPositions");
+    }
+    if (lazy()){
+      result.append(",lazy");
+    }
+    
+    return result.toString();
+  }
+}
Index: lucene/src/java/org/apache/lucene/document/StringField.java
===================================================================
--- lucene/src/java/org/apache/lucene/document/StringField.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/document/StringField.java	(revision 0)
@@ -0,0 +1,53 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public final class StringField extends Field {
+
+  public static final FieldType TYPE_UNSTORED = new FieldType();
+  public static final FieldType TYPE_STORED = new FieldType();
+  static {
+    TYPE_UNSTORED.setIndexed(true);
+    TYPE_UNSTORED.setOmitNorms(true);
+    TYPE_UNSTORED.setOmitTermFreqAndPositions(true);
+    TYPE_UNSTORED.freeze();
+
+    TYPE_STORED.setIndexed(true);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.setOmitNorms(true);
+    TYPE_STORED.setOmitTermFreqAndPositions(true);
+    TYPE_STORED.freeze();
+  }
+  
+  public StringField(String name, boolean internName, String value) {
+    super(name, StringField.TYPE_UNSTORED, value);
+  }
+  
+  public StringField(String name, String value) {
+    this(name, true, value);
+  }
+  
+  @Override
+  public String stringValue() {
+    return (fieldsData == null) ? null : fieldsData.toString();
+  }
+  
+  public boolean isNumeric() {
+    return false;
+  }  
+}
Index: lucene/src/java/org/apache/lucene/document/TextField.java
===================================================================
--- lucene/src/java/org/apache/lucene/document/TextField.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/document/TextField.java	(revision 0)
@@ -0,0 +1,54 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+
+public final class TextField extends Field {
+
+  public static final FieldType TYPE_UNSTORED = new FieldType();
+  public static final FieldType TYPE_STORED = new FieldType();
+  static {
+    TYPE_UNSTORED.setIndexed(true);
+    TYPE_UNSTORED.setTokenized(true);
+    TYPE_UNSTORED.freeze();
+
+    TYPE_STORED.setIndexed(true);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.setTokenized(true);
+    TYPE_STORED.freeze();
+  }
+  
+  public TextField(String name, Reader reader) {
+    super(name, TextField.TYPE_UNSTORED, reader);
+  }
+
+  public TextField(String name, String value) {
+    super(name, TextField.TYPE_UNSTORED, value);
+  }
+  
+  public TextField(String name, TokenStream stream) {
+    super(name, TextField.TYPE_UNSTORED, stream);
+  }
+
+  public boolean isNumeric() {
+    return false;
+  }
+}
Index: lucene/src/java/org/apache/lucene/document/package.html
===================================================================
--- lucene/src/java/org/apache/lucene/document/package.html	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/document/package.html	(working copy)
@@ -0,0 +1,56 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>The logical representation of a {@link org.apache.lucene.document.Document} for indexing and searching.</p>
+<p>The document package provides the user level logical representation of content to be indexed and searched.  The
+package also provides utilities for working with {@link org.apache.lucene.document.Document}s and {@link org.apache.lucene.document.Fieldable}s.</p>
+<h2>Document and Fieldable</h2>
+<p>A {@link org.apache.lucene.document.Document} is a collection of {@link org.apache.lucene.document.Fieldable}s.  A
+  {@link org.apache.lucene.document.Fieldable} is a logical representation of a user's content that needs to be indexed or stored.
+  {@link org.apache.lucene.document.Fieldable}s have a number of properties that tell Lucene how to treat the content (like indexed, tokenized,
+  stored, etc.)  See the {@link org.apache.lucene.document.Field} implementation of {@link org.apache.lucene.document.Fieldable}
+  for specifics on these properties.
+</p>
+<p>Note: it is common to refer to {@link org.apache.lucene.document.Document}s having {@link org.apache.lucene.document.Field}s, even though technically they have
+{@link org.apache.lucene.document.Fieldable}s.</p>
+<h2>Working with Documents</h2>
+<p>First and foremost, a {@link org.apache.lucene.document.Document} is something created by the user application.  It is your job
+  to create Documents based on the content of the files you are working with in your application (Word, txt, PDF, Excel or any other format.)
+  How this is done is completely up to you.  That being said, there are many tools available in other projects that can make
+  the process of taking a file and converting it into a Lucene {@link org.apache.lucene.document.Document}.  To see an example of this,
+  take a look at the Lucene <a href="../../../../../../gettingstarted.html" target="top">demo</a> and the associated source code
+  for extracting content from HTML.
+</p>
+<p>The {@link org.apache.lucene.document.DateTools} is a utility class to make dates and times searchable
+(remember, Lucene only searches text). {@link org.apache.lucene.document.NumericField} is a special helper class
+to simplify indexing of numeric values (and also dates) for fast range range queries with {@link org.apache.lucene.search.NumericRangeQuery}
+(using a special sortable string representation of numeric values).</p>
+<p>The {@link org.apache.lucene.index.StoredFieldVisitor} class provides a mechanism to customize how the stored fields values are processed.
+If no StoredFieldVisitor is used, all Fields are loaded into a returned Document.  As an example of the StoredFieldVisitor usage, consider
+  the common use case of
+displaying search results on a web page and then having users click through to see the full document.  In this scenario, it is often
+  the case that there are many small fields and one or two large fields (containing the contents of the original file). Before StoredFieldVisitor,
+the full Document had to be loaded, including the large fields, in order to display the results.  Now, using the StoredFieldVisitor, one
+can choose which fields should be loaded and how their values should be recorded.  If you simply want to load only certain fields, consider using
+{@link org.apache.lucene.index.DocumentStoredFieldVisitor}.
+</body>
+</html>
Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/CheckIndex.java	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/index/CheckIndex.java	(working copy)
@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 
+import org.apache.lucene.document.FieldType; // for javadocs
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
@@ -175,7 +176,7 @@
 
       /** True if at least one of the fields in this segment
        *  does not omitTermFreqAndPositions.
-       *  @see AbstractField#setOmitTermFreqAndPositions */
+       *  @see FieldType#setOmitTermFreqAndPositions */
       public boolean hasProx;
 
       /** Map that includes certain
Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/FieldInfos.java	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/index/FieldInfos.java	(working copy)
@@ -37,8 +37,8 @@
 import org.apache.lucene.util.CodecUtil;
 import org.apache.lucene.util.StringHelper;
 
-/** Access to the Fieldable Info file that describes document fields and whether or
- *  not they are indexed. Each segment has a separate Fieldable Info file. Objects
+/** Access to the Field Info file that describes document fields and whether or
+ *  not they are indexed. Each segment has a separate Field Info file. Objects
  *  of this class are thread-safe for multiple readers, but only one thread can
  *  be adding documents at a time, with no other reader or writer threads
  *  accessing this object.
@@ -359,7 +359,7 @@
   /**
    * Calls 5 parameter add with false for all TermVector parameters.
    * 
-   * @param name The name of the Fieldable
+   * @param name The name of the IndexableField
    * @param isIndexed true if the field is indexed
    * @see #addOrUpdate(String, boolean, boolean, boolean, boolean)
    */
Index: lucene/src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/IndexReader.java	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/index/IndexReader.java	(working copy)
@@ -861,7 +861,6 @@
    * @return array of term frequency vectors. May be null if no term vectors have been
    *  stored for the specified document.
    * @throws IOException if index cannot be accessed
-   * @see org.apache.lucene.document.Field.TermVector
    */
   abstract public TermFreqVector[] getTermFreqVectors(int docNumber)
           throws IOException;
@@ -879,7 +878,6 @@
    * @return term frequency vector May be null if field does not exist in the specified
    * document or term vector was not stored.
    * @throws IOException if index cannot be accessed
-   * @see org.apache.lucene.document.Field.TermVector
    */
   abstract public TermFreqVector getTermFreqVector(int docNumber, String field)
           throws IOException;
Index: lucene/src/java/org/apache/lucene/index/TermFreqVector.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/TermFreqVector.java	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/index/TermFreqVector.java	(working copy)
@@ -26,7 +26,7 @@
  */
 public interface TermFreqVector {
   /**
-   * The {@link org.apache.lucene.document.Fieldable} name. 
+   * The {@link org.apache.lucene.index.IndexableField} name. 
    * @return The name of the field this vector is associated with.
    * 
    */ 
Index: lucene/src/java/org/apache/lucene/search/Similarity.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/Similarity.java	(revision 1154109)
+++ lucene/src/java/org/apache/lucene/search/Similarity.java	(working copy)
@@ -453,12 +453,8 @@
  *      <b><i>norm(t,d)</i></b> encapsulates a few (indexing time) boost and length factors:
  *
  *      <ul>
- *        <li><b>Document boost</b> - set by calling
- *        {@link org.apache.lucene.document.Document#setBoost(float) doc.setBoost()}
- *        before adding the document to the index.
- *        </li>
  *        <li><b>Field boost</b> - set by calling
- *        {@link org.apache.lucene.document.Fieldable#setBoost(float) field.setBoost()}
+ *        {@link org.apache.lucene.document.Field#setBoost(float) field.setBoost()}
  *        before adding the field to a document.
  *        </li>
  *        <li><b>lengthNorm</b> - computed
@@ -478,9 +474,6 @@
  *      <table cellpadding="1" cellspacing="0" border="0"n align="center">
  *        <tr>
  *          <td valign="middle" align="right" rowspan="1">
- *            norm(t,d) &nbsp; = &nbsp;
- *            {@link org.apache.lucene.document.Document#getBoost() doc.getBoost()}
- *            &nbsp;&middot;&nbsp;
  *            lengthNorm
  *            &nbsp;&middot;&nbsp;
  *          </td>
@@ -488,7 +481,7 @@
  *            <big><big><big>&prod;</big></big></big>
  *          </td>
  *          <td valign="middle" align="right" rowspan="1">
- *            {@link org.apache.lucene.document.Fieldable#getBoost() f.getBoost}()
+ *            {@link org.apache.lucene.index.IndexableField#boost() f.getBoost}()
  *          </td>
  *        </tr>
  *        <tr valigh="top">
Index: lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java
===================================================================
--- lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java	(revision 0)
+++ lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java	(revision 0)
@@ -0,0 +1,115 @@
+package org.apache.lucene.document;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+import org.apache.lucene.document.BinaryField;
+import org.apache.lucene.document.CompressionTools;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tests {@link Document} class.
+ */
+public class TestBinaryDocument extends LuceneTestCase {
+
+  String binaryValStored = "this text will be stored as a byte array in the index";
+  String binaryValCompressed = "this text will be also stored and compressed as a byte array in the index";
+  
+  public void testBinaryFieldInIndex()
+    throws Exception
+  {
+    FieldType ft = new FieldType();
+    ft.setStored(true);
+    IndexableField binaryFldStored = new BinaryField("binaryStored", binaryValStored.getBytes());
+    IndexableField stringFldStored = new Field("stringStored", ft, binaryValStored);
+
+    Document doc = new Document();
+    
+    doc.add(binaryFldStored);
+    
+    doc.add(stringFldStored);
+
+    /** test for field count */
+    assertEquals(2, doc.fields.size());
+    
+    /** add the doc to a ram index */
+    Directory dir = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+    writer.addDocument(doc);
+    
+    /** open a reader and fetch the document */ 
+    IndexReader reader = writer.getReader();
+    Document docFromReader = reader.document(0);
+    assertTrue(docFromReader != null);
+    
+    /** fetch the binary stored field and compare it's content with the original one */
+    String binaryFldStoredTest = new String(docFromReader.getBinaryValue("binaryStored"));
+    assertTrue(binaryFldStoredTest.equals(binaryValStored));
+    
+    /** fetch the string field and compare it's content with the original one */
+    String stringFldStoredTest = docFromReader.get("stringStored");
+    assertTrue(stringFldStoredTest.equals(binaryValStored));
+    
+    writer.close();    
+    reader.close();
+    
+    reader = IndexReader.open(dir, false);
+    /** delete the document from index */
+    reader.deleteDocument(0);
+    assertEquals(0, reader.numDocs());
+    
+    reader.close();
+    dir.close();
+  }
+  
+  public void testCompressionTools() throws Exception {
+    IndexableField binaryFldCompressed = new BinaryField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes()));
+    IndexableField stringFldCompressed = new BinaryField("stringCompressed", CompressionTools.compressString(binaryValCompressed));
+    
+    Document doc = new Document();
+    
+    doc.add(binaryFldCompressed);
+    doc.add(stringFldCompressed);
+    
+    /** add the doc to a ram index */
+    Directory dir = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+    writer.addDocument(doc);
+    
+    /** open a reader and fetch the document */ 
+    IndexReader reader = writer.getReader();
+    Document docFromReader = reader.document(0);
+    assertTrue(docFromReader != null);
+    
+    /** fetch the binary compressed field and compare it's content with the original one */
+    String binaryFldCompressedTest = new String(CompressionTools.decompress(docFromReader.getBinaryValue("binaryCompressed")));
+    assertTrue(binaryFldCompressedTest.equals(binaryValCompressed));
+    assertTrue(CompressionTools.decompressString(docFromReader.getBinaryValue("stringCompressed")).equals(binaryValCompressed));
+
+    writer.close();
+    reader.close();
+    dir.close();
+  }
+}
Index: lucene/src/test/org/apache/lucene/document/TestDateTools.java
===================================================================
--- lucene/src/test/org/apache/lucene/document/TestDateTools.java	(revision 0)
+++ lucene/src/test/org/apache/lucene/document/TestDateTools.java	(revision 0)
@@ -0,0 +1,200 @@
+package org.apache.lucene.document;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+import java.util.Locale;
+
+import org.apache.lucene.document.DateTools;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class TestDateTools extends LuceneTestCase {
+
+  public void testStringToDate() throws ParseException {
+    
+    Date d = null;
+    d = DateTools.stringToDate("2004");
+    assertEquals("2004-01-01 00:00:00:000", isoFormat(d));
+    d = DateTools.stringToDate("20040705");
+    assertEquals("2004-07-05 00:00:00:000", isoFormat(d));
+    d = DateTools.stringToDate("200407050910");
+    assertEquals("2004-07-05 09:10:00:000", isoFormat(d));
+    d = DateTools.stringToDate("20040705091055990");
+    assertEquals("2004-07-05 09:10:55:990", isoFormat(d));
+
+    try {
+      d = DateTools.stringToDate("97");    // no date
+      fail();
+    } catch(ParseException e) { /* expected exception */ }
+    try {
+      d = DateTools.stringToDate("200401011235009999");    // no date
+      fail();
+    } catch(ParseException e) { /* expected exception */ }
+    try {
+      d = DateTools.stringToDate("aaaa");    // no date
+      fail();
+    } catch(ParseException e) { /* expected exception */ }
+
+  }
+  
+  public void testStringtoTime() throws ParseException {
+    long time = DateTools.stringToTime("197001010000");
+    Calendar cal = new GregorianCalendar();
+    cal.clear();
+    cal.set(1970, 0, 1,    // year=1970, month=january, day=1
+        0, 0, 0);          // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 0);
+    cal.setTimeZone(TimeZone.getTimeZone("GMT"));
+    assertEquals(cal.getTime().getTime(), time);
+    cal.set(1980, 1, 2,    // year=1980, month=february, day=2
+        11, 5, 0);          // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 0);
+    time = DateTools.stringToTime("198002021105");
+    assertEquals(cal.getTime().getTime(), time);
+  }
+  
+  public void testDateAndTimetoString() throws ParseException {
+    Calendar cal = new GregorianCalendar();
+    cal.clear();
+    cal.setTimeZone(TimeZone.getTimeZone("GMT"));
+    cal.set(2004, 1, 3,   // year=2004, month=february(!), day=3
+        22, 8, 56);       // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 333);
+    
+    String dateString;
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.YEAR);
+    assertEquals("2004", dateString);
+    assertEquals("2004-01-01 00:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+    
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MONTH);
+    assertEquals("200402", dateString);
+    assertEquals("2004-02-01 00:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.DAY);
+    assertEquals("20040203", dateString);
+    assertEquals("2004-02-03 00:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+    
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.HOUR);
+    assertEquals("2004020322", dateString);
+    assertEquals("2004-02-03 22:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+    
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MINUTE);
+    assertEquals("200402032208", dateString);
+    assertEquals("2004-02-03 22:08:00:000", isoFormat(DateTools.stringToDate(dateString)));
+    
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.SECOND);
+    assertEquals("20040203220856", dateString);
+    assertEquals("2004-02-03 22:08:56:000", isoFormat(DateTools.stringToDate(dateString)));
+    
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MILLISECOND);
+    assertEquals("20040203220856333", dateString);
+    assertEquals("2004-02-03 22:08:56:333", isoFormat(DateTools.stringToDate(dateString)));
+
+    // date before 1970:
+    cal.set(1961, 2, 5,   // year=1961, month=march(!), day=5
+        23, 9, 51);       // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 444);
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MILLISECOND);
+    assertEquals("19610305230951444", dateString);
+    assertEquals("1961-03-05 23:09:51:444", isoFormat(DateTools.stringToDate(dateString)));
+
+    dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.HOUR);
+    assertEquals("1961030523", dateString);
+    assertEquals("1961-03-05 23:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+    // timeToString:
+    cal.set(1970, 0, 1, // year=1970, month=january, day=1
+        0, 0, 0); // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 0);
+    dateString = DateTools.timeToString(cal.getTime().getTime(),
+        DateTools.Resolution.MILLISECOND);
+    assertEquals("19700101000000000", dateString);
+        
+    cal.set(1970, 0, 1, // year=1970, month=january, day=1
+        1, 2, 3); // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 0);
+    dateString = DateTools.timeToString(cal.getTime().getTime(),
+        DateTools.Resolution.MILLISECOND);
+    assertEquals("19700101010203000", dateString);
+  }
+  
+  public void testRound() {
+    Calendar cal = new GregorianCalendar();
+    cal.clear();
+    cal.setTimeZone(TimeZone.getTimeZone("GMT"));
+    cal.set(2004, 1, 3,   // year=2004, month=february(!), day=3
+        22, 8, 56);       // hour, minute, second
+    cal.set(Calendar.MILLISECOND, 333);
+    Date date = cal.getTime();
+    assertEquals("2004-02-03 22:08:56:333", isoFormat(date));
+
+    Date dateYear = DateTools.round(date, DateTools.Resolution.YEAR);
+    assertEquals("2004-01-01 00:00:00:000", isoFormat(dateYear));
+
+    Date dateMonth = DateTools.round(date, DateTools.Resolution.MONTH);
+    assertEquals("2004-02-01 00:00:00:000", isoFormat(dateMonth));
+
+    Date dateDay = DateTools.round(date, DateTools.Resolution.DAY);
+    assertEquals("2004-02-03 00:00:00:000", isoFormat(dateDay));
+
+    Date dateHour = DateTools.round(date, DateTools.Resolution.HOUR);
+    assertEquals("2004-02-03 22:00:00:000", isoFormat(dateHour));
+
+    Date dateMinute = DateTools.round(date, DateTools.Resolution.MINUTE);
+    assertEquals("2004-02-03 22:08:00:000", isoFormat(dateMinute));
+
+    Date dateSecond = DateTools.round(date, DateTools.Resolution.SECOND);
+    assertEquals("2004-02-03 22:08:56:000", isoFormat(dateSecond));
+
+    Date dateMillisecond = DateTools.round(date, DateTools.Resolution.MILLISECOND);
+    assertEquals("2004-02-03 22:08:56:333", isoFormat(dateMillisecond));
+
+    // long parameter:
+    long dateYearLong = DateTools.round(date.getTime(), DateTools.Resolution.YEAR);
+    assertEquals("2004-01-01 00:00:00:000", isoFormat(new Date(dateYearLong)));
+
+    long dateMillisecondLong = DateTools.round(date.getTime(), DateTools.Resolution.MILLISECOND);
+    assertEquals("2004-02-03 22:08:56:333", isoFormat(new Date(dateMillisecondLong)));
+  }
+
+  private String isoFormat(Date date) {
+    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS", Locale.US);
+    sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+    return sdf.format(date);
+  }
+
+  public void testDateToolsUTC() throws Exception {
+    // Sun, 30 Oct 2005 00:00:00 +0000 -- the last second of 2005's DST in Europe/London
+    long time = 1130630400;
+    try {
+        TimeZone.setDefault(TimeZone.getTimeZone(/* "GMT" */ "Europe/London"));
+        String d1 = DateTools.dateToString(new Date(time*1000), DateTools.Resolution.MINUTE);
+        String d2 = DateTools.dateToString(new Date((time+3600)*1000), DateTools.Resolution.MINUTE);
+        assertFalse("different times", d1.equals(d2));
+        assertEquals("midnight", DateTools.stringToTime(d1), time*1000);
+        assertEquals("later", DateTools.stringToTime(d2), (time+3600)*1000);
+    } finally {
+        TimeZone.setDefault(null);
+    }
+  }
+
+}
\ No newline at end of file
Index: lucene/src/test/org/apache/lucene/document/TestDocument.java
===================================================================
--- lucene/src/test/org/apache/lucene/document/TestDocument.java	(revision 0)
+++ lucene/src/test/org/apache/lucene/document/TestDocument.java	(revision 0)
@@ -0,0 +1,288 @@
+package org.apache.lucene.document;
+
+import org.apache.lucene.document.BinaryField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tests {@link Document} class.
+ */
+public class TestDocument extends LuceneTestCase {
+  
+  String binaryVal = "this text will be stored as a byte array in the index";
+  String binaryVal2 = "this text will be also stored as a byte array in the index";
+  
+  public void testBinaryField() throws Exception {
+    Document doc = new Document();
+    
+    FieldType ft = new FieldType();
+    ft.setStored(true);
+    IndexableField stringFld = new Field("string", ft, binaryVal);
+    IndexableField binaryFld = new BinaryField("binary", binaryVal.getBytes());
+    IndexableField binaryFld2 = new BinaryField("binary", binaryVal2.getBytes());
+    
+    doc.add(stringFld);
+    doc.add(binaryFld);
+    
+    assertEquals(2, doc.fields.size());
+    
+    assertTrue(binaryFld.binaryValue(null) != null);
+    assertTrue(binaryFld.stored());
+    assertFalse(binaryFld.indexed());
+    assertFalse(binaryFld.tokenized());
+    
+    String binaryTest = new String(doc.getBinaryValue("binary"));
+    assertTrue(binaryTest.equals(binaryVal));
+    
+    String stringTest = doc.get("string");
+    assertTrue(binaryTest.equals(stringTest));
+    
+    doc.add(binaryFld2);
+    
+    assertEquals(3, doc.fields.size());
+    
+    byte[][] binaryTests = doc.getBinaryValues("binary");
+    
+    assertEquals(2, binaryTests.length);
+    
+    binaryTest = new String(binaryTests[0]);
+    String binaryTest2 = new String(binaryTests[1]);
+    
+    assertFalse(binaryTest.equals(binaryTest2));
+    
+    assertTrue(binaryTest.equals(binaryVal));
+    assertTrue(binaryTest2.equals(binaryVal2));
+    
+    doc.removeField("string");
+    assertEquals(2, doc.fields.size());
+    
+    doc.removeFields("binary");
+    assertEquals(0, doc.fields.size());
+  }
+  
+  /**
+   * Tests {@link Document#removeField(String)} method for a brand new Document
+   * that has not been indexed yet.
+   * 
+   * @throws Exception on error
+   */
+  public void testRemoveForNewDocument() throws Exception {
+    Document doc = makeDocumentWithFields();
+    assertEquals(8, doc.fields.size());
+    doc.removeFields("keyword");
+    assertEquals(6, doc.fields.size());
+    doc.removeFields("doesnotexists"); // removing non-existing fields is
+                                       // siltenlty ignored
+    doc.removeFields("keyword"); // removing a field more than once
+    assertEquals(6, doc.fields.size());
+    doc.removeField("text");
+    assertEquals(5, doc.fields.size());
+    doc.removeField("text");
+    assertEquals(4, doc.fields.size());
+    doc.removeField("text");
+    assertEquals(4, doc.fields.size());
+    doc.removeField("doesnotexists"); // removing non-existing fields is
+                                      // siltenlty ignored
+    assertEquals(4, doc.fields.size());
+    doc.removeFields("unindexed");
+    assertEquals(2, doc.fields.size());
+    doc.removeFields("unstored");
+    assertEquals(0, doc.fields.size());
+    doc.removeFields("doesnotexists"); // removing non-existing fields is
+                                       // siltenlty ignored
+    assertEquals(0, doc.fields.size());
+  }
+  
+  public void testConstructorExceptions() {
+    FieldType ft = new FieldType();
+    ft.setStored(true);
+    new Field("name", ft, "value"); // okay
+    new StringField("name", "value"); // okay
+    try {
+      new Field("name", new FieldType(), "value");
+      fail();
+    } catch (IllegalArgumentException e) {
+      // expected exception
+    }
+    new Field("name", ft, "value"); // okay
+    try {
+      FieldType ft2 = new FieldType();
+      ft2.setStored(true);
+      ft2.setStoreTermVectors(true);
+      new Field("name", ft2, "value");
+      fail();
+    } catch (IllegalArgumentException e) {
+      // expected exception
+    }
+  }
+  
+  /**
+   * Tests {@link Document#getValues(String)} method for a brand new Document
+   * that has not been indexed yet.
+   * 
+   * @throws Exception on error
+   */
+  public void testGetValuesForNewDocument() throws Exception {
+    doAssert(makeDocumentWithFields(), false);
+  }
+  
+  /**
+   * Tests {@link Document#getValues(String)} method for a Document retrieved
+   * from an index.
+   * 
+   * @throws Exception on error
+   */
+  public void testGetValuesForIndexedDocument() throws Exception {
+    Directory dir = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+    writer.addDocument(makeDocumentWithFields());
+    IndexReader reader = writer.getReader();
+    
+    IndexSearcher searcher = newSearcher(reader);
+    
+    // search for something that does exists
+    Query query = new TermQuery(new Term("keyword", "test1"));
+    
+    // ensure that queries return expected results without DateFilter first
+    ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+    assertEquals(1, hits.length);
+    
+    doAssert(searcher.doc(hits[0].doc), true);
+    writer.close();
+    searcher.close();
+    reader.close();
+    dir.close();
+  }
+  
+  private Document makeDocumentWithFields() {
+    Document doc = new Document();
+    FieldType stored = new FieldType();
+    stored.setStored(true);
+    doc.add(new Field("keyword", StringField.TYPE_STORED, "test1"));
+    doc.add(new Field("keyword", StringField.TYPE_STORED, "test2"));
+    doc.add(new Field("text", TextField.TYPE_STORED, "test1"));
+    doc.add(new Field("text", TextField.TYPE_STORED, "test2"));
+    doc.add(new Field("unindexed", stored, "test1"));
+    doc.add(new Field("unindexed", stored, "test2"));
+    doc
+        .add(new TextField("unstored", "test1"));
+    doc
+        .add(new TextField("unstored", "test2"));
+    return doc;
+  }
+  
+  private void doAssert(Document doc, boolean fromIndex) {
+    IndexableField[] keywordFieldValues = doc.getFields("keyword");
+    IndexableField[] textFieldValues = doc.getFields("text");
+    IndexableField[] unindexedFieldValues = doc.getFields("unindexed");
+    IndexableField[] unstoredFieldValues = doc.getFields("unstored");
+    
+    assertTrue(keywordFieldValues.length == 2);
+    assertTrue(textFieldValues.length == 2);
+    assertTrue(unindexedFieldValues.length == 2);
+    // this test cannot work for documents retrieved from the index
+    // since unstored fields will obviously not be returned
+    if (!fromIndex) {
+      assertTrue(unstoredFieldValues.length == 2);
+    }
+    
+    assertTrue(keywordFieldValues[0].stringValue().equals("test1"));
+    assertTrue(keywordFieldValues[1].stringValue().equals("test2"));
+    assertTrue(textFieldValues[0].stringValue().equals("test1"));
+    assertTrue(textFieldValues[1].stringValue().equals("test2"));
+    assertTrue(unindexedFieldValues[0].stringValue().equals("test1"));
+    assertTrue(unindexedFieldValues[1].stringValue().equals("test2"));
+    // this test cannot work for documents retrieved from the index
+    // since unstored fields will obviously not be returned
+    if (!fromIndex) {
+      assertTrue(unstoredFieldValues[0].stringValue().equals("test1"));
+      assertTrue(unstoredFieldValues[1].stringValue().equals("test2"));
+    }
+  }
+  
+  public void testFieldSetValue() throws Exception {
+    
+    Field field = new Field("id", StringField.TYPE_STORED, "id1");
+    Document doc = new Document();
+    doc.add(field);
+    doc.add(new Field("keyword", StringField.TYPE_STORED, "test"));
+    
+    Directory dir = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+    writer.addDocument(doc);
+    field.setValue("id2");
+    writer.addDocument(doc);
+    field.setValue("id3");
+    writer.addDocument(doc);
+    
+    IndexReader reader = writer.getReader();
+    IndexSearcher searcher = newSearcher(reader);
+    
+    Query query = new TermQuery(new Term("keyword", "test"));
+    
+    // ensure that queries return expected results without DateFilter first
+    ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+    assertEquals(3, hits.length);
+    int result = 0;
+    for (int i = 0; i < 3; i++) {
+      Document doc2 = searcher.doc(hits[i].doc);
+      Field f = (Field) doc2.getField("id");
+      if (f.stringValue().equals("id1")) result |= 1;
+      else if (f.stringValue().equals("id2")) result |= 2;
+      else if (f.stringValue().equals("id3")) result |= 4;
+      else fail("unexpected id field");
+    }
+    writer.close();
+    searcher.close();
+    reader.close();
+    dir.close();
+    assertEquals("did not see all IDs", 7, result);
+  }
+  
+  public void testFieldSetValueChangeBinary() {
+    Field field1 = new BinaryField("field1", new byte[0]);
+    Field field2 = new Field("field2", TextField.TYPE_STORED, "");
+    try {
+      field1.setValue("abc");
+      fail("did not hit expected exception");
+    } catch (IllegalArgumentException iae) {
+      // expected
+    }
+    try {
+      field2.setValue(new byte[0]);
+      fail("did not hit expected exception");
+    } catch (IllegalArgumentException iae) {
+      // expected
+    }
+  }
+}
Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestIndexReader.java	(revision 1154109)
+++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java	(working copy)
@@ -929,11 +929,11 @@
         if (delDocs1 == null || !delDocs1.get(i)) {
           Document doc1 = index1.document(i);
           Document doc2 = index2.document(i);
-          List<IndexableField> fieldable1 = doc1.getFields();
-          List<IndexableField> fieldable2 = doc2.getFields();
-          assertEquals("Different numbers of fields for doc " + i + ".", fieldable1.size(), fieldable2.size());
-          Iterator<IndexableField> itField1 = fieldable1.iterator();
-          Iterator<IndexableField> itField2 = fieldable2.iterator();
+          List<IndexableField> field1 = doc1.getFields();
+          List<IndexableField> field2 = doc2.getFields();
+          assertEquals("Different numbers of fields for doc " + i + ".", field1.size(), field2.size());
+          Iterator<IndexableField> itField1 = field1.iterator();
+          Iterator<IndexableField> itField2 = field2.iterator();
           while (itField1.hasNext()) {
             Field curField1 = (Field) itField1.next();
             Field curField2 = (Field) itField2.next();
Index: modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
===================================================================
--- modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java	(revision 1154109)
+++ modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java	(working copy)
@@ -300,9 +300,9 @@
    * @return A Collection of Field names (Strings)
    */
   protected Collection<String> getFieldsToHighlight(Document document) {
-    List<IndexableField> fieldables = document.getFields();
-    Set<String> result = new HashSet<String>(fieldables.size());
-    for (final IndexableField f : fieldables) {
+    List<IndexableField> fields = document.getFields();
+    Set<String> result = new HashSet<String>(fields.size());
+    for (final IndexableField f : fields) {
       result.add(f.name());
     }
     return result;
Index: solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
===================================================================
--- solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java	(revision 1154109)
+++ solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java	(working copy)
@@ -163,7 +163,7 @@
   
 
   /**
-   * @return a string representing a Fieldable's flags.  
+   * @return a string representing a IndexableField's flags.  
    */
   private static String getFieldFlags( IndexableField f )
   {
@@ -237,34 +237,34 @@
     final CharsRef spare = new CharsRef();
     SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
     for( Object o : doc.getFields() ) {
-      Field fieldable = (Field)o;
+      Field field = (Field)o;
       SimpleOrderedMap<Object> f = new SimpleOrderedMap<Object>();
       
-      SchemaField sfield = schema.getFieldOrNull( fieldable.name() );
+      SchemaField sfield = schema.getFieldOrNull( field.name() );
       FieldType ftype = (sfield==null)?null:sfield.getType();
 
       f.add( "type", (ftype==null)?null:ftype.getTypeName() );
       f.add( "schema", getFieldFlags( sfield ) );
-      f.add( "flags", getFieldFlags( fieldable ) );
+      f.add( "flags", getFieldFlags( field ) );
 
-      Term t = new Term(fieldable.name(), ftype!=null ? ftype.storedToIndexed(fieldable) : fieldable.stringValue());
+      Term t = new Term(field.name(), ftype!=null ? ftype.storedToIndexed(field) : field.stringValue());
 
-      f.add( "value", (ftype==null)?null:ftype.toExternal( fieldable ) );
+      f.add( "value", (ftype==null)?null:ftype.toExternal( field ) );
 
       // TODO: this really should be "stored"
-      f.add( "internal", fieldable.stringValue() );  // may be a binary number
+      f.add( "internal", field.stringValue() );  // may be a binary number
 
-      BytesRef bytes = fieldable.binaryValue(null);
+      BytesRef bytes = field.binaryValue(null);
       if (bytes != null) {
         f.add( "binary", Base64.byteArrayToBase64(bytes.bytes, bytes.offset, bytes.length));
       }
-      f.add( "boost", fieldable.boost() );
+      f.add( "boost", field.boost() );
       f.add( "docFreq", t.text()==null ? 0 : reader.docFreq( t ) ); // this can be 0 for non-indexed fields
             
       // If we have a term vector, return that
-      if( fieldable.storeTermVectors() ) {
+      if( field.storeTermVectors() ) {
         try {
-          TermFreqVector v = reader.getTermFreqVector( docId, fieldable.name() );
+          TermFreqVector v = reader.getTermFreqVector( docId, field.name() );
           if( v != null ) {
             SimpleOrderedMap<Integer> tfv = new SimpleOrderedMap<Integer>();
             for( int i=0; i<v.size(); i++ ) {
@@ -278,7 +278,7 @@
         }
       }
       
-      finfo.add( fieldable.name(), f );
+      finfo.add( field.name(), f );
     }
     return finfo;
   }
Index: solr/src/java/org/apache/solr/schema/FieldType.java
===================================================================
--- solr/src/java/org/apache/solr/schema/FieldType.java	(revision 1154109)
+++ solr/src/java/org/apache/solr/schema/FieldType.java	(working copy)
@@ -262,40 +262,6 @@
     return createField(field.getName(), val, newType, boost);
   }
 
-  /*protected Fieldable createField(String name, String val, org.apache.lucene.document.Field.Store storage, org.apache.lucene.document.Field.Index index,
-      org.apache.lucene.document.Field.TermVector vec, boolean omitNorms, boolean omitTFPos, float boost){
-    org.apache.lucene.document.Field f = new org.apache.lucene.document.Field(name,
-                        val,
-                        storage,
-                        index,
-                        vec);
-    f.setOmitNorms(omitNorms);
-    f.setOmitTermFreqAndPositions(omitTFPos);
-    f.setBoost(boost);
-    return f;
-  }
-
-  public Fieldable createField2(SchemaField field, Object value, float boost) {
-    if (!field.indexed() && !field.stored()) {
-      if (log.isTraceEnabled())
-        log.trace("Ignoring unindexed/unstored field: " + field);
-      return null;
-    }
-    
-    String val;
-    try {
-      val = toInternal(value.toString());
-    } catch (RuntimeException e) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + value + "'", e, false);
-    }
-    if (val==null) return null;
-
-    return createField(field.getName(), val, getFieldStore(field, val),
-            getFieldIndex(field, val), getFieldTermVec(field, val), field.omitNorms(),
-            field.omitTf(), boost);
-  }
-  */
-
   /**
    * Create the field from native Lucene parts.  Mostly intended for use by FieldTypes outputing multiple
    * Fields per SchemaField
