Index: modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java
===================================================================
--- modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java	(revision 1159007)
+++ modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java	(working copy)
@@ -107,7 +107,7 @@
   }
 
   public static void main(String[] args) throws Exception {
-
+    args = new String[] {"-i", "/home/j/wikien/enwiki-latest-pages-articles18.xml", "-o", "/home/j/wikien/latest-pages-articles18"};
     File wikipedia = null;
     File outputDir = new File("./enwiki");
     boolean keepImageOnlyDocs = true;
Index: lucene/src/test/org/apache/lucene/index/TestPayloads.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestPayloads.java	(revision 1159007)
+++ lucene/src/test/org/apache/lucene/index/TestPayloads.java	(working copy)
@@ -409,14 +409,14 @@
     /**
      * This Analyzer uses an WhitespaceTokenizer and PayloadFilter.
      */
-    private static class PayloadAnalyzer extends Analyzer {
+    public static class PayloadAnalyzer extends Analyzer {
         Map<String,PayloadData> fieldToData = new HashMap<String,PayloadData>();
         
-        void setPayloadData(String field, byte[] data, int offset, int length) {
+        public void setPayloadData(String field, byte[] data, int offset, int length) {
             fieldToData.put(field, new PayloadData(0, data, offset, length));
         }
 
-        void setPayloadData(String field, int numFieldInstancesToSkip, byte[] data, int offset, int length) {
+        public void setPayloadData(String field, int numFieldInstancesToSkip, byte[] data, int offset, int length) {
             fieldToData.put(field, new PayloadData(numFieldInstancesToSkip, data, offset, length));
         }
         
Index: lucene/src/test/org/apache/lucene/index/TestRAMReader.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestRAMReader.java	(revision 0)
+++ lucene/src/test/org/apache/lucene/index/TestRAMReader.java	(revision 0)
@@ -0,0 +1,451 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.RAMReaderManager.RAMReader;
+import org.apache.lucene.index.TestPayloads.PayloadAnalyzer;
+import org.apache.lucene.search.FieldCacheImpl;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.cache.CachedArray.IntValues;
+import org.apache.lucene.search.cache.IntValuesCreator;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
+
+public class TestRAMReader extends LuceneTestCase {
+  static String field1 = "text";
+  /**
+  public static class TempDoc {
+    public List<String> terms;
+    
+    public List<String> sortTerms() {
+      List<String> sortedTerms = new ArrayList<String>(terms);
+      Collections.sort(sortedTerms);
+      return sortedTerms;
+    }
+  }
+  **/
+  
+  //public void testMultipleTermsPerDoc() throws Exception {
+  //  Directory directory = new RAMDirectory();
+    
+  //}
+  
+  // TODO: test live docs, norms, and the field cache arrays grow automatically
+  
+  public void testFieldCaches() throws Exception {
+    Directory directory = new RAMDirectory();
+    AtomicInteger seq = new AtomicInteger(0);
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT,
+        new WhitespaceAnalyzer(Version.LUCENE_40));
+    iwc.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(1));
+    iwc.setRAMBufferSizeMB(32.0);
+    IndexWriter writer = new IndexWriter(directory, iwc);
+    for (int i = 0; i < 5; i++) {
+      writer.addDocument(createDoc5(seq.getAndIncrement()));
+    }
+    RAMReader reader1 = writer.getRAMReaders()[0];
+    
+    IndexSearcher searcher = new IndexSearcher(reader1);
+    // perform a sorting query to generate a field cache
+    TermQuery query = new TermQuery(new Term("text", "apple2"));
+    // create an int[] field cache on the id field
+    Sort sort = new Sort(new SortField("id", SortField.Type.INT));
+    TopFieldDocs topDocs = searcher.search(query, 10, sort);
+    // the ids int[] should be larger than the max doc of the reader
+    //int[] ids = FieldCacheImpl.DEFAULT.getInts(reader1, "id");
+    IntValues intValues = FieldCacheImpl.DEFAULT.getInts(reader1, "id", new IntValuesCreator( "id", null ));
+    assertTrue( intValues != null );
+    System.out.println("ids.length:"+intValues.values.length);
+    int[] arr1 = intValues.values;
+    int numTerms1 = intValues.numTerms;
+    int numDocs1 = intValues.numDocs;
+    assertTrue(intValues.values.length > reader1.maxDoc());
+    // there are 5 docs that have been indexed
+    // we will add 10 docs now
+    for (int i = 0; i < 10; i++) {
+      writer.addDocument(createDoc5(seq.getAndIncrement()));
+    }
+    // the intValues.values array should have grown
+    // and should have new values
+    assertTrue(intValues.values.length > arr1.length);
+    assertTrue(intValues.numDocs > numDocs1);
+    assertTrue(intValues.numTerms > numTerms1);
+    System.out.println("ids.length:"+intValues.values.length);
+    
+    reader1.close();
+    writer.close();
+    directory.close();
+  }
+  
+  public void testMultipleTermsPerDoc() throws Exception {
+    Directory directory = new RAMDirectory();
+    
+    AtomicInteger seq = new AtomicInteger(0);
+    
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT,
+        new WhitespaceAnalyzer(Version.LUCENE_40));
+    
+    iwc.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(1));
+    
+    iwc.setRAMBufferSizeMB(32.0);
+    IndexWriter writer = new IndexWriter(directory, iwc);
+    for (int i = 0; i < 5; i++) {
+      writer.addDocument(createDoc5(seq.getAndIncrement()));
+    }
+    RAMReader reader1 = writer.getRAMReaders()[0];
+    Terms terms = reader1.terms("text");
+    TermsEnum termsEnum = terms.iterator();
+    assertTrue( termsEnum.seekExact(new BytesRef("nextstep"), false) );
+    
+    DocsAndPositionsEnum docsPos = termsEnum.docsAndPositions(null, null);
+    while (docsPos.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+      int docID = docsPos.docID();
+      int freq = docsPos.freq();
+      assertEquals(1, freq);
+      int position = docsPos.nextPosition();
+      assertEquals(1, position);
+      //System.out.println("docID:"+docID+" freq:"+freq+" pos:"+pos);
+    }
+    
+    reader1.close();
+    writer.close();
+    directory.close();
+  }
+  
+  // test to ensure that the norms array and deleted docs
+  // grows automatically as documents are added and their
+  // respectie array lengths are exceeded
+  public void testNormsAndDeletesGrowing() throws Exception {
+    
+  }
+  
+  public void testTermPositions() throws Exception {
+    Directory directory = new RAMDirectory();
+    AtomicInteger seq = new AtomicInteger(0);
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT,
+        new WhitespaceAnalyzer(Version.LUCENE_40));
+    iwc.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(1));
+    iwc.setRAMBufferSizeMB(32.0);
+    IndexWriter writer = new IndexWriter(directory, iwc);
+    for (int i = 0; i < 5; i++) {
+      writer.addDocument(createDoc5(seq.getAndIncrement()));
+    }
+    RAMReader ramReader = writer.getRAMReaders()[0];
+    Terms cterms = ramReader.terms(field1);    
+    TermsEnum ctermsEnum = cterms.iterator();
+    boolean found = ctermsEnum.seekExact(new BytesRef("apple2"), false);
+    assertTrue(found);
+    DocsAndPositionsEnum positionsEnum = ctermsEnum.docsAndPositions(null, null);
+    while (positionsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+      int docID = positionsEnum.docID();
+      int freq = positionsEnum.freq();
+      System.out.println("docID:"+docID);
+      assertTrue(freq > 0);
+      int position = positionsEnum.nextPosition();
+      for (int x=0; x < freq; x++) {
+        System.out.println("position:"+position);
+      }
+    }
+    ramReader.close();
+    writer.close();
+    directory.close();
+  }
+
+  public void testDeletes() throws Exception {
+    Directory directory = new RAMDirectory();
+    
+    AtomicInteger seq = new AtomicInteger(0);
+    
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT,
+        new WhitespaceAnalyzer(Version.LUCENE_40));
+    iwc.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(1));
+    
+    iwc.setRAMBufferSizeMB(32.0);
+    IndexWriter writer = new IndexWriter(directory, iwc);
+    for (int i = 0; i < 5; i++) {
+      writer.addDocument(createDoc1(seq.getAndIncrement()));
+      writer.addDocument(createDoc2(seq.getAndIncrement()));
+    }
+    RAMReader reader1 = writer.getRAMReaders()[0];
+    Term delTerm1 = new Term("id", new BytesRef("2"));
+    int[] termdocs = toDocsArray(delTerm1, null, reader1);
+    assertEquals(termdocs[0], 2);
+    
+    writer.deleteDocuments(delTerm1);
+
+    RAMReader reader2 = writer.getRAMReaders()[0];
+    
+    // assert the document is deleted in reader2
+    assertTrue(reader1.getLiveDocs().get(2));
+    assertFalse(reader2.getLiveDocs().get(2));
+    
+    reader1.close();
+    reader2.close();
+    writer.close();
+    directory.close();
+  }
+  
+  // tests:
+  //   getting a RAM reader
+  //   getting a terms enum
+  //   terms enum seek to exact term
+  //   terms enum doc freq
+  //   docs enum / posting iteration
+  //   norms written and read properly
+  //   delete by term; and apply deletes
+  //   delete by query
+  //   payloads
+  //   document loadings
+  
+  // nocommit: need to test term vectors
+  public void test() throws Exception {
+    AtomicInteger seq = new AtomicInteger(0);
+    
+    Directory directory = new RAMDirectory();
+    
+    // test payloads
+    PayloadAnalyzer payloadAnalyzer = new PayloadAnalyzer();
+    byte[] payloadBytes = new byte[] {1, 2, 89};
+    payloadAnalyzer.setPayloadData(field1, payloadBytes, 0, payloadBytes.length);
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT,
+        payloadAnalyzer);
+    
+    iwc.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(1));
+    
+    iwc.setRAMBufferSizeMB(32.0);
+    IndexWriter writer = new IndexWriter(directory, iwc);
+    for (int i = 0; i < 5; i++) {
+      writer.addDocument(createDoc1(seq.getAndIncrement()));
+      writer.addDocument(createDoc2(seq.getAndIncrement()));
+    }
+    RAMReader[] ramReaders = writer.getRAMReaders();
+    RAMReader reader = ramReaders[0];
+    
+    Terms terms = reader.terms(field1);
+    String[] conTermsArr = toArray(terms.iterator());
+    // verify that the terms dictionary is sorted
+    assertTrue(Arrays.equals(conTermsArr, new String[] {"apple", "beos", "hp",
+        "motorola"}));
+    
+    // test docfreq and seeking
+    Terms cterms = reader.terms(field1);    
+    TermsEnum ctermsEnum = cterms.iterator();
+    assertTrue( ctermsEnum.seekExact(new BytesRef("beos"), false) );
+    int docFreq = ctermsEnum.docFreq();
+    assertEquals(5, docFreq);
+    
+    // assert postings are correct
+    DocsEnum docsEnum = ctermsEnum.docs(null, null);
+    int[] docs = toArray(docsEnum);
+    assertTrue(Arrays.equals(docs, new int[] {1, 3, 5, 7, 9}));
+    
+    // add more docs with (ipad, iphone) as terms
+    for (int i = 0; i < 6; i++) {
+      writer.addDocument(createDoc3(seq.getAndIncrement()));
+    }
+    
+    Term delTerm1 = new Term("id", "2");
+    writer.deleteDocuments(delTerm1);
+    
+    RAMReader reader2 = writer.getRAMReaders()[0];
+    byte[] norms = reader2.norms("text");
+    for (int x=0; x < reader2.maxDoc(); x++) {
+      assertTrue(norms[x] > 0);
+    }
+    
+    // assert the document is shown as deleted only in reader2
+    assertTrue(reader.getLiveDocs().get(2));
+    assertFalse(reader2.getLiveDocs().get(2));
+    
+    Term motTerm = new Term(field1, "motorola");
+    
+    int[] delTerm1Docs1 = toDocsArray(motTerm, reader.getLiveDocs(), reader);
+    System.out.println("delterms1:"+Arrays.toString(delTerm1Docs1));
+    assertEquals(5, delTerm1Docs1.length);
+    int[] delTerm1Docs12 = toDocsArray(motTerm, null, reader);
+    
+    assertTrue(Arrays.equals(delTerm1Docs1, delTerm1Docs12));
+    
+    int[] delTerm1Docs2 = toDocsArray(motTerm, reader2.getLiveDocs(), reader2);
+    
+    //System.out.println("delterms12:" Arrays.toString(delTerm1Docs12));
+    //System.out.println("delterms2:" Arrays.toString(delTerm1Docs2));
+    
+    assertTrue(arrayContains(2, delTerm1Docs1));
+    assertTrue(arrayContains(2, delTerm1Docs12));
+    assertFalse(arrayContains(2, delTerm1Docs2));
+    
+    // assert the new terms are present
+    Terms terms2 = reader2.terms(field1);
+    String[] conTermsArr2 = toArray(terms2.iterator());
+    assertTrue(Arrays.equals(conTermsArr2, new String[] {"apple", "beos", "hp",
+        "ipad", "iphone", "motorola"}));
+    
+    // assert that the previous reader is returning the same terms
+    terms = reader.terms(field1);
+    conTermsArr = toArray(terms.iterator());
+    assertTrue(Arrays.equals(conTermsArr, new String[] {"apple", "beos", "hp",
+        "motorola"}));
+    
+    // assert that seeking that a new term isn't found
+    // in the first reader
+    assertFalse( ctermsEnum.seekExact(new BytesRef("iphone"), false) );
+    
+    // test reading a document
+    assertEquals("apple beos", ramReaders[0].document(3).getField("text").stringValue());
+
+    // delete by query
+    Query dq = new TermQuery(new Term(field1, "apple"));
+    writer.deleteDocuments(dq);
+    RAMReader reader3 = writer.getRAMReaders()[0];
+    int[] appleDelDocs = toDocsArray(new Term(field1, "apple"), reader3.getLiveDocs(), reader3);
+    assertEquals(0, appleDelDocs.length);
+    //System.out.println("appleDelDocs:" Arrays.toString(appleDelDocs));
+    
+    TermFreqVector tfv = ramReaders[0].getTermFreqVector(1, field1);
+    BytesRef[] tfvTerms = tfv.getTerms();
+    assertTrue(terms.getComparator().compare(new BytesRef("apple"), tfvTerms[0]) == 0);
+    
+    // test the term freq and postings length matches
+    // later we can check per indexreader to insure we're not going over the max doc id
+    
+    // assert that payloads work
+    boolean found = ctermsEnum.seekExact(new BytesRef("apple"), false);
+    DocsAndPositionsEnum docsPos = ctermsEnum.docsAndPositions(null, null);
+    PayloadAttribute payload = null;
+    BytesRef payloadMatch = new BytesRef(payloadBytes);
+    while (docsPos.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+      int docID = docsPos.docID();
+      assertEquals(1, docsPos.freq());
+      int pos = docsPos.nextPosition();
+      assertEquals(0, pos);
+      assertTrue(docsPos.hasPayload());
+      assertEquals(new BytesRef(payloadBytes), docsPos.getPayload());
+    }
+    reader.close();
+    reader2.close();
+    reader3.close();
+    
+    writer.close();
+    directory.close();
+  }
+  
+  private static boolean arrayContains(int v, int[] arr) {
+    for (int x=0; x < arr.length; x++) {
+      if (arr[x] == v) return true;
+    }
+    return false;
+  }
+  
+  public static int[] toDocsArray(Term term, Bits liveDocs, RAMReader reader) throws IOException {
+    Terms cterms = reader.terms(term.field);    
+    TermsEnum ctermsEnum = cterms.iterator();
+    boolean found = ctermsEnum.seekExact(term.bytes(), false);
+    System.out.println("term: "+ctermsEnum.term());
+    if (found) {
+      DocsEnum docsEnum = ctermsEnum.docs(liveDocs, null);
+      return toArray(docsEnum);
+    }
+    return null;
+  }
+  
+  public static int[] toArray(DocsEnum docsEnum)
+      throws IOException {
+    List<Integer> docs = new ArrayList<Integer>();
+    while (docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+      int docID = docsEnum.docID();
+      docs.add(docID);
+    }
+    return ArrayUtil.toIntArray(docs);
+  }
+  
+  public static String[] toArray(TermsEnum termsEnum) throws IOException {
+    BytesRef term = null;
+    List<String> list = new ArrayList<String>();
+    while ((term = termsEnum.next()) != null) {
+      list.add(term.utf8ToString());
+    }
+    return (String[]) list.toArray(new String[0]);
+  }
+  
+  public static Document createDoc1(int id) {
+    Document doc = new Document();
+    doc.add(new Field("id", id+"", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+    doc.add(new Field(field1, "motorola hp", Field.Store.YES,
+        Field.Index.ANALYZED));
+    return doc;
+  }
+  
+  public static Document createDoc2(int id) {
+    Document doc = new Document();
+    doc.add(new Field("id", id+"", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+    doc.add(new Field(field1, "apple beos", Field.Store.YES,
+        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    return doc;
+  }
+  
+  /**
+   * Here we're adding multiple of the same term into the document
+   * so that the unit test will see more than one return from freq()
+   * of the docsenum.
+   */
+  public static Document createDoc4(int id) {
+    Document doc = new Document();
+    doc.add(new Field("id", id+"", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+    doc.add(new Field(field1, "apple beos apple beos steve jobs", Field.Store.YES,
+        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    return doc;
+  }
+  
+  public static Document createDoc5(int id) {
+    Document doc = new Document();
+    doc.add(new Field("id", id+"", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+    // a 1 token / term document
+    doc.add(new Field(field1, "apple2 nextstep", Field.Store.YES,
+        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    return doc;
+  }
+  
+  public static Document createDoc3(int id) {
+    Document doc = new Document();
+    doc.add(new Field("id", id+"", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+    doc.add(new Field(field1, "ipad iphone", Field.Store.YES,
+        Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    return doc;
+  }
+}
\ No newline at end of file
Index: lucene/src/test/org/apache/lucene/index/TestByteSlices.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestByteSlices.java	(revision 1159007)
+++ lucene/src/test/org/apache/lucene/index/TestByteSlices.java	(working copy)
@@ -47,7 +47,7 @@
           System.out.println("write stream=" + stream);
 
         if (starts[stream] == -1) {
-          final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
+          final int spot = pool.newSliceByLevel(0);
           starts[stream] = uptos[stream] = spot + pool.byteOffset;
           if (VERBOSE)
             System.out.println("  init to " + starts[stream]);
Index: lucene/src/test/org/apache/lucene/index/BenchmarkRAMReader.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/BenchmarkRAMReader.java	(revision 0)
+++ lucene/src/test/org/apache/lucene/index/BenchmarkRAMReader.java	(revision 0)
@@ -0,0 +1,204 @@
+package org.apache.lucene.index;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
+import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException;
+import org.apache.lucene.benchmark.byTask.utils.Config;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.NRTCachingDirectory;
+import org.apache.lucene.util.Version;
+
+// Run ExtractWikipedia to extract XML based wiki-en 
+// files into line based files
+// Wiki EN downloads: http://dumps.wikimedia.org/enwiki/latest/
+/**
+ * 1) We test the raw indexing speed up to an index size (the max RAM buffer
+ * size in MB) 2) Test the speed of indexing the same number of documents using
+ * NRT.
+ * 
+ * First we index and query using RT, up to the maximum size of the RAM buffer.
+ * We then throw away that segment because we're not testing flushing.
+ * 
+ * Second we index using NRTCachingDirectory and instead of getting an RT
+ * reader, we get an NRT reader which first flushes the RAM buffer to the
+ * underlying RAM directory. The merge scheduler is set to SerialMergeScheduler
+ * because with the RT test we are only indexing using one thread. It would be
+ * unfair to consume more CPU when indexing using NRT, even though in a
+ * production NRT system, ConcurrentMergeScheduler would [likeyl] be turned on.
+ * 
+ * A single thread performs a query in the background.  This is mainly to 
+ * cause a new IndexReader to be created while another thread is indexing
+ * at the maximum rate.
+ * 
+ * The type of query we are running is not important. Only that it should
+ * minimally impact indexing performance. This is not a search benchmark, we are
+ * mainly concerned about indexing performance. For search benchmarking, we need
+ * to measure the performance of an already created index, for RT the existing
+ * RAM buffer, for NRT, an index created using NRT.
+ */
+public class BenchmarkRAMReader {
+  public File rootDirectory;
+  int rounds = 3;
+  
+  public static void main(String[] args) throws Exception {
+    BenchmarkRAMReader b = new BenchmarkRAMReader();
+    
+    
+  }
+  
+  // 1) index until the RAM buffer is filled to the
+  // given size
+  // 2) Record the number of documents indexed
+  // 3) Index that number of documents using
+  // the NRT indexing system
+  
+  // nocommit: add a flush trigger that stops indexing
+  // when the maximum RAM buffer size is reached
+  public Result benchmark1(Directory dir, int maxdoc, MergePolicy mergePolicy,
+      Query query, boolean rt, long queryDelay) throws Exception {
+    ScheduledExecutorService exec = Executors.newScheduledThreadPool(1);
+    
+    IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_40,
+        new WhitespaceAnalyzer(Version.LUCENE_40));
+    // limit the number of DWPTs to one
+    // we're only using 1 thread to index
+    iwc.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(1));
+    iwc.setRAMBufferSizeMB(512.0);
+    iwc.setMergePolicy(mergePolicy);
+    // ideally we will index only up to the size of the buffer
+    iwc.setMergeScheduler(new SerialMergeScheduler());
+    // overwrite the existing index
+    iwc.setOpenMode(OpenMode.CREATE);
+    // FSDirectory fsdir = FSDirectory.open(path);
+    // NRTCachingDirectory nrtDir = new NRTCachingDirectory(fsdir, double
+    // maxMergeSizeMB, double maxCachedMB);
+    IndexWriter writer = new IndexWriter(dir, iwc);
+    
+    SearchTask searchTask = new SearchTask(query, writer, rt);
+    exec.scheduleAtFixedRate(searchTask, queryDelay, queryDelay, TimeUnit.MILLISECONDS);
+    
+    Result result = new Result();
+    
+    long start = System.currentTimeMillis();
+    result.docCount = benchmark2(writer, maxdoc);
+    result.duration = System.currentTimeMillis() - start;
+    writer.close();
+    return result;
+  }
+  
+  public class SearchTask implements Runnable {
+    IndexWriter writer;
+    boolean rt = true;
+    int timesRun = 0;
+    Query query;
+    
+    public SearchTask(Query query, IndexWriter writer, boolean rt) {
+      this.query = query;
+      this.writer = writer;
+      this.rt = rt;
+    }
+    
+    public void run() {
+      try {
+        // we're running a query just for kicks
+        // we're mainly measuring the cost of opening
+        // a new reader during all out single-threaded indexing
+        if (rt) {
+          // get the RT style reader
+          IndexReader[] readers = writer.getRTReaders();
+          MultiReader rtReader = new MultiReader(readers);
+          runQuery(query, rtReader);
+          // close the readers one by one
+          close(readers);
+          // not sure if we even need to close the MultiReader
+          rtReader.close();
+        } else {
+          // get the NRT style reader
+          IndexReader nrtReader = writer.getReader(true);
+          runQuery(query, nrtReader);
+          nrtReader.close();
+        }
+        timesRun++;
+      } catch (IOException ioe) {
+        ioe.printStackTrace();
+      }
+    }
+  }
+  
+  private static void close(IndexReader[] readers) throws IOException {
+    for (IndexReader reader : readers) {
+      reader.close();
+    }
+  }
+  
+  public void runQuery(Query query, IndexReader reader) throws IOException {
+    IndexSearcher searcher = new IndexSearcher(reader);
+    searcher.search(query, 10);
+  }
+  
+  static class Result {
+    public long duration;
+    public int docCount;
+  }
+  
+  public void testRT(Query query, long queryDelay) throws Exception {
+    File nrtDir = new File(rootDirectory, "rt");
+    nrtDir.mkdirs();
+    FSDirectory fsdir = FSDirectory.open(nrtDir);
+    
+    benchmark1(fsdir, Integer.MAX_VALUE, NoMergePolicy.NO_COMPOUND_FILES, query, true);
+    fsdir.close();
+  }
+  
+  public void testNRT(int maxdoc, Query query, long queryDelay) throws Exception {
+    File nrtDir = new File(rootDirectory, "nrt");
+    nrtDir.mkdirs();
+    FSDirectory fsdir = FSDirectory.open(nrtDir);
+    NRTCachingDirectory cachingDir = new NRTCachingDirectory(fsdir, 2.0, 25.0);
+    benchmark1(fsdir, maxdoc, null, query, false);
+    // closing the cachingDir closes the fsdir
+    cachingDir.close();
+  }
+  
+  public int benchmark2(IndexWriter writer, int maxdoc) throws Exception {
+    File dir = new File("/home/j/wikien/latest-pages-articles15");
+    DocMaker docMaker = new DocMaker();
+    Properties properties = new Properties();
+    properties.setProperty("content.source",
+        "org.apache.lucene.benchmark.byTask.feeds.DirContentSource");
+    properties.setProperty("docs.dir", dir.getAbsolutePath());
+    properties.setProperty("content.source.forever", "false");
+    docMaker.setConfig(new Config(properties));
+    docMaker.resetInputs();
+    Document doc = null;
+    int count = 0;
+    try {
+      while ((doc = docMaker.makeDocument()) != null) {
+        writer.addDocument(doc);
+        count++;
+        if (count == maxdoc) {
+          return count;
+        }
+        if (count % 1000 == 0) {
+          System.out.println("count: " + count);
+        }
+      }
+      System.out.println("final count:" + count);
+    } catch (NoMoreDataException e) {
+      // continue
+    }
+    return count;
+  }
+}
Index: lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java	(working copy)
@@ -137,7 +137,8 @@
               break;
             }
             if(vals.values == null) {
-              vals.values = new long[maxDoc];
+              int arraySize = getArraySize(reader);
+              vals.values = new long[arraySize];
             }
             vals.values[docID] = termval;
             vals.numDocs++;
@@ -155,7 +156,8 @@
     }
 
     if(vals.values == null) {
-      vals.values = new long[maxDoc];
+      int arraySize = getArraySize(reader);
+      vals.values = new long[arraySize];
     }
 
     if( vals.valid == null && vals.numDocs < 1 ) {
Index: lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java	(working copy)
@@ -108,7 +108,8 @@
 
     Terms terms = MultiFields.getTerms(reader, field);
     int maxDoc = reader.maxDoc();
-    vals.values = new short[maxDoc];
+    int arraySize = getArraySize(reader);
+    vals.values = new short[arraySize];
     if (terms != null) {
       final TermsEnum termsEnum = terms.iterator();
       FixedBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FixedBitSet( maxDoc ) : null;
Index: lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java	(working copy)
@@ -50,6 +50,11 @@
     this.parser = parser;
   }
 
+  public void addValue(BytesRef term, int docID, byte[] array) {
+    byte b = parser.parseByte(term);
+    array[docID] = b;
+  }
+  
   @Override
   public Class getArrayType() {
     return Byte.class;
@@ -107,7 +112,8 @@
 
     Terms terms = MultiFields.getTerms(reader, field);
     int maxDoc = reader.maxDoc();
-    vals.values = new byte[maxDoc];
+    int arraySize = getArraySize(reader);
+    vals.values = new byte[arraySize];
     if (terms != null) {
       final TermsEnum termsEnum = terms.iterator();
       FixedBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FixedBitSet( maxDoc ) : null;
Index: lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java	(working copy)
@@ -137,7 +137,8 @@
               break;
             }
             if(vals.values == null) {
-              vals.values = new float[maxDoc];
+              int arraySize = getArraySize(reader);
+              vals.values = new float[arraySize];
             }
             vals.values[docID] = termval;
             vals.numDocs++;
@@ -155,7 +156,8 @@
     }
 
     if(vals.values == null) {
-      vals.values = new float[maxDoc];
+      int arraySize = getArraySize(reader);
+      vals.values = new float[arraySize];
     }
 
     if( vals.valid == null && vals.numDocs < 1 ) {
Index: lucene/src/java/org/apache/lucene/search/cache/RTDocTermsCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/RTDocTermsCreator.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/search/cache/RTDocTermsCreator.java	(revision 0)
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.cache;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RAMReaderManager.RAMReader;
+import org.apache.lucene.search.FieldCache.DocTerms;
+
+public class RTDocTermsCreator extends EntryCreator<DocTerms> {
+  public String field;
+  
+  public RTDocTermsCreator(String field) {
+    if (field == null) {
+      throw new IllegalArgumentException("field can not be null");
+    }
+    this.field = field;
+  }
+
+  @Override
+  public DocTerms create( IndexReader reader ) throws IOException {
+    if (! (reader instanceof RAMReader) ) {
+      throw new RuntimeException("A non-RAM reader is not allowed.");
+    }
+    // the DocTerms object management is delegated to the RAMReader
+    RAMReader ramReader = (RAMReader)reader;
+    return ramReader.createRTDocTerms(field);
+  }
+  
+  @Override
+  public DocTerms validate(DocTerms entry, IndexReader reader) throws IOException {
+    return entry;
+  }
+
+  @Override
+  public SimpleEntryKey getCacheKey() {
+    return new SimpleEntryKey(RTDocTermsCreator.class, field);
+  }
+}
Index: lucene/src/java/org/apache/lucene/search/cache/CachedArray.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/CachedArray.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/CachedArray.java	(working copy)
@@ -22,14 +22,14 @@
 public abstract class CachedArray 
 {
   public Integer parserHashCode; // a flag to make sure you don't change what you are asking for in subsequent requests
-  public int numDocs;
-  public int numTerms;
+  public volatile int numDocs;
+  public volatile int numTerms;
 
   /**
    * NOTE: these Bits may have false positives for deleted documents.  That is,
    * Documents that are deleted may be marked as valid but the array value is not.
    */
-  public Bits valid;
+  public volatile Bits valid;
 
   public CachedArray() {
     this.parserHashCode = null;
@@ -47,32 +47,32 @@
   //-------------------------------------------------------------
 
   public static class ByteValues extends CachedArray {
-    public byte[] values = null;
+    public volatile byte[] values = null;
     @Override public byte[] getRawArray() { return values; }
   };
 
   public static class ShortValues extends CachedArray {
-    public short[] values = null;
+    public volatile short[] values = null;
     @Override public short[] getRawArray() { return values; }
   };
 
   public static class IntValues extends CachedArray {
-    public int[] values = null;
+    public volatile int[] values = null;
     @Override public int[] getRawArray() { return values; }
   };
 
   public static class FloatValues extends CachedArray {
-    public float[] values = null;
+    public volatile float[] values = null;
     @Override public float[] getRawArray() { return values; }
   };
 
   public static class LongValues extends CachedArray {
-    public long[] values = null;
+    public volatile long[] values = null;
     @Override public long[] getRawArray() { return values; }
   };
 
   public static class DoubleValues extends CachedArray {
-    public double[] values = null;
+    public volatile double[] values = null;
     @Override public double[] getRawArray() { return values; }
   };
 }
Index: lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java	(working copy)
@@ -136,7 +136,8 @@
               break;
             }
             if(vals.values == null) {
-              vals.values = new double[maxDoc];
+              int arraySize = getArraySize(reader);
+              vals.values = new double[arraySize];
             }
             vals.values[docID] = termval;
             vals.numDocs++;
@@ -154,7 +155,8 @@
     }
 
     if(vals.values == null) {
-      vals.values = new double[maxDoc];
+      int arraySize = getArraySize(reader);
+      vals.values = new double[arraySize];
     }
 
     if( vals.valid == null && vals.numDocs < 1 ) {
Index: lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java	(working copy)
@@ -24,12 +24,15 @@
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.RAMReaderManager.RAMReader;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache.Parser;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.RamUsageEstimator;
 
 public abstract class CachedArrayCreator<T extends CachedArray> extends EntryCreatorWithOptions<T>
 {
@@ -62,6 +65,21 @@
   }
 
   /**
+   * For the given reader, return the array (potentially over) sized
+   */
+  protected int getArraySize(IndexReader reader) {
+    // if the reader is a RAM reader then return an oversized array size
+    if (reader instanceof RAMReader) {
+      RAMReader ramReader = (RAMReader)reader;
+      // nocommit: the bytesPerElement is defaulted to int
+      return ArrayUtil.oversize(ramReader.maxDoc()+1, RamUsageEstimator.NUM_BYTES_INT);
+    } else {
+      // for normal [static] readers return the max doc 
+      return reader.maxDoc();
+    }
+  }
+  
+  /**
    * Note that the 'flags' are not part of the key -- subsequent calls to the cache
    * with different options will use the same cache entry.
    */
@@ -124,7 +142,8 @@
     Terms terms = MultiFields.getTerms(reader, field);
     if (terms != null) {
       final TermsEnum termsEnum = terms.iterator();
-      FixedBitSet validBits = new FixedBitSet( reader.maxDoc() );
+      int size = getArraySize(reader);
+      FixedBitSet validBits = new FixedBitSet( size );
       DocsEnum docs = null;
       while(true) {
         final BytesRef term = termsEnum.next();
Index: lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java	(working copy)
@@ -137,7 +137,8 @@
               break;
             }
             if(vals.values == null) {
-              vals.values = new int[maxDoc];
+              int arraySize = getArraySize(reader);
+              vals.values = new int[arraySize];
             }
             vals.values[docID] = termval;
             vals.numDocs++;
@@ -155,7 +156,8 @@
     }
 
     if(vals.values == null) {
-      vals.values = new int[maxDoc];
+      int arraySize = getArraySize(reader);
+      vals.values = new int[arraySize];
     }
 
     if( vals.valid == null && vals.numDocs < 1 ) {
Index: lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java	(working copy)
@@ -39,8 +39,9 @@
  * @since   lucene 1.4
  */
 public class FieldCacheImpl implements FieldCache {  // Made Public so that 
-	
-  private Map<Class<?>,Cache> caches;
+  private List<CacheCreationListener> cacheCreationListeners;
+  
+  protected Map<Class<?>,Cache> caches;
   FieldCacheImpl() {
     init();
   }
@@ -57,6 +58,39 @@
     caches.put(DocTermOrds.class, new Cache<DocTermOrds>(this));
   }
   
+  /**
+   * When a field value / entry is created, notify the given
+   * listener(s). 
+   */
+  public static abstract class CacheCreationListener {
+    public abstract void cacheCreated(IndexReader reader, Entry entry, Object value);
+  }
+  
+  public synchronized void addCacheCreationListener(CacheCreationListener listener) {
+    if (cacheCreationListeners == null) {
+      cacheCreationListeners = new ArrayList<CacheCreationListener>();
+    }
+    cacheCreationListeners.add(listener);
+  }
+
+  public synchronized boolean removeCacheCreationListener(CacheCreationListener listener) {
+    if (cacheCreationListeners != null) {
+      return cacheCreationListeners.remove(listener);
+    }
+    return false;
+  }
+  
+  /**
+   * Notify cache creation listeners that a new value has been created
+   */
+  private void notifyCacheCreationListeners(IndexReader reader, Entry entry, Object value) {
+    if (cacheCreationListeners != null) {
+      for (CacheCreationListener listener : cacheCreationListeners) {
+        listener.cacheCreated(reader, entry, value);
+      }
+    }
+  }
+  
   public synchronized void purgeAllCaches() {
     init();
   }
@@ -182,6 +216,10 @@
             progress.value = createValue(reader, key);
             synchronized (readerCache) {
               innerCache.put(key, progress.value);
+              if (wrapper instanceof FieldCacheImpl) {
+                // notify the 
+                ((FieldCacheImpl)wrapper).notifyCacheCreationListeners(reader, key, progress.value);
+              }
             }
 
             // Only check if key.custom (the parser) is
@@ -224,12 +262,12 @@
   }
 
   /** Expert: Every composite-key in the internal cache is of this type. */
-  static class Entry<T> {
-    final String field;        // which Fieldable
-    final EntryCreator<T> creator;       // which custom comparator or parser
+  public static class Entry<T> {
+    public final String field;        // which Fieldable
+    public final EntryCreator<T> creator;       // which custom comparator or parser
 
     /** Creates one of these objects for a custom comparator/parser. */
-    Entry (String field, EntryCreator<T> custom) {
+    public Entry (String field, EntryCreator<T> custom) {
       this.field = field;
       this.creator = custom;
     }
Index: lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/DocInverterPerField.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/DocInverterPerField.java	(working copy)
@@ -20,9 +20,11 @@
 import java.io.IOException;
 import java.io.Reader;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.search.Similarity;
 
 /**
  * Holds state for inverting all occurrences of a single
@@ -41,6 +43,7 @@
   final InvertedDocEndConsumerPerField endConsumer;
   final DocumentsWriterPerThread.DocState docState;
   final FieldInvertState fieldState;
+  final Similarity similarity;
 
   public DocInverterPerField(DocInverter parent, FieldInfo fieldInfo) {
     this.parent = parent;
@@ -49,6 +52,7 @@
     fieldState = parent.fieldState;
     this.consumer = parent.consumer.addField(this, fieldInfo);
     this.endConsumer = parent.endConsumer.addField(this, fieldInfo);
+    similarity = docState.similarityProvider.get(fieldInfo.name);
   }
 
   @Override
@@ -71,7 +75,7 @@
     for(int i=0;i<count;i++) {
 
       final Fieldable field = fields[i];
-
+      System.out.println("field:"+field.name());
       // TODO FI: this should be "genericized" to querying
       // consumer if it wants to see this particular field
       // tokenized.
@@ -190,13 +194,26 @@
 
         fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field);
         fieldState.boost *= field.getBoost();
+        
+        // nocommit: we can calculate the RT norms here
+        int docID = docState.docID;
+        // create or grow the norms array
+        if (fieldState.norms == null) {
+          fieldState.norms = new byte[1024];
+        } else if (fieldState.norms.length >= docID) {
+          fieldState.norms = ArrayUtil.grow(fieldState.norms, docID+1);
+        }
+        // calculate the norms array
+        fieldState.norms[docID] = similarity.computeNorm(fieldState);
+        
+        System.out.println("fieldState.length 1) "+fieldState.length+" norm["+docID+"]:"+fieldState.norms[docID]);
       }
 
       // LUCENE-2387: don't hang onto the field, so GC can
       // reclaim
       fields[i] = null;
     }
-
+    //System.out.println("fieldState.length 2) :"+fieldState.length);
     consumer.finish();
     endConsumer.finish();
   }
Index: lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java	(working copy)
@@ -24,9 +24,14 @@
 import java.io.PrintStream;
 import java.text.NumberFormat;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.Map;
+import java.util.HashMap;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.ConcurrentTermsDictPerField.RTReaderContextPerField;
+import org.apache.lucene.index.RAMReaderManager.RAMReader;
 import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.store.Directory;
@@ -37,7 +42,7 @@
 import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
 import org.apache.lucene.util.RamUsageEstimator;
 
-public class DocumentsWriterPerThread {
+public class DocumentsWriterPerThread extends AbstractDeletableIndex {
 
   /**
    * The IndexingChain must define the {@link #getChain(DocumentsWriter)} method
@@ -173,7 +178,12 @@
   DeleteSlice deleteSlice;
   private final NumberFormat nf = NumberFormat.getInstance();
   final Allocator byteBlockAllocator;
-
+  DocFieldProcessor docFieldProcessor;
+  RAMReaderManager ramReaderManager;
+  private long bufferedDeletesGen;
+  BitVector liveDocs;
+  // a lock for when a reader is created or documents are updated
+  ReentrantLock indexingLock = new ReentrantLock();
   
   public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent,
       FieldInfos fieldInfos, IndexingChain indexingChain) {
@@ -185,17 +195,129 @@
     this.docState = new DocState(this);
     this.docState.similarityProvider = parent.indexWriter.getConfig()
         .getSimilarityProvider();
+    
     bytesUsed = new AtomicLong(0);
     byteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
     consumer = indexingChain.getChain(this);
+    
+    if (consumer instanceof DocFieldProcessor) {
+      docFieldProcessor = (DocFieldProcessor) consumer;
+    }
+    
+    // set the liveDocs BV to be larger than needed
+    liveDocs = new BitVector(1024);
+    liveDocs.invertAll();
+    
     pendingDeletes = new BufferedDeletes();
+    ramReaderManager = new RAMReaderManager(this);
     initialize();
   }
   
+  // clone the live docs also set them as the latest
+  BitVector cloneLiveDocs(int newSize) {
+    BitVector newLiveDocs = liveDocs.clone(newSize);
+    this.liveDocs = newLiveDocs;
+    return newLiveDocs;
+  }
+
+  @Override
+  boolean isLive() {
+    return true;
+  }
+
+  @Override
+  void release(IndexReader reader) throws IOException {
+    // nocommit: we need to decref the reader here?
+  }
+
+  @Override
+  long getBufferedDeletesGen() {
+    return bufferedDeletesGen;
+  }
+
+  @Override
+  void setBufferedDeletesGen(long v) {
+    bufferedDeletesGen = v;
+  }
+  
+  public RAMReader getReader() throws IOException {
+    indexingLock.lock();
+    try {
+      return ramReaderManager.getReader();
+    } finally {
+      indexingLock.unlock();
+    }
+  }
+  
+  Map<String,RTReaderContextPerField> getReaderContextPerFields(long readerSeqID) {
+    assert indexingLock.isHeldByCurrentThread();
+    
+    int maxDocID = numDocsInRAM;
+    Map<String,TermsHashPerField> termsHashes = docFieldProcessor.getTermsHashPerFields();
+    Map<String,RTReaderContextPerField> map = new HashMap<String,RTReaderContextPerField>(termsHashes.size());
+    for (Map.Entry<String,TermsHashPerField> entry : termsHashes.entrySet()) {
+      String field = entry.getKey();
+      TermsHashPerField thfield = entry.getValue();
+      int numTerms = thfield.getNumTerms();
+      // get the norms from the FieldInvertState
+      byte[] norms = thfield.fieldState.norms;
+      RTReaderContextPerField readerCxt = thfield.getReaderContext(maxDocID, 
+          numTerms, readerSeqID, norms);
+      map.put(field, readerCxt);
+    }
+    return map;
+  }
+  
+  public FieldInfos getFieldInfos() {
+    return fieldInfos;
+  }
+  
   public DocumentsWriterPerThread(DocumentsWriterPerThread other, FieldInfos fieldInfos) {
     this(other.directory, other.parent, fieldInfos, other.parent.chain);
   }
+
+  public Map<String,TermsHashPerField> getTermsHashPerFields() {
+    return docFieldProcessor.getTermsHashPerFields();
+  }
   
+  /**
+   * Add a FieldListener to the given field
+   */
+  void addFieldListener(String field, FieldListener fieldListener) {
+    indexingLock.lock();
+    try {
+      //assert indexingLock.isHeldByCurrentThread();
+      Map<String,TermsHashPerField> map = getTermsHashPerFields();
+      TermsHashPerField termsHashField = map.get(field);
+      termsHashField.addFieldListener(fieldListener);
+    } finally {
+      indexingLock.unlock();
+    }
+  }
+  
+  StoredFieldsWriter getStoredFieldsWriter() {
+    return docFieldProcessor.fieldsWriter;
+  }
+
+  TermsHash getTermsHash() {
+    DocInverter docInverter = (DocInverter)docFieldProcessor.consumer;
+    TermsHash termsHash = (TermsHash)docInverter.consumer;
+    return termsHash;
+  }
+  
+  TermVectorsTermsWriter getTermVectorsTermsWriter() {
+    TermsHash termsHash = getTermsHash();
+    return (TermVectorsTermsWriter)termsHash.nextTermsHash.consumer;
+  }
+  
+  void flushTermVectors() throws IOException {
+    getTermVectorsTermsWriter().flush();
+  }
+  
+  void flushStoredFields() throws IOException {
+    getStoredFieldsWriter().fieldsWriter.flush();
+  }
+  
   void initialize() {
     deleteQueue = parent.deleteQueue;
     assert numDocsInRAM == 0 : "num docs " + numDocsInRAM;
@@ -214,134 +336,144 @@
   }
 
   public void updateDocument(Document doc, Analyzer analyzer, Term delTerm) throws IOException {
-    assert writer.testPoint("DocumentsWriterPerThread addDocument start");
-    assert deleteQueue != null;
-    docState.doc = doc;
-    docState.analyzer = analyzer;
-    docState.docID = numDocsInRAM;
-    if (segment == null) {
-      // this call is synchronized on IndexWriter.segmentInfos
-      segment = writer.newSegmentName();
-      assert numDocsInRAM == 0;
+    indexingLock.lock();
+    try {
+      assert writer.testPoint("DocumentsWriterPerThread addDocument start");
+      assert deleteQueue != null;
+      docState.doc = doc;
+      docState.analyzer = analyzer;
+      docState.docID = numDocsInRAM;
+      if (segment == null) {
+        // this call is synchronized on IndexWriter.segmentInfos
+        segment = writer.newSegmentName();
+        assert numDocsInRAM == 0;
+        if (INFO_VERBOSE) {
+          message(Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
+        }
+      
+      }
       if (INFO_VERBOSE) {
-        message(Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
+        message(Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
       }
-      
-    }
-    if (INFO_VERBOSE) {
-      message(Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
-    }
-    boolean success = false;
-    try {
+      boolean success = false;
       try {
-        consumer.processDocument(fieldInfos);
+        try {
+          consumer.processDocument(fieldInfos);
+        } finally {
+          docState.clear();
+        }
+        success = true;
       } finally {
-        docState.clear();
+        if (!success) {
+          if (!aborting) {
+            // mark document as deleted
+            deleteDocID(docState.docID);
+            numDocsInRAM++;
+            fieldInfos.revertUncommitted();
+          } else {
+            abort();
+          }
+        }
       }
-      success = true;
-    } finally {
-      if (!success) {
-        if (!aborting) {
-          // mark document as deleted
-          deleteDocID(docState.docID);
-          numDocsInRAM++;
-          fieldInfos.revertUncommitted();
-        } else {
+      success = false;
+      try {
+        consumer.finishDocument();
+        success = true;
+      } finally {
+        if (!success) {
           abort();
         }
       }
-    }
-    success = false;
-    try {
-      consumer.finishDocument();
-      success = true;
+      finishDocument(delTerm);
     } finally {
-      if (!success) {
-        abort();
-      }
+      indexingLock.unlock();
     }
-    finishDocument(delTerm);
   }
   
   public int updateDocuments(Iterable<Document> docs, Analyzer analyzer, Term delTerm) throws IOException {
-    assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
-    assert deleteQueue != null;
-    docState.analyzer = analyzer;
-    if (segment == null) {
-      // this call is synchronized on IndexWriter.segmentInfos
-      segment = writer.newSegmentName();
-      assert numDocsInRAM == 0;
+    indexingLock.lock();
+    try {
+      assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
+      assert deleteQueue != null;
+      docState.analyzer = analyzer;
+      if (segment == null) {
+        // this call is synchronized on IndexWriter.segmentInfos
+        segment = writer.newSegmentName();
+        assert numDocsInRAM == 0;
+        if (INFO_VERBOSE) {
+          message(Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
+        }
+      }
       if (INFO_VERBOSE) {
-        message(Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
+        message(Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
       }
-    }
-    if (INFO_VERBOSE) {
-      message(Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
-    }
-    int docCount = 0;
-    try {
-      for(Document doc : docs) {
-        docState.doc = doc;
-        docState.docID = numDocsInRAM;
-        docCount++;
+      int docCount = 0;
+      try {
+        for(Document doc : docs) {
+          docState.doc = doc;
+          docState.docID = numDocsInRAM;
+          docCount++;
 
-        boolean success = false;
-        try {
-          consumer.processDocument(fieldInfos);
-          success = true;
-        } finally {
-          if (!success) {
-            // An exc is being thrown...
+          boolean success = false;
+          try {
+            consumer.processDocument(fieldInfos);
+            success = true;
+          } finally {
+            if (!success) {
+              // An exc is being thrown...
 
-            if (!aborting) {
-              // One of the documents hit a non-aborting
-              // exception (eg something happened during
-              // analysis).  We now go and mark any docs
-              // from this batch that we had already indexed
-              // as deleted:
-              int docID = docState.docID;
-              final int endDocID = docID - docCount;
-              while (docID > endDocID) {
-                deleteDocID(docID);
-                docID--;
+              if (!aborting) {
+                // One of the documents hit a non-aborting
+                // exception (eg something happened during
+                // analysis).  We now go and mark any docs
+                // from this batch that we had already indexed
+                // as deleted:
+                int docID = docState.docID;
+                final int endDocID = docID - docCount;
+                while (docID > endDocID) {
+                  deleteDocID(docID);
+                  docID--;
+                }
+
+                // Incr here because finishDocument will not
+                // be called (because an exc is being thrown):
+                numDocsInRAM++;
+                fieldInfos.revertUncommitted();
+              } else {
+                abort();
               }
-
-              // Incr here because finishDocument will not
-              // be called (because an exc is being thrown):
-              numDocsInRAM++;
-              fieldInfos.revertUncommitted();
-            } else {
+            }
+          }
+          success = false;
+          try {
+            consumer.finishDocument();
+            success = true;
+          } finally {
+            if (!success) {
               abort();
             }
           }
+
+          finishDocument(null);
         }
-        success = false;
-        try {
-          consumer.finishDocument();
-          success = true;
-        } finally {
-          if (!success) {
-            abort();
-          }
+
+        // Apply delTerm only after all indexing has
+        // succeeded, but apply it only to docs prior to when
+        // this batch started:
+        if (delTerm != null) {
+          deleteQueue.add(delTerm, deleteSlice);
+          assert deleteSlice.isTailItem(delTerm) : "expected the delete term as the tail item";
+          deleteSlice.apply(pendingDeletes, numDocsInRAM-docCount);
         }
 
-        finishDocument(null);
+      } finally {
+        docState.clear();
       }
 
-      // Apply delTerm only after all indexing has
-      // succeeded, but apply it only to docs prior to when
-      // this batch started:
-      if (delTerm != null) {
-        deleteQueue.add(delTerm, deleteSlice);
-        assert deleteSlice.isTailItem(delTerm) : "expected the delete term as the tail item";
-        deleteSlice.apply(pendingDeletes, numDocsInRAM-docCount);
-      }
-
+      return docCount;
     } finally {
-      docState.clear();
+      indexingLock.unlock();
     }
-
-    return docCount;
   }
   
   private void finishDocument(Term delTerm) throws IOException {
Index: lucene/src/java/org/apache/lucene/index/AbstractDeletableIndex.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/AbstractDeletableIndex.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/index/AbstractDeletableIndex.java	(revision 0)
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+// methods from SegmentInfo and reader pool
+// that enable a reader or DWPT to be included
+// in the delete method of BufferedDeletesStream
+abstract class AbstractDeletableIndex {
+  abstract boolean isLive();
+  
+  abstract long getBufferedDeletesGen();
+  
+  abstract void setBufferedDeletesGen(long v);
+  
+  abstract IndexReader getReader() throws IOException;
+  
+  abstract void release(IndexReader reader) throws IOException;
+}
Index: lucene/src/java/org/apache/lucene/index/FieldsWriter.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/FieldsWriter.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/FieldsWriter.java	(working copy)
@@ -93,6 +93,11 @@
     indexStream = fdx;
   }
 
+  void flush() throws IOException {
+    fieldsStream.flush();
+    indexStream.flush();
+  }
+  
   void setFieldsStream(IndexOutput stream) {
     this.fieldsStream = stream;
   }
Index: lucene/src/java/org/apache/lucene/index/ByteSliceReader.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/ByteSliceReader.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/ByteSliceReader.java	(working copy)
@@ -22,6 +22,7 @@
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.ByteBlocks;
 
 /* IndexInput that knows how to read the byte slices written
  * by Posting and PostingVector.  We read the bytes in
@@ -29,7 +30,7 @@
  * point we read the forwarding address of the next slice
  * and then jump to it.*/
 final class ByteSliceReader extends DataInput {
-  ByteBlockPool pool;
+  ByteBlocks pool;
   int bufferUpto;
   byte[] buffer;
   public int upto;
@@ -39,7 +40,7 @@
 
   public int endIndex;
 
-  public void init(ByteBlockPool pool, int startIndex, int endIndex) {
+  public void init(ByteBlocks pool, int startIndex, int endIndex) {
 
     assert endIndex-startIndex >= 0;
     assert startIndex >= 0;
Index: lucene/src/java/org/apache/lucene/index/FieldListener.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/FieldListener.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/index/FieldListener.java	(revision 0)
@@ -0,0 +1,33 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Used by the field cache to be notified of new values being indexed.
+ */
+public abstract class FieldListener {
+  /**
+   * 
+   * @param termID Corresponds to the DWPT's term id for this term
+   * @param term
+   * @param docID
+   */
+  public abstract void newValue(int termID, BytesRef term, int docID, TermsHashPerField termsHashPerField);
+}
Index: lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java	(working copy)
@@ -28,6 +28,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexWriter.ReaderPool;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
@@ -144,9 +145,9 @@
   }
 
   // Sorts SegmentInfos from smallest to biggest bufferedDelGen:
-  private static final Comparator<SegmentInfo> sortSegInfoByDelGen = new Comparator<SegmentInfo>() {
+  private static final Comparator<AbstractDeletableIndex> sortSegInfoByDelGen = new Comparator<AbstractDeletableIndex>() {
     @Override
-    public int compare(SegmentInfo si1, SegmentInfo si2) {
+    public int compare(AbstractDeletableIndex si1, AbstractDeletableIndex si2) {
       final long cmp = si1.getBufferedDeletesGen() - si2.getBufferedDeletesGen();
       if (cmp > 0) {
         return 1;
@@ -160,11 +161,32 @@
   
   /** Resolves the buffered deleted Term/Query/docIDs, into
    *  actual deleted docIDs in the liveDocs BitVector for
-   *  each SegmentReader. */
-  public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List<SegmentInfo> infos) throws IOException {
+   *  each SegmentReader. 
+   *  
+   *  Also apply all existing deletes to DWPTs for realtime search
+   */
+  // nocommit: anywhere we apply deletes to segments
+  // we also apply them to DWPTs
+  public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, 
+      List<SegmentInfo> infos, List<DocumentsWriterPerThread> dwpts) throws IOException {
+    List<AbstractDeletableIndex> deletables = new ArrayList<AbstractDeletableIndex>();
+    // put the DWPTs and reader together
+    for (SegmentInfo info : infos) {
+      deletables.add(new InfoDeletableIndex(info, readerPool));
+    }
+    if (dwpts != null) {
+      for (DocumentsWriterPerThread dwpt : dwpts) {
+        deletables.add(dwpt);
+      }
+    }
+    return applyDeletes(readerPool, deletables);
+  }
+  
+  public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, 
+      List<AbstractDeletableIndex> deletables) throws IOException {
     final long t0 = System.currentTimeMillis();
 
-    if (infos.size() == 0) {
+    if (deletables.size() == 0) {
       return new ApplyDeletesResult(false, nextGen++, null);
     }
 
@@ -176,17 +198,17 @@
     }
 
     if (infoStream != null) {
-      message("applyDeletes: infos=" + infos + " packetCount=" + deletes.size());
+      message("applyDeletes: infos=" + deletables + " packetCount=" + deletes.size());
     }
 
-    List<SegmentInfo> infos2 = new ArrayList<SegmentInfo>();
-    infos2.addAll(infos);
-    Collections.sort(infos2, sortSegInfoByDelGen);
+    List<AbstractDeletableIndex> deletables2 = new ArrayList<AbstractDeletableIndex>();
+    deletables2.addAll(deletables);
+    Collections.sort(deletables2, sortSegInfoByDelGen);
 
     CoalescedDeletes coalescedDeletes = null;
     boolean anyNewDeletes = false;
 
-    int infosIDX = infos2.size()-1;
+    int infosIDX = deletables2.size()-1;
     int delIDX = deletes.size()-1;
 
     List<SegmentInfo> allDeleted = null;
@@ -195,8 +217,8 @@
       //System.out.println("BD: cycle delIDX=" + delIDX + " infoIDX=" + infosIDX);
 
       final FrozenBufferedDeletes packet = delIDX >= 0 ? deletes.get(delIDX) : null;
-      final SegmentInfo info = infos2.get(infosIDX);
-      final long segGen = info.getBufferedDeletesGen();
+      final AbstractDeletableIndex deletable = deletables2.get(infosIDX);
+      final long segGen = deletable.getBufferedDeletesGen();
 
       if (packet != null && segGen < packet.delGen()) {
         //System.out.println("  coalesce");
@@ -220,8 +242,9 @@
         //System.out.println("  eq");
 
         // Lock order: IW -> BD -> RP
-        assert readerPool.infoIsLive(info);
-        final SegmentReader reader = readerPool.get(info, false, IOContext.READ);
+        assert deletable.isLive();
+        
+        final IndexReader reader = deletable.getReader();
         int delCount = 0;
         final boolean segAllDeletes;
         try {
@@ -236,7 +259,7 @@
           delCount += applyQueryDeletes(packet.queriesIterable(), reader);
           segAllDeletes = reader.numDocs() == 0;
         } finally {
-          readerPool.release(reader, IOContext.Context.READ);
+          deletable.release(reader);
         }
         anyNewDeletes |= delCount > 0;
 
@@ -244,11 +267,13 @@
           if (allDeleted == null) {
             allDeleted = new ArrayList<SegmentInfo>();
           }
-          allDeleted.add(info);
+          if (deletable instanceof InfoDeletableIndex) {
+            allDeleted.add( ((InfoDeletableIndex)deletable).info );
+          }
         }
 
         if (infoStream != null) {
-          message("seg=" + info + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] delCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
+          message("seg=" + deletables + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] delCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
         }
 
         if (coalescedDeletes == null) {
@@ -262,15 +287,16 @@
          */
         delIDX--;
         infosIDX--;
-        info.setBufferedDeletesGen(nextGen);
+        deletable.setBufferedDeletesGen(nextGen);
 
       } else {
         //System.out.println("  gt");
 
         if (coalescedDeletes != null) {
           // Lock order: IW -> BD -> RP
-          assert readerPool.infoIsLive(info);
-          SegmentReader reader = readerPool.get(info, false, IOContext.READ);
+          assert deletable.isLive();
+          IndexReader reader = deletable.getReader();
+
           int delCount = 0;
           final boolean segAllDeletes;
           try {
@@ -278,7 +304,7 @@
             delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), reader);
             segAllDeletes = reader.numDocs() == 0;
           } finally {
-            readerPool.release(reader, IOContext.Context.READ);
+            deletable.release(reader);
           }
           anyNewDeletes |= delCount > 0;
 
@@ -286,14 +312,16 @@
             if (allDeleted == null) {
               allDeleted = new ArrayList<SegmentInfo>();
             }
-            allDeleted.add(info);
+            if (deletable instanceof InfoDeletableIndex) {
+              allDeleted.add( ((InfoDeletableIndex)deletable).info );
+            }
           }
 
           if (infoStream != null) {
-            message("seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] delCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
+            message("seg=" + deletable + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] delCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
           }
         }
-        info.setBufferedDeletesGen(nextGen);
+        deletable.setBufferedDeletesGen(nextGen);
 
         infosIDX--;
       }
@@ -358,7 +386,7 @@
   }
 
   // Delete by Term
-  private synchronized long applyTermDeletes(Iterable<Term> termsIter, SegmentReader reader) throws IOException {
+  private synchronized long applyTermDeletes(Iterable<Term> termsIter, IndexReader reader) throws IOException {
     long delCount = 0;
     Fields fields = reader.fields();
     if (fields == null) {
@@ -431,7 +459,7 @@
   }
 
   // Delete by query
-  private synchronized long applyQueryDeletes(Iterable<QueryAndLimit> queriesIter, SegmentReader reader) throws IOException {
+  private synchronized long applyQueryDeletes(Iterable<QueryAndLimit> queriesIter, IndexReader reader) throws IOException {
     long delCount = 0;
     final AtomicReaderContext readerContext = (AtomicReaderContext) reader.getTopReaderContext();
     for (QueryAndLimit ent : queriesIter) {
Index: lucene/src/java/org/apache/lucene/index/ConcurrentTermsDictPerField.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/ConcurrentTermsDictPerField.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/index/ConcurrentTermsDictPerField.java	(revision 0)
@@ -0,0 +1,541 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.ByteBlocks;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Per field terms dictionary.
+ * 
+ * The ConcurrentTermsDictPerField can be updated while
+ * while current or previous index readers are opening
+ * and iterating over the terms dictionary.
+ * 
+ * The exact terms per reader is maintained by limiting
+ * terms to the readerSeqID of the reader (ie, 
+ * at or below the reader seq id).
+ * 
+ * 
+ */
+public class ConcurrentTermsDictPerField {
+  private final ConcurrentSkipListMap<BytesRef,TermInfo> termMap;
+  private final String field;
+  private final Comparator<BytesRef> comparator;
+  private RTReaderContextPerField prevReaderContext;
+  
+  public ConcurrentTermsDictPerField(String field,
+      Comparator<BytesRef> comparator) {
+    this.field = field;
+    this.comparator = comparator;
+    termMap = new ConcurrentSkipListMap<BytesRef,TermInfo>(comparator);
+  }
+  
+  public Terms terms(RTReaderContextPerField readerContext) {
+    return new ConcurrentTerms(readerContext);
+  }
+  
+  // the value in a terms dictionary map
+  public static class TermInfo {
+    public int termID;
+    public long readerGen; // seq id at the point-in-time this term was added
+    
+    public TermInfo(int termID, long readerGen) {
+      this.termID = termID;
+      this.readerGen = readerGen;
+    }
+  }
+  
+  private static boolean isValid(TermInfo ti, long readerGen) {
+    return ti.readerGen <= readerGen;
+  }
+  
+  private class ConcurrentTermsEnum extends TermsEnum {
+    ConcurrentNavigableMap<BytesRef,TermInfo> subMap;
+    Iterator<Map.Entry<BytesRef,TermInfo>> iterator;
+    int termID = -1;
+    BytesRef term;
+    RTReaderContextPerField readerContext;
+    
+    public ConcurrentTermsEnum(RTReaderContextPerField readerContext) {
+      this.readerContext = readerContext;
+      this.subMap = termMap;
+      iterator = subMap.entrySet().iterator();
+    }
+    
+    @Override
+    // nocommit: we can return the correct totalTermFreq
+    // however that will require maintaining a specific 
+    // array that is copied per reader
+    public long totalTermFreq() throws IOException {
+      return -1;
+    }
+    
+    @Override
+    public Comparator<BytesRef> getComparator() {
+      return comparator;
+    }
+    
+    @Override
+    public long ord() {
+      return termID;
+    }
+    
+    @Override
+    public int docFreq() {
+      // return the unique term freq maintained
+      // per reader
+      return readerContext.termFreqs[termID];
+    }
+    
+    /**
+     * If the term is valid for this reader context
+     */
+    private boolean isValid(TermInfo termInfo) {
+      return ConcurrentTermsDictPerField.isValid(termInfo, readerContext.readerGen);
+    }
+    
+    @Override
+    public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+      return getDocsEnum(termID, readerContext, skipDocs);
+    }
+    
+    @Override
+    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs,
+        DocsAndPositionsEnum reuse) throws IOException {
+      return getDocsEnum(termID, readerContext, skipDocs);
+    }
+    
+    /**
+     * @param useCache
+     *          is being ignored
+     */
+    @Override
+    public SeekStatus seekCeil(BytesRef start, boolean useCache)
+        throws IOException {
+      if (start != null) {
+        subMap = termMap.tailMap(start, true);
+      } else {
+        subMap = termMap;
+      }
+      iterator = subMap.entrySet().iterator();
+      TermInfo termInfo = subMap.firstEntry().getValue();
+      //System.out.println(""+termInfo.)
+      termID = termInfo.termID;
+      if (comparator.compare(subMap.firstKey(), start) == 0
+          && isValid(termInfo)) {
+        return SeekStatus.FOUND;
+      } else if (subMap.size() == 0) {
+        return SeekStatus.END;
+      } else {
+        return SeekStatus.NOT_FOUND;
+      }
+    }
+    
+    @Override
+    public void seekExact(long ord) throws IOException {
+      // we can simply and directly specify the
+      // the term id, which should be the same as the ordinal
+      termID = (int) ord;
+    }
+    
+    @Override
+    public BytesRef next() throws IOException {
+      if (iterator.hasNext()) {
+        Map.Entry<BytesRef,TermInfo> entry = iterator.next();
+        TermInfo ti = entry.getValue();
+        if (isValid(ti)) {
+          termID = ti.termID;
+          term = entry.getKey();
+          return term;
+        } else {
+          return next();
+        }
+      } else {
+        return null;
+      }
+    }
+    
+    @Override
+    public BytesRef term() throws IOException {
+      return term;
+    }
+  }
+  
+  /**
+   * Frozen point in time set of values per field per reader
+   */
+  public static class RTReaderContextPerField {
+    final int uniqueTermCount;
+    final int maxDocID;
+    final long readerGen;
+    final FreqProxPostingsArray postingsArray;
+    final int[] termFreqs;
+    final ByteBlocks byteBlocks;
+    final IntBlocks intRead;
+    final FieldInfo fieldInfo;
+    final byte[] norms;
+    
+    public RTReaderContextPerField(int uniqueTermCount, int maxDocID,
+        long readerGen, FreqProxPostingsArray postingsArray, int[] termFreqs,
+        ByteBlocks byteBlocks, IntBlocks intRead, FieldInfo fieldInfo,
+        byte[] norms) {
+      this.uniqueTermCount = uniqueTermCount;
+      this.maxDocID = maxDocID;
+      this.readerGen = readerGen;
+      this.postingsArray = postingsArray;
+      this.byteBlocks = byteBlocks;
+      this.intRead = intRead;
+      this.termFreqs = termFreqs;
+      this.fieldInfo = fieldInfo;
+      this.norms = norms;
+    }
+    
+    /**
+     * Get the start of this posting directly from the byteStarts array
+     */
+    int getStart(int termID, int stream) {
+      return postingsArray.byteStarts[termID] + stream
+          * ByteBlockPool.FIRST_LEVEL_SIZE;
+    }
+    
+    /**
+     * Init the byte slice reader for the freq stream using 
+     * the start and end positions.  The end position is taken 
+     * from the postings array.
+     */
+    public void initReaderFreq(ByteSliceReader reader, int termID) {
+      int start = getStart(termID, 0);
+      int end = postingsArray.freqUptosRT[termID];
+      if (RAMReaderManager.DEBUG)
+        System.out.println("initReaderFreq start:"+start+" end:"+end);
+      reader.init(byteBlocks, start, end);
+    }
+    
+    /**
+     * Init the byte slice reader for the prox stream using 
+     * the start and end positions.  The end position is taken 
+     * from the postings array.
+     */
+    public void initReaderProx(ByteSliceReader reader, int termID) {
+      int start = getStart(termID, 1);
+      int end = postingsArray.proxUptosRT[termID];
+      reader.init(byteBlocks, start, end);
+    }
+  }
+  
+  private class ConcurrentTerms extends Terms {
+    RTReaderContextPerField readerContext;
+    
+    public ConcurrentTerms(RTReaderContextPerField readerContext) {
+      this.readerContext = readerContext;
+    }
+    
+    public long getSumTotalTermFreq() throws IOException {
+      return -1;
+    }
+    
+    public long getSumDocFreq() throws IOException {
+      return -1;
+    }
+    
+    public Comparator<BytesRef> getComparator() throws IOException {
+      return comparator;
+    }
+    
+    int getTermID(BytesRef term) {
+      TermInfo ti = termMap.get(term);
+      if (ti == null) {
+        return -1;
+      }
+      if (isValid(ti, readerContext.readerGen)) {
+        return ti.termID;
+      } else {
+        return -1;
+      }
+    }
+    
+    @Override
+    public TermsEnum iterator() throws IOException {
+      return new ConcurrentTermsEnum(readerContext);
+    }
+    
+    @Override
+    public long getUniqueTermCount() throws IOException {
+      return readerContext.uniqueTermCount;
+    }
+    
+    @Override
+    public int docFreq(BytesRef text) throws IOException {
+      int termID = getTermID(text);
+      if (termID == -1) {
+        return 0;
+      }
+      return readerContext.termFreqs[termID];
+    }
+    
+    @Override
+    public DocsEnum docs(Bits skipDocs, BytesRef term, DocsEnum reuse)
+        throws IOException {
+      int termID = getTermID(term);
+      if (termID != -1) {
+        return getDocsEnum(termID, readerContext, skipDocs);
+      } else {
+        return null;
+      }
+    }
+    
+    @Override
+    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef term,
+        DocsAndPositionsEnum reuse) throws IOException {
+      int termID = getTermID(term);
+      if (termID != -1) {
+        return getDocsEnum(termID, readerContext, skipDocs);
+      } else {
+        return null;
+      }
+    }
+  }
+  
+  DocsAndPositionsEnum getDocsEnum(int termID,
+      RTReaderContextPerField readerContext, Bits skipDocs) {
+    return new Postings(termID, readerContext, readerContext.fieldInfo,
+        skipDocs);
+  }
+  
+  // add the new terms to the term map
+  public void update(RTReaderContextPerField newReader) {
+    // nocommit: assert we're in the indexing lock here
+    int numPostings = newReader.uniqueTermCount;
+    int start = 0;
+    if (prevReaderContext != null) {
+      start = prevReaderContext.uniqueTermCount;
+      assert newReader.readerGen > prevReaderContext.readerGen;
+    }
+    for (int x = start; x < numPostings; x++) {
+      BytesRef term = new BytesRef();
+      int textStart = newReader.postingsArray.textStarts[x];
+      newReader.byteBlocks.setBytesRef(term, textStart);
+      if (RAMReaderManager.DEBUG)
+        System.out.println("addterm "+field+":"+term.utf8ToString());
+      termMap.put(term, new TermInfo(x, newReader.readerGen));
+    }
+    prevReaderContext = newReader;
+  }
+  
+  public class Postings extends DocsAndPositionsEnum {
+    int numDocs = 0;
+    int docID = 0;
+    int docFreq = 0;
+    final ByteSliceReader freq;
+    ByteSliceReader prox;
+    final int termID;
+    final int maxDocID;
+    final FieldInfo fieldInfo;
+    BytesRef payload;
+    int payloadLength;
+    int position = 0;
+    int positionIndex = 0;
+    final Bits bits;
+    RTReaderContextPerField readerContext;
+    final FreqProxPostingsArray postingsArray;
+    final FieldInfo.IndexOptions indexOptions;
+    boolean endReached = false;
+    
+    public Postings(int termID, RTReaderContextPerField readerContext,
+        FieldInfo fieldInfo, Bits bits) {
+      this.termID = termID;
+      this.readerContext = readerContext;
+      this.maxDocID = readerContext.maxDocID;
+      this.postingsArray = readerContext.postingsArray;
+      this.fieldInfo = fieldInfo;
+      this.bits = bits;
+      
+      BytesRef text = new BytesRef();
+      final int textStart = readerContext.postingsArray.textStarts[termID];
+      
+      freq = new ByteSliceReader();
+      readerContext.byteBlocks.setBytesRef(text, textStart);
+      readerContext.initReaderFreq(freq, termID);
+      
+      this.indexOptions = fieldInfo.indexOptions;
+      
+      if (indexOptions != IndexOptions.DOCS_ONLY) {
+        prox = new ByteSliceReader();
+        readerContext.initReaderProx(prox, termID);
+      }
+    }
+    
+    public int read() throws IOException {
+      initBulkResult();
+      int count = 0;
+      final int[] docs = bulkResult.docs.ints;
+      final int[] freqs = bulkResult.freqs.ints;
+      while (count < docs.length) {
+        final int doc = nextDoc();
+        if (doc != NO_MORE_DOCS) {
+          docs[count] = doc;
+          freqs[count] = freq();
+          count++;
+        } else {
+          break;
+        }
+      }
+      return count;
+    }
+    
+    @Override
+    public BulkReadResult getBulkResult() {
+      initBulkResult();
+      return bulkResult;
+    }
+    
+    @Override
+    public BytesRef getPayload() throws IOException {
+      return payload;
+    }
+    
+    @Override
+    public boolean hasPayload() {
+      return payload != null;
+    }
+    
+    @Override
+    public int nextPosition() throws IOException {
+      if (indexOptions == IndexOptions.DOCS_ONLY) {
+        throw new IOException("no position information for this field: "
+            + fieldInfo.name);
+      }
+      if (positionIndex >= docFreq) {
+        throw new IOException("over the term freq of " + freq);
+      }
+      // omitTermFreqAndPositions == false so we do write positions &
+      // payload
+      final int code = prox.readVInt();
+      position = code >> 1;
+      
+      final int payloadLength;
+      // nocommit: not sure what this is for
+      final BytesRef thisPayload;
+      
+      if ((code & 1) != 0) {
+        // This position has a payload
+        payloadLength = prox.readVInt();
+        if (payload == null) {
+          payload = new BytesRef();
+          payload.bytes = new byte[payloadLength];
+        } else if (payload.bytes.length < payloadLength) {
+          payload.grow(payloadLength);
+        }
+        prox.readBytes(payload.bytes, 0, payloadLength);
+        payload.length = payloadLength;
+        thisPayload = payload;
+      } else {
+        payloadLength = 0;
+        thisPayload = null;
+      }
+      positionIndex++;
+      return position;
+    }
+    
+    @Override
+    public int freq() {
+      return docFreq;
+    }
+    
+    @Override
+    public int docID() {
+      return docID;
+    }
+    
+    @Override
+    public int nextDoc() throws IOException {
+      position = 0;
+      payload = null;
+      payloadLength = -1;
+      positionIndex = 0;
+      while (true) {
+        int termFreq = readerContext.termFreqs[termID];
+        //System.out.println("termFreq: "+termFreq);
+        // if the termFreq is 1 then the 
+        // lastDocIDsRT has the doc id value
+        // ie, it has not been written to the 
+        // byte buffers yet
+        if (freq.eof() || termFreq == 1) {
+          if (!endReached) {
+            // take a look at the last doc array
+            // it could be less than what we have already read?
+            docID = postingsArray.lastDocIDsRT[termID];
+            if (indexOptions != IndexOptions.DOCS_ONLY) {
+              docFreq = postingsArray.lastDocFreqsRT[termID];
+            }
+            endReached = true;
+          } else {
+            // EOF
+            docID = DocsAndPositionsEnum.NO_MORE_DOCS;
+            break;
+          }
+        } else {
+          final int code = freq.readVInt();
+          if (indexOptions == IndexOptions.DOCS_ONLY) {
+            docID += code;
+          } else {
+            docID += code >>> 1;
+            if ((code & 1) != 0) {
+              docFreq = 1;
+            } else {
+              docFreq = freq.readVInt();
+            }
+          }
+          assert docID != postingsArray.lastDocIDsRT[termID];
+        }
+        // we could have doc ids greater than the max doc id
+        // for this reader
+        // return no more docs, if the max doc id has already been reached
+        if (docID >= maxDocID) {
+          return DocIdSetIterator.NO_MORE_DOCS;
+        }
+        if (bits == null || bits.get(docID)) {
+          break;
+        }
+      }
+      return docID;
+    }
+    
+    @Override
+    public int advance(int target) throws IOException {
+      while (nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+        if (target >= docID) {
+          return docID;
+        }
+      }
+      return docID = NO_MORE_DOCS;
+    }
+  }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java	(working copy)
@@ -44,6 +44,10 @@
   private Fieldable[] storedFields;
   private int[] fieldNumbers;
 
+  public void flush() throws IOException {
+    fieldsWriter.flush();
+  }
+  
   public void reset() {
     numStoredFields = 0;
     storedFields = new Fieldable[1];
Index: lucene/src/java/org/apache/lucene/index/RAMReaderManager.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/RAMReaderManager.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/index/RAMReaderManager.java	(revision 0)
@@ -0,0 +1,493 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.ConcurrentTermsDictPerField.RTReaderContextPerField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexReader.ReaderFinishedListener;
+import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldCache.DocTerms;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitVector;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.util.MapBackedSet;
+
+public class RAMReaderManager {
+  public static final boolean DEBUG = true;
+  private Map<String,ConcurrentTermsDictPerField> map = new HashMap<String,ConcurrentTermsDictPerField>();
+  private DocumentsWriterPerThread dwpt;
+  private Comparator<BytesRef> comparator;
+  private CloseableThreadLocal<FieldsReader> fieldsReaderLocal = new FieldsReaderLocal();
+  private CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new TermVectorsLocal();
+  private long termVectorsFlushSeqID = -1;
+  private long fieldsReaderFlushSeqID = -1;
+  private TermVectorsReader termVectorsReader;
+  private FieldsReader fieldsReader;
+  private int numDeleted = 0;
+  private AtomicInteger ref = new AtomicInteger(1);
+  private AtomicLong readerSeqID = new AtomicLong(0);
+  
+  public RAMReaderManager(DocumentsWriterPerThread dwpt) {
+    this.dwpt = dwpt;
+    comparator = BytesRef.getUTF8SortedAsUTF16Comparator();
+  }
+  
+  void ensureTermVectorsFlushed(long seqID) throws IOException {
+    if (seqID > termVectorsFlushSeqID) {
+      dwpt.flushTermVectors();
+    }
+  }
+  
+  void ensureFieldsReaderFlushed(long seqID) throws IOException {
+    if (seqID > fieldsReaderFlushSeqID) {
+      dwpt.flushStoredFields();
+    }
+  }
+
+  FieldsReader getFieldsReaderLocal() {
+    return fieldsReaderLocal.get();
+  }
+
+  synchronized void decRef() throws IOException {
+    if (ref.decrementAndGet() == 0) {
+      if (termVectorsReader != null) {
+        termVectorsReader.close();
+      }
+      if (fieldsReader != null) {
+        fieldsReader.close();
+      }
+    }
+  }
+  
+  private void createTermVectorsReader() throws IOException {
+    assert termVectorsReader == null;
+    FieldInfos fieldInfos = dwpt.getFieldInfos();
+    IOContext ioContext = IOContext.READ;
+    termVectorsReader = new TermVectorsReader(dwpt.directory, dwpt.segment,
+        fieldInfos, ioContext);
+  }
+  
+  private void createFieldsReader() throws IOException {
+    assert fieldsReader == null;
+    FieldInfos fieldInfos = dwpt.getFieldInfos();
+    fieldsReader = new FieldsReader(dwpt.directory, dwpt.segment, fieldInfos);
+  }
+  
+  public RAMReader getReader() throws IOException {
+    int maxDoc = dwpt.getNumDocsInRAM();
+    long readerID = readerSeqID.getAndIncrement();
+    
+    Map<String,RTReaderContextPerField> contextMap = dwpt
+        .getReaderContextPerFields(readerID);
+    FieldInfos fieldInfos = dwpt.getFieldInfos();
+    List<FieldInfo> fieldInfoList = new ArrayList<FieldInfo>();
+    for (int x = 0; x < fieldInfos.size(); x++) {
+      fieldInfoList.add(fieldInfos.fieldInfo(x));
+    }
+    // ensure the size of the live docs bitvector
+    // is large enough
+    BitVector liveDocs = dwpt.cloneLiveDocs(maxDoc);
+    RAMReader reader = new RAMReader(readerID, fieldInfoList, numDeleted,
+        liveDocs,
+        maxDoc, contextMap);
+    ref.incrementAndGet();
+    return reader;
+  }
+  
+  public class RTDocTerms extends DocTerms {
+    final String field;
+    final RAMReader ramReader;
+    final RTReaderContextPerField readerContext;
+    final int[] docTermIDs;
+    final int size;
+    
+    public RTDocTerms(String field, RAMReader ramReader, 
+        RTReaderContextPerField readerContext, 
+        int[] docTermIDs, int size) {
+      this.field = field;
+      this.ramReader = ramReader;
+      this.readerContext = readerContext;
+      this.docTermIDs = docTermIDs;
+      this.size = size;
+    }
+    
+    @Override
+    public BytesRef getTerm(int docID, BytesRef ret) {
+      int termID = docTermIDs[docID];
+      int textStart = readerContext.postingsArray.textStarts[termID];
+      readerContext.byteBlocks.setBytesRef(ret, textStart);
+      return ret;
+    }
+
+    @Override
+    public boolean exists(int docID) {
+      return false;
+    }
+
+    @Override
+    public int size() {
+      return size;
+    }
+  }
+  
+  public class RAMReader extends IndexReader {
+    private final Map<String,RTReaderContextPerField> contextMap;
+    private final int maxDoc;
+    final int numDeleted;
+    final BitVector liveDocs;
+    final List<FieldInfo> fieldInfos;
+    private final ReaderContext readerContext = new AtomicReaderContext(this);
+    final long readerID;
+    
+    public RAMReader(long readerID, List<FieldInfo> fieldInfos, 
+        int numDeleted,
+        BitVector liveDocs, int maxDoc,
+        Map<String,RTReaderContextPerField> contextMap) {
+      this.readerID = readerID;
+      this.fieldInfos = fieldInfos;
+      this.numDeleted = numDeleted;
+      this.liveDocs = liveDocs;
+      this.maxDoc = maxDoc;
+      this.contextMap = contextMap;
+      
+      readerFinishedListeners = new MapBackedSet<ReaderFinishedListener>(new ConcurrentHashMap<ReaderFinishedListener,Boolean>());
+      
+      // update each terms dict to the current point-in-time
+      for (Map.Entry<String,RTReaderContextPerField> entry : contextMap
+          .entrySet()) {
+        String field = entry.getKey();
+        RTReaderContextPerField context = entry.getValue();
+        ConcurrentTermsDictPerField termsDict = getTermsDict(field);
+        termsDict.update(context);
+      }
+    }
+
+    // create the DocTerms for this RAM reader
+    public RTDocTerms createRTDocTerms(String field) throws IOException {
+      RTReaderContextPerField readerContext = contextMap.get(field);
+      int size = readerContext.uniqueTermCount;
+      // nocommit: plugin the static var that attaches to the size of an int
+      // allocate an array pointing of doc ids to term ids
+      int[] termDocIDs = new int[ArrayUtil.oversize(size+1, 4)];
+      // iterate over the terms of the field
+      // attach the termID to the docid in the int[]
+      Terms terms = terms(field);
+      final TermsEnum termsEnum = terms.iterator();      
+      DocsEnum docs = null;
+      while(true) {
+        final BytesRef term = termsEnum.next();
+        if (term == null) {
+          break;
+        }
+        docs = termsEnum.docs(liveDocs, docs);
+        while (true) {
+          final int docID = docs.nextDoc();
+          if (docID == DocIdSetIterator.NO_MORE_DOCS) {
+            break;
+          }
+          // the ord is the termID
+          termDocIDs[docID] = (int)termsEnum.ord();
+        }
+      }
+      return new RTDocTerms(field, this, readerContext, termDocIDs, size);
+    }
+
+    @Override
+    public Object getCoreCacheKey() {
+      // return the DWPT as the cache core key
+      return dwpt;
+    }
+
+    @Override
+    public ReaderContext getTopReaderContext() {
+      return readerContext;
+    }
+    
+    @Override
+    public PerDocValues perDocValues() throws IOException {
+      return null;
+    }
+    
+    @Override
+    public byte[] norms(String field) throws IOException {
+      RTReaderContextPerField readerContext = contextMap.get(field);
+      return readerContext.norms;
+    }
+    
+    private long getReaderID() {
+      return readerSeqID.get();
+    }
+    
+    @Override
+    public TermFreqVector[] getTermFreqVectors(int docNum) throws IOException {
+      ensureTermVectorsFlushed(getReaderID());
+      return termVectorsLocal.get().get(docNum);
+    }
+    
+    @Override
+    public TermFreqVector getTermFreqVector(int docNum, String field)
+        throws IOException {
+      ensureTermVectorsFlushed(getReaderID());
+      return termVectorsLocal.get().get(docNum, field);
+    }
+    
+    @Override
+    public void getTermFreqVector(int docNum, String field,
+        TermVectorMapper mapper) throws IOException {
+      ensureTermVectorsFlushed(getReaderID());
+      termVectorsLocal.get().get(docNum, field, mapper);
+    }
+    
+    @Override
+    public void getTermFreqVector(int docNum, TermVectorMapper mapper)
+        throws IOException {
+      ensureTermVectorsFlushed(getReaderID());
+      termVectorsLocal.get().get(docNum, mapper);
+    }
+    
+    @Override
+    public boolean isCurrent() throws CorruptIndexException, IOException {
+      return true;
+    }
+    
+    @Override
+    public int maxDoc() {
+      return maxDoc;
+    }
+    
+    @Override
+    public Document document(int doc, FieldSelector fieldSelector)
+        throws CorruptIndexException, IOException {
+      ensureFieldsReaderFlushed(getReaderID());
+      FieldsReader fieldsReader = getFieldsReaderLocal();
+      return fieldsReader.doc(doc, fieldSelector);
+    }
+    
+    @Override
+    public Bits getLiveDocs() {
+      return liveDocs;
+    }
+    
+    @Override
+    public Terms terms(String field) {
+      RTReaderContextPerField context = contextMap.get(field);
+      if (context == null) return null;
+      ConcurrentTermsDictPerField termsDict = getTermsDict(field);
+      return termsDict.terms(context);
+    }
+    
+    public class RAMFields extends Fields {
+      public FieldsEnum iterator() throws IOException {
+        return new RAMFieldsEnum(fieldInfos);
+      }
+      
+      public Terms terms(String field) throws IOException {
+        return RAMReader.this.terms(field);
+      }
+    }
+    
+    public class RAMFieldsEnum extends FieldsEnum {
+      Iterator<FieldInfo> iterator;
+      String field = null;
+      
+      public RAMFieldsEnum(List<FieldInfo> fieldInfos) {
+        iterator = fieldInfos.iterator();
+      }
+      
+      @Override
+      public String next() throws IOException {
+        if (iterator.hasNext()) {
+          return field = iterator.next().name;
+        }
+        return null;
+      }
+      
+      @Override
+      public TermsEnum terms() throws IOException {
+        return RAMReader.this.terms(field).iterator();
+      }
+    }
+    
+    @Override
+    protected void doCommit(Map<String,String> commitUserData)
+        throws IOException {
+      // not implemented, there is nothing to commit
+      // for a RT RAM reader
+    }
+    
+    @Override
+    public int numDocs() {
+      return maxDoc - numDeleted;
+    }
+    
+    @Override
+    public Collection<String> getFieldNames(IndexReader.FieldOption fieldOption) {
+      Set<String> fieldSet = new HashSet<String>();
+      for (int i = 0; i < fieldInfos.size(); i++) {
+        FieldInfo fi = fieldInfos.get(i);
+        if (fieldOption == IndexReader.FieldOption.ALL) {
+          fieldSet.add(fi.name);
+        } else if (!fi.isIndexed
+            && fieldOption == IndexReader.FieldOption.UNINDEXED) {
+          fieldSet.add(fi.name);
+        } else if (fi.indexOptions == IndexOptions.DOCS_ONLY
+            && fieldOption == IndexReader.FieldOption.OMIT_TERM_FREQ_AND_POSITIONS) {
+          fieldSet.add(fi.name);
+        } else if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS
+            && fieldOption == IndexReader.FieldOption.OMIT_POSITIONS) {
+          fieldSet.add(fi.name);
+        } else if (fi.storePayloads
+            && fieldOption == IndexReader.FieldOption.STORES_PAYLOADS) {
+          fieldSet.add(fi.name);
+        } else if (fi.isIndexed
+            && fieldOption == IndexReader.FieldOption.INDEXED) {
+          fieldSet.add(fi.name);
+        } else if (fi.isIndexed && fi.storeTermVector == false
+            && fieldOption == IndexReader.FieldOption.INDEXED_NO_TERMVECTOR) {
+          fieldSet.add(fi.name);
+        } else if (fi.storeTermVector == true
+            && fi.storePositionWithTermVector == false
+            && fi.storeOffsetWithTermVector == false
+            && fieldOption == IndexReader.FieldOption.TERMVECTOR) {
+          fieldSet.add(fi.name);
+        } else if (fi.isIndexed && fi.storeTermVector
+            && fieldOption == IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR) {
+          fieldSet.add(fi.name);
+        } else if (fi.storePositionWithTermVector
+            && fi.storeOffsetWithTermVector == false
+            && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION) {
+          fieldSet.add(fi.name);
+        } else if (fi.storeOffsetWithTermVector
+            && fi.storePositionWithTermVector == false
+            && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET) {
+          fieldSet.add(fi.name);
+        } else if ((fi.storeOffsetWithTermVector && fi.storePositionWithTermVector)
+            && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) {
+          fieldSet.add(fi.name);
+        }
+      }
+      return fieldSet;
+    }
+    
+    @Override
+    protected void doDelete(int docNum) throws CorruptIndexException,
+        IOException {
+      // delete from the live docs
+      liveDocs.getAndClear(docNum);
+    }
+    
+    @Override
+    protected void doSetNorm(int doc, String field, byte value)
+        throws CorruptIndexException, IOException {
+      throw new UnsupportedOperationException("");
+    }
+    
+    @Override
+    protected void doUndeleteAll() throws CorruptIndexException, IOException {
+      throw new UnsupportedOperationException("");
+    }
+    
+    @Override
+    public boolean hasDeletions() {
+      return numDeleted > 0;
+    }
+    
+    @Override
+    public Fields fields() throws IOException {
+      return new RAMFields();
+    }
+    
+    @Override
+    public long getUniqueTermCount() throws IOException {
+      long termCount = 0;
+      for (RTReaderContextPerField context : contextMap.values()) {
+        termCount = context.uniqueTermCount;
+      }
+      return termCount;
+    }
+    
+    @Override
+    protected void doClose() throws IOException {
+      termVectorsLocal.close();
+      fieldsReaderLocal.close();
+      RAMReaderManager.this.decRef();
+    }
+  }
+
+  private class FieldsReaderLocal extends CloseableThreadLocal<FieldsReader> {
+    @Override
+    protected FieldsReader initialValue() {
+      if (fieldsReader == null) {
+        try {
+          createFieldsReader();
+        } catch (IOException ioe) {
+          throw new RuntimeException(ioe);
+        }
+      }
+      return (FieldsReader) fieldsReader.clone();
+    }
+  }
+  
+  private class TermVectorsLocal extends
+      CloseableThreadLocal<TermVectorsReader> {
+    @Override
+    protected TermVectorsReader initialValue() {
+      if (termVectorsReader == null) {
+        try {
+          createTermVectorsReader();
+        } catch (IOException ioe) {
+          throw new RuntimeException(ioe);
+        }
+      }
+      try {
+        return (TermVectorsReader) termVectorsReader.clone();
+      } catch (CloneNotSupportedException ce) {
+        throw new RuntimeException(ce);
+      }
+    }
+  }
+  
+  ConcurrentTermsDictPerField getTermsDict(String field) {
+    ConcurrentTermsDictPerField ctdpf = map.get(field);
+    if (ctdpf == null) {
+      ctdpf = new ConcurrentTermsDictPerField(field, comparator);
+      map.put(field, ctdpf);
+    }
+    return ctdpf;
+  }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/index/IntBlockPool.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/IntBlockPool.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/IntBlockPool.java	(working copy)
@@ -19,10 +19,8 @@
  * limitations under the License.
  */
 
-final class IntBlockPool {
+final class IntBlockPool extends IntBlocks {
 
-  public int[][] buffers = new int[10][];
-
   int bufferUpto = -1;                        // Which buffer we are upto
   public int intUpto = DocumentsWriterPerThread.INT_BLOCK_SIZE;             // Where we are in head buffer
 
@@ -32,6 +30,7 @@
   final private DocumentsWriterPerThread docWriter;
 
   public IntBlockPool(DocumentsWriterPerThread docWriter) {
+    super(new int[10][]);
     this.docWriter = docWriter;
   }
 
Index: lucene/src/java/org/apache/lucene/index/InfoDeletableIndex.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/InfoDeletableIndex.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/index/InfoDeletableIndex.java	(revision 0)
@@ -0,0 +1,38 @@
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.IndexWriter.ReaderPool;
+import org.apache.lucene.store.IOContext;
+
+public class InfoDeletableIndex extends AbstractDeletableIndex {
+  SegmentInfo info;
+  ReaderPool readerPool;
+  
+  public InfoDeletableIndex(SegmentInfo info, ReaderPool readerPool) {
+    this.info = info;
+    this.readerPool = readerPool;
+  }
+  
+  boolean isLive() {
+    return readerPool.infoIsLive(info);
+  }
+  
+  long getBufferedDeletesGen() {
+    return info.getBufferedDeletesGen();
+  }
+  
+  void setBufferedDeletesGen(long v) {
+    info.setBufferedDeletesGen(v);
+  }
+  
+  IndexReader getReader() throws IOException {
+    final SegmentReader reader = readerPool.get(info, false, IOContext.READ);
+    return reader;
+  }
+  
+  void release(IndexReader reader) throws IOException {
+    assert reader instanceof SegmentReader;
+    readerPool.release((SegmentReader)reader, IOContext.Context.READ);
+  }
+}
Index: lucene/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java	(working copy)
@@ -37,7 +37,7 @@
   public abstract boolean hasPayload();
 
   @Override
-  public final int read() {
+  public int read() throws IOException {
     throw new UnsupportedOperationException();
   }
 
Index: lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/TermsHashPerField.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/TermsHashPerField.java	(working copy)
@@ -20,14 +20,19 @@
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.List;
+import java.util.ArrayList;
 
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
 import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.ByteBlocks;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 import org.apache.lucene.util.BytesRefHash.BytesStartArray;
 import org.apache.lucene.util.BytesRefHash.MaxBytesLengthExceededException;
+import org.apache.lucene.index.ConcurrentTermsDictPerField.RTReaderContextPerField;
 
 final class TermsHashPerField extends InvertedDocConsumerPerField {
   private static final int HASH_INIT_SIZE = 4;
@@ -56,6 +61,10 @@
 
   ParallelPostingsArray postingsArray;
   private final AtomicLong bytesUsed;
+  int lastContextTermID;
+  BytesRef fcTempTerm = new BytesRef();
+  boolean capturingTerms = false;
+  List<FieldListener> fieldListeners = new ArrayList<FieldListener>();
 
   public TermsHashPerField(DocInverterPerField docInverterPerField, final TermsHash termsHash, final TermsHash nextTermsHash, final FieldInfo fieldInfo) {
     intPool = termsHash.intPool;
@@ -77,7 +86,65 @@
     else
       nextPerField = null;
   }
+  
+  void addFieldListener(FieldListener fieldListener) {
+    capturingTerms = true;
+    this.fieldListeners.add(fieldListener);
+  }
+  
+  // this is called to add the BytesRef to the field caches
+  // associated with this field
+  void captureTerm(int termID, int docID) {
+    assert capturingTerms;
+    if (fieldListeners.size() > 0) {
+      // fill in this term into the BytesRef
+      bytesHash.get(termID, fcTempTerm);
+      if (RAMReaderManager.DEBUG)
+        System.out.println("captureTerm termID:"+termID+" "+fcTempTerm.utf8ToString()+" docID:"+docID);
+    }
+    // notify the field listeners
+    for (FieldListener fieldListener : fieldListeners) {
+      fieldListener.newValue(termID, fcTempTerm, docID, this);
+    }
+  }
+  
+  public int getNumTerms() {
+    return bytesHash.size();
+  }
+  
+  // nocommit: assert that we're in a lock
+  public RTReaderContextPerField getReaderContext(int maxDocID, int numPostings, 
+      long seqID, byte[] norms) {
+    //assert numPostings >= lastGetContextTermID;
+    FreqProxPostingsArray readerPostings = (FreqProxPostingsArray)postingsArray; 
+    // copy the freq uptos to the read only array
+    System.arraycopy(readerPostings.freqUptos, 0, readerPostings.freqUptosRT, 0, readerPostings.freqUptos.length);
+    // copy the prox uptos to the read only array
+    System.arraycopy(readerPostings.proxUptos, 0, readerPostings.proxUptosRT, 0, readerPostings.proxUptos.length);
+    // copy the lastDocIDs
+    System.arraycopy(readerPostings.lastDocIDs, 0, readerPostings.lastDocIDsRT, 0, readerPostings.lastDocIDs.length);
+    // copy the last docFreqs
+    System.arraycopy(readerPostings.docFreqs, 0, readerPostings.lastDocFreqsRT, 0, readerPostings.docFreqs.length);
+    
+    // make a read-only copy of the byte blocks
+    ByteBlocks byteBlocks = ByteBlocks.copyRef(bytePool);
+    // make a read-only copy of the int blocks
+    IntBlocks intRead = intPool.copyRef();
 
+    // nocommit: we can pool the termFreq arrays
+    int[] termFreqs = new int[readerPostings.termFreqs.length];
+    // make a copy of the term freqs
+    System.arraycopy(readerPostings.termFreqs, 0, termFreqs, 0, termFreqs.length);
+    
+    RTReaderContextPerField readerContext = new RTReaderContextPerField(numPostings, 
+        maxDocID, seqID,
+      readerPostings, termFreqs, 
+      byteBlocks, intRead, 
+      fieldInfo, norms);
+    lastContextTermID = numPostings;
+    return readerContext;
+  }
+  
   void shrinkHash(int targetSize) {
     // Fully free the bytesHash on each flush but keep the pool untouched
     // bytesHash.clear will clear the ByteStartArray and in turn the ParallelPostingsArray too
@@ -157,7 +224,7 @@
       postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset;
 
       for(int i=0;i<streamCount;i++) {
-        final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
+        final int upto = bytePool.newSliceByLevel(0);
         intUptos[intUptoStart+i] = upto + bytePool.byteOffset;
       }
       postingsArray.byteStarts[termID] = intUptos[intUptoStart];
@@ -220,7 +287,7 @@
       postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset;
 
       for(int i=0;i<streamCount;i++) {
-        final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
+        final int upto = bytePool.newSliceByLevel(0);
         intUptos[intUptoStart+i] = upto + bytePool.byteOffset;
       }
       postingsArray.byteStarts[termID] = intUptos[intUptoStart];
@@ -242,7 +309,7 @@
   int[] intUptos;
   int intUptoStart;
 
-  void writeByte(int stream, byte b) {
+  void writeByte(int stream, byte b, int termID) {
     int upto = intUptos[intUptoStart+stream];
     byte[] bytes = bytePool.buffers[upto >> ByteBlockPool.BYTE_BLOCK_SHIFT];
     assert bytes != null;
@@ -255,22 +322,31 @@
     }
     bytes[offset] = b;
     (intUptos[intUptoStart+stream])++;
+    
+    if (postingsArray instanceof FreqProxPostingsArray) {
+      FreqProxPostingsArray postings = (FreqProxPostingsArray)postingsArray;
+      if (stream == 0) {
+        postings.freqUptos[termID] = (intUptos[intUptoStart+stream]);
+      } else if (stream == 1) {
+        postings.proxUptos[termID] = (intUptos[intUptoStart+stream]);
+      }
+    }
   }
 
-  public void writeBytes(int stream, byte[] b, int offset, int len) {
+  public void writeBytes(int stream, byte[] b, int offset, int len, int termID) {
     // TODO: optimize
     final int end = offset + len;
     for(int i=offset;i<end;i++)
-      writeByte(stream, b[i]);
+      writeByte(stream, b[i], termID);
   }
 
-  void writeVInt(int stream, int i) {
+  void writeVInt(int stream, int i, int termID) {
     assert stream < streamCount;
     while ((i & ~0x7F) != 0) {
-      writeByte(stream, (byte)((i & 0x7f) | 0x80));
+      writeByte(stream, (byte)((i & 0x7f) | 0x80), termID);
       i >>>= 7;
     }
-    writeByte(stream, (byte) i);
+    writeByte(stream, (byte) i, termID);
   }
 
   @Override
Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java	(working copy)
@@ -67,6 +67,16 @@
     fieldsWriter = new StoredFieldsWriter(docWriter);
   }
 
+  public Map<String,TermsHashPerField> getTermsHashPerFields() {
+    Map<String,TermsHashPerField> map = new HashMap<String,TermsHashPerField>();
+    Collection<DocFieldConsumerPerField> fields = fields();
+    for (DocFieldConsumerPerField f : fields) {
+      DocInverterPerField dipf = (DocInverterPerField)f;
+      map.put(f.getFieldInfo().name, (TermsHashPerField)dipf.consumer);
+    }
+    return map;
+  }
+  
   @Override
   public void flush(SegmentWriteState state) throws IOException {
 
@@ -286,7 +296,6 @@
       final DocFieldProcessorPerField perField = fields[i];
       perField.consumer.processFields(perField.fields, perField.fieldCount);
     }
-
     if (docState.maxTermPrefix != null && docState.infoStream != null) {
       docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
       docState.maxTermPrefix = null;
Index: lucene/src/java/org/apache/lucene/index/TermsEnum.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/TermsEnum.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/TermsEnum.java	(working copy)
@@ -27,7 +27,7 @@
 /** Iterator to seek ({@link #seekCeil(BytesRef)}, {@link
  * #seekExact(BytesRef,boolean)}) or step through ({@link
  * #next} terms to obtain frequency information ({@link
- * #docFreq}), {@link DocsEnum} or {@link
+ * #freq}), {@link DocsEnum} or {@link
  * DocsAndPositionsEnum} for the current term ({@link
  * #docs}.
  * 
Index: lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java	(working copy)
@@ -107,12 +107,12 @@
     }
 
     if (payload != null && payload.length > 0) {
-      termsHashPerField.writeVInt(1, (proxCode<<1)|1);
-      termsHashPerField.writeVInt(1, payload.length);
-      termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length);
+      termsHashPerField.writeVInt(1, (proxCode<<1)|1, termID);
+      termsHashPerField.writeVInt(1, payload.length, termID);
+      termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length, termID);
       hasPayloads = true;
     } else
-      termsHashPerField.writeVInt(1, proxCode<<1);
+      termsHashPerField.writeVInt(1, proxCode<<1, termID);
 
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
     postings.lastPositions[termID] = fieldState.position;
@@ -121,12 +121,17 @@
 
   @Override
   void newTerm(final int termID) {
+    System.out.println("newTerm termID:"+termID);
     // First time we're seeing this term since the last
     // flush
     assert docState.testPoint("FreqProxTermsWriterPerField.newTerm start");
 
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
     postings.lastDocIDs[termID] = docState.docID;
+    
+    // increment the term count
+    postings.termFreqs[termID]++;
+    
     if (indexOptions == IndexOptions.DOCS_ONLY) {
       postings.lastDocCodes[termID] = docState.docID;
     } else {
@@ -138,8 +143,16 @@
     }
     fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
     fieldState.uniqueTermCount++;
+    
+    if (termsHashPerField.capturingTerms) {
+      captureTerm(termID, docState.docID);
+    }
   }
-
+  
+  void captureTerm(int termID, int docID) {
+    termsHashPerField.captureTerm(termID, docID);
+  }
+  
   @Override
   void addTerm(final int termID) {
 
@@ -148,11 +161,14 @@
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
 
     assert indexOptions == IndexOptions.DOCS_ONLY || postings.docFreqs[termID] > 0;
-
+    
+    // increment the term count
+    postings.termFreqs[termID]++;
+    
     if (indexOptions == IndexOptions.DOCS_ONLY) {
       if (docState.docID != postings.lastDocIDs[termID]) {
         assert docState.docID > postings.lastDocIDs[termID];
-        termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
+        termsHashPerField.writeVInt(0, postings.lastDocCodes[termID], termID);
         postings.lastDocCodes[termID] = docState.docID - postings.lastDocIDs[termID];
         postings.lastDocIDs[termID] = docState.docID;
         fieldState.uniqueTermCount++;
@@ -166,10 +182,10 @@
         // Now that we know doc freq for previous doc,
         // write it & lastDocCode
         if (1 == postings.docFreqs[termID])
-          termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]|1);
+          termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]|1, termID);
         else {
-          termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
-          termsHashPerField.writeVInt(0, postings.docFreqs[termID]);
+          termsHashPerField.writeVInt(0, postings.lastDocCodes[termID], termID);
+          termsHashPerField.writeVInt(0, postings.docFreqs[termID], termID);
         }
         postings.docFreqs[termID] = 1;
         fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
@@ -186,6 +202,9 @@
         }
       }
     }
+    if (termsHashPerField.capturingTerms) {
+      captureTerm(termID, docState.docID);
+    }
   }
 
   @Override
@@ -200,12 +219,45 @@
       lastDocIDs = new int[size];
       lastDocCodes = new int[size];
       lastPositions = new int[size];
+      
+      freqUptos = new int[size];
+      proxUptos = new int[size];
+
+      // realtime
+      termFreqs = new int[size];
+      
+      // rt optional vars
+      //sumTotalTermFreqs = new int[size];
+      //sumDocFreqs = new int[size];
+            
+      freqUptosRT = new int[size];
+      proxUptosRT = new int[size];
+      lastDocIDsRT = new int[size];
+      lastDocFreqsRT = new int[size];
     }
 
     int docFreqs[];                                    // # times this term occurs in the current doc
     int lastDocIDs[];                                  // Last docID where this term occurred
     int lastDocCodes[];                                // Code for prior doc
     int lastPositions[];                               // Last position where this term occurred
+    
+    int freqUptos[];
+    int proxUptos[];
+    
+    // for realtime, changes as indexing occurs
+    // number of times the term occurs in the
+    // entire index
+    int termFreqs[];
+    //int sumTotalTermFreqs[];
+    //int sumDocFreqs[];
+    
+    // for realtime
+    // these are read-only / unchanging
+    // while a copy of them is being made
+    int freqUptosRT[];
+    int proxUptosRT[];
+    int lastDocIDsRT[];
+    int lastDocFreqsRT[];
 
     @Override
     ParallelPostingsArray newInstance(int size) {
@@ -223,6 +275,20 @@
       System.arraycopy(lastDocIDs, 0, to.lastDocIDs, 0, numToCopy);
       System.arraycopy(lastDocCodes, 0, to.lastDocCodes, 0, numToCopy);
       System.arraycopy(lastPositions, 0, to.lastPositions, 0, numToCopy);
+
+      System.arraycopy(freqUptos, 0, to.freqUptos, 0, numToCopy);
+      System.arraycopy(proxUptos, 0, to.proxUptos, 0, numToCopy);
+
+      // realtime
+      System.arraycopy(termFreqs, 0, to.termFreqs, 0, numToCopy);
+      // rt optional
+      //System.arraycopy(sumTotalTermFreqs, 0, to.sumTotalTermFreqs, 0, numToCopy);
+      //System.arraycopy(sumDocFreqs, 0, to.sumDocFreqs, 0, numToCopy);
+      
+      System.arraycopy(freqUptosRT, 0, to.freqUptosRT, 0, numToCopy);
+      System.arraycopy(proxUptosRT, 0, to.proxUptosRT, 0, numToCopy);
+      System.arraycopy(lastDocIDsRT, 0, to.lastDocIDsRT, 0, numToCopy);
+      System.arraycopy(lastDocFreqsRT, 0, to.lastDocFreqsRT, 0, numToCopy);
     }
 
     @Override
Index: lucene/src/java/org/apache/lucene/index/FieldInvertState.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/FieldInvertState.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/FieldInvertState.java	(working copy)
@@ -34,6 +34,9 @@
   int uniqueTermCount;
   float boost;
   AttributeSource attributeSource;
+  // RT variable for maintaining the norms calculation as indexing is happening
+  // nocommit: is there a better place for the norms array?
+  byte[] norms;
 
   public FieldInvertState() {
   }
Index: lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java	(working copy)
@@ -49,7 +49,13 @@
     this.docWriter = docWriter;
     docState = docWriter.docState;
   }
-
+  
+  void flush() throws IOException {
+    tvx.flush();
+    tvd.flush();
+    tvf.flush();
+  }
+  
   @Override
   void flush(Map<FieldInfo, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
     if (tvx != null) {
Index: lucene/src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/IndexReader.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/IndexReader.java	(working copy)
@@ -1208,7 +1208,7 @@
    * deleted it will not appear in TermDocs or TermPositions enumerations.
    * Attempts to read its field with the {@link #document}
    * method will result in an error.  The presence of this document may still be
-   * reflected in the {@link #docFreq} statistic, though
+   * reflected in the {@link #freq} statistic, though
    * this will be corrected eventually as the index is further modified.
    *
    * @throws StaleReaderException if the index has changed
Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java	(working copy)
@@ -23,6 +23,7 @@
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Queue;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -32,11 +33,28 @@
 import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
 import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.search.FieldCacheImpl;
+import org.apache.lucene.search.FieldCache.ByteParser;
+import org.apache.lucene.search.FieldCache.IntParser;
+import org.apache.lucene.search.FieldCacheImpl.CacheCreationListener;
+import org.apache.lucene.search.FieldCacheImpl.Entry;
+
+import org.apache.lucene.search.cache.ByteValuesCreator;
+import org.apache.lucene.search.cache.CachedArray;
+import org.apache.lucene.search.cache.CachedArray.ByteValues;
+import org.apache.lucene.search.cache.IntValuesCreator;
+import org.apache.lucene.search.cache.CachedArray.IntValues;
+
+import org.apache.lucene.search.cache.CachedArrayCreator;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
 
+import org.apache.lucene.index.RAMReaderManager.RAMReader;
+
 /**
  * This class accepts multiple added documents and directly
  * writes segment files.
@@ -138,13 +156,134 @@
     assert flushPolicy != null;
     flushPolicy.init(this);
     flushControl = new DocumentsWriterFlushControl(this, config);
+    
+    // nocommit: there may be a better place to place this code
+    if (FieldCacheImpl.DEFAULT instanceof FieldCacheImpl) {
+      FieldCacheImpl fieldCacheImpl = (FieldCacheImpl)FieldCacheImpl.DEFAULT;
+      fieldCacheImpl.addCacheCreationListener(new CacheCreationHandler());
+    }
   }
 
+  private class CacheCreationHandler extends CacheCreationListener {
+    @Override
+    public void cacheCreated(IndexReader reader, Entry entry, Object value) {
+      if (RAMReaderManager.DEBUG) {
+        System.out.println("cacheCreated reader:"+reader+" entry:"+entry+" value:"+value);
+      }
+      // when a new value has been created, inform all active DWPTs
+      if (reader instanceof RAMReader) {
+        RAMReader ramReader = (RAMReader)reader;
+        DocumentsWriterPerThread dwpt = (DocumentsWriterPerThread)ramReader.getCoreCacheKey();
+        String field = entry.field;
+        if (entry.creator instanceof ByteValuesCreator) {
+          ByteValuesCreator c = (ByteValuesCreator)entry.creator;
+          ByteParser parser = (ByteParser)c.getParser();
+          ByteValues byteValues = (ByteValues)value;
+          ByteFieldHandler h = new ByteFieldHandler(parser, byteValues);
+          dwpt.addFieldListener(field, h);
+          if (RAMReaderManager.DEBUG) 
+            System.out.println("added bytes field listener: "+field);
+        } else if (entry.creator instanceof IntValuesCreator) {
+          IntValuesCreator c = (IntValuesCreator)entry.creator;
+          IntParser parser = (IntParser)c.getParser();
+          IntValues values = (IntValues)value;
+          IntFieldHandler h = new IntFieldHandler(parser, values);
+          dwpt.addFieldListener(field, h);
+          if (RAMReaderManager.DEBUG) 
+            System.out.println("added ints field listener: "+field);
+        }        
+      }
+    }
+  }
+  
+  private static void defaultIncrementDocAndTerms(CachedArray cachedArray) {
+    cachedArray.numDocs++;
+    cachedArray.numTerms++;
+  }
+  
+  // class that handles newly indexed byte array values 
+  private class ByteFieldHandler extends FieldListener {
+    final ByteParser parser;
+    final ByteValues values;
+    
+    public ByteFieldHandler(ByteParser parser, ByteValues values) {
+      this.parser = parser;
+      this.values = values;
+    }
+    
+    @Override
+    public void newValue(int termID, BytesRef term, int docID,
+        TermsHashPerField termsHashPerField) {
+      byte b = parser.parseByte(term);
+      if (docID >= values.values.length) {
+        values.values = ArrayUtil.grow(values.values);
+      }
+      values.values[docID] = b;
+      defaultIncrementDocAndTerms(values);
+    }    
+  }
+  
+  private class IntFieldHandler extends FieldListener {
+    final IntParser parser;
+    final IntValues values;
+    
+    public IntFieldHandler(IntParser parser, IntValues values) {
+      this.parser = parser;
+      this.values = values;
+    }
+    
+    @Override
+    public void newValue(int termID, BytesRef term, int docID,
+        TermsHashPerField termsHashPerField) {
+      int i = parser.parseInt(term);
+      if (docID >= values.values.length) {
+        values.values = ArrayUtil.grow(values.values);
+      }
+      values.values[docID] = i;
+      defaultIncrementDocAndTerms(values);
+    }    
+  }
+  
+  List<DocumentsWriterPerThread> getDocumentsWriterPerThreadsActive() {
+    List<DocumentsWriterPerThread> dwpts = new ArrayList<DocumentsWriterPerThread>();
+    Iterator<ThreadState> iterator = perThreadPool.getActivePerThreadsIterator();
+    while (iterator.hasNext()) {
+      ThreadState state = iterator.next();
+      DocumentsWriterPerThread dwpt = state.getDocumentsWriterPerThread();
+      dwpts.add(dwpt);
+    }
+    return dwpts;
+  }
+  
+  public RAMReader[] getRAMReaders() throws IOException {
+    List<DocumentsWriterPerThread> dwpts = getAllDocumentsWriterPerThreads();
+    RAMReader[] readers = new RAMReader[dwpts.size()];
+    for (int x=0; x < dwpts.size(); x++) {
+      readers[x] = dwpts.get(x).getReader();
+    }
+    return readers;
+  }
+  
+  public List<DocumentsWriterPerThread> getAllDocumentsWriterPerThreads() {
+    Iterator<ThreadState> it = perThreadPool.getAllPerThreadsIterator();
+    List<DocumentsWriterPerThread> list = new ArrayList<DocumentsWriterPerThread>();
+    while (it.hasNext()) {
+      ThreadState state = it.next();
+      DocumentsWriterPerThread dwpt = state.getDocumentsWriterPerThread();
+      
+      assert dwpt != null;
+      if (dwpt != null) {
+        list.add(dwpt);
+      }
+    }
+    return list;
+  }
+  
   synchronized void deleteQueries(final Query... queries) throws IOException {
     deleteQueue.addDelete(queries);
     flushControl.doOnDelete();
     if (flushControl.doApplyAllDeletes()) {
-      applyAllDeletes(deleteQueue);
+      applyAllDeletes(deleteQueue, true);
     }
   }
 
@@ -156,15 +295,25 @@
     deleteQueue.addDelete(terms);
     flushControl.doOnDelete();
     if (flushControl.doApplyAllDeletes()) {
-      applyAllDeletes(deleteQueue);
+      applyAllDeletes(deleteQueue, true);
     }
   }
-
+  
+  // apply all deletes for RT
+  synchronized void applyAllDeletesRT() throws IOException {
+    final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
+    flushControl.doOnDelete();
+    applyAllDeletes(deleteQueue, true);
+  }
+  
   DocumentsWriterDeleteQueue currentDeleteSession() {
     return deleteQueue;
   }
   
-  private void applyAllDeletes(DocumentsWriterDeleteQueue deleteQueue) throws IOException {
+  // nocommit: there is probably an sync issue where
+  // we need to get the index lock on the DWPT
+  // when applying these deletes because of memory usage
+  private void applyAllDeletes(DocumentsWriterDeleteQueue deleteQueue, boolean includeDWPTs) throws IOException {
     if (deleteQueue != null && !flushControl.isFullFlush()) {
       synchronized (ticketQueue) {
         // Freeze and insert the delete flush ticket in the queue
@@ -172,7 +321,7 @@
         applyFlushTickets();
       }
     }
-    indexWriter.applyAllDeletes();
+    indexWriter.applyAllDeletes(includeDWPTs);
     indexWriter.flushCount.incrementAndGet();
   }
 
@@ -306,7 +455,7 @@
 
   private boolean postUpdate(DocumentsWriterPerThread flushingDWPT, boolean maybeMerge) throws IOException {
     if (flushControl.doApplyAllDeletes()) {
-      applyAllDeletes(deleteQueue);
+      applyAllDeletes(deleteQueue, false);
     }
     if (flushingDWPT != null) {
       maybeMerge |= doFlush(flushingDWPT);
@@ -353,7 +502,7 @@
 
   boolean updateDocument(final Document doc, final Analyzer analyzer,
       final Term delTerm) throws CorruptIndexException, IOException {
-
+    
     boolean maybeMerge = preUpdate();
 
     final ThreadState perThread = flushControl.obtainAndLock();
@@ -459,7 +608,7 @@
       if (infoStream != null) {
         message("force apply deletes bytesUsed=" + flushControl.getDeleteBytesUsed() + " vs ramBuffer=" + (1024*1024*ramBufferSizeMB));
       }
-      applyAllDeletes(deleteQueue);
+      applyAllDeletes(deleteQueue, true);
     }
 
     return maybeMerge;
Index: lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java	(working copy)
@@ -207,13 +207,13 @@
       int startOffset = fieldState.offset + offsetAttribute.startOffset();
       int endOffset = fieldState.offset + offsetAttribute.endOffset();
 
-      termsHashPerField.writeVInt(1, startOffset);
-      termsHashPerField.writeVInt(1, endOffset - startOffset);
+      termsHashPerField.writeVInt(1, startOffset, termID);
+      termsHashPerField.writeVInt(1, endOffset - startOffset, termID);
       postings.lastOffsets[termID] = endOffset;
     }
 
     if (doVectorPositions) {
-      termsHashPerField.writeVInt(0, fieldState.position);
+      termsHashPerField.writeVInt(0, fieldState.position, termID);
       postings.lastPositions[termID] = fieldState.position;
     }
   }
@@ -231,13 +231,13 @@
       int startOffset = fieldState.offset + offsetAttribute.startOffset();
       int endOffset = fieldState.offset + offsetAttribute.endOffset();
 
-      termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
-      termsHashPerField.writeVInt(1, endOffset - startOffset);
+      termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID], termID);
+      termsHashPerField.writeVInt(1, endOffset - startOffset, termID);
       postings.lastOffsets[termID] = endOffset;
     }
 
     if (doVectorPositions) {
-      termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID]);
+      termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID], termID);
       postings.lastPositions[termID] = fieldState.position;
     }
   }
Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/IndexWriter.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -41,6 +41,7 @@
 import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor;
+import org.apache.lucene.index.RAMReaderManager.RAMReader;
 import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.search.Query;
@@ -277,6 +278,84 @@
   // for testing
   boolean anyNonBulkMerges;
 
+  public RAMReader[] getRAMReaders() throws IOException {
+    // first apply all deletes to all segments including
+    // the in-RAM DWPTs
+    
+    docWriter.applyAllDeletesRT();
+    
+    List<DocumentsWriterPerThread> dwpts = docWriter.getDocumentsWriterPerThreadsActive();
+    RAMReader[] ramReaders = new RAMReader[dwpts.size()];
+    int c = 0;
+    for (DocumentsWriterPerThread dwpt : dwpts) {
+      ramReaders[c++] = dwpt.getReader();
+    }
+    return ramReaders;
+  }
+
+  /**
+   * Combine the RAM readers with the Directory readers.
+   * Apply all deletes to all RAM readers and Directory
+   * readers.
+   */
+  public IndexReader[] getRTReaders() throws IOException {
+    List<IndexReader> readers = new ArrayList<IndexReader>();
+    RAMReader[] ramReaders = getRAMReaders();
+    IndexReader[] dirReaders = getDirReaders(true);
+    for (RAMReader ramReader : ramReaders) {
+      readers.add(ramReader);
+    }
+    for (IndexReader reader : dirReaders) {
+      readers.add(reader);
+    }
+    return (IndexReader[])readers.toArray(new IndexReader[0]);
+  }
+  
+  /**
+   * Return only the readers that already exist in the 
+   * Directory.  We do not flush here, because with RT
+   * we can read the get the index directly from
+   * the DWPT RAM buffer.
+   */
+  IndexReader[] getDirReaders(boolean applyAllDeletes) throws IOException {
+    ensureOpen();
+
+    final long tStart = System.currentTimeMillis();
+
+    poolReaders = true;
+    final IndexReader r;
+    boolean success = false;
+    try {
+      success = true;
+      // Prevent segmentInfos from changing while opening the
+      // reader; in theory we could do similar retry logic,
+      // just like we do when loading segments_N
+      synchronized(this) {
+        maybeApplyDeletes(applyAllDeletes);
+        r = new DirectoryReader(this, segmentInfos, codecs, applyAllDeletes);
+        if (infoStream != null) {
+          message("return reader version=" + r.getVersion() + " reader=" + r);
+        }
+      }
+    } catch (OutOfMemoryError oom) {
+      handleOOM(oom, "getReader");
+      // never reached but javac disagrees:
+      return null;
+    } finally {
+      if (!success && infoStream != null) {
+        message("hit exception during NRT reader");
+      }
+      // Done: finish the full flush!
+      docWriter.finishFullFlush(success);
+      doAfterFlush();
+    }
+    if (infoStream != null) {
+      message("getReader took " + (System.currentTimeMillis() - tStart) + " msec");
+    }
+    // return an array of readers
+    return r.getSequentialSubReaders();
+  }
+  
   IndexReader getReader() throws IOException {
     return getReader(true);
   }
@@ -2990,16 +3069,21 @@
       if (infoStream != null) {
         message("apply all deletes during flush");
       }
-      applyAllDeletes();
+      applyAllDeletes(true);
     } else if (infoStream != null) {
       message("don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
     }
   }
   
-  final synchronized void applyAllDeletes() throws IOException {
+  final synchronized void applyAllDeletes(boolean includeDWPTs) throws IOException {
     flushDeletesCount.incrementAndGet();
     final BufferedDeletesStream.ApplyDeletesResult result;
-    result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos.asList());
+    
+    List<DocumentsWriterPerThread> dwpts = null;
+    if (includeDWPTs) {
+      dwpts = docWriter.getAllDocumentsWriterPerThreads();
+    }
+    result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos.asList(), dwpts);
     if (result.anyDeletes) {
       checkpoint();
     }
@@ -3435,7 +3519,7 @@
     merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)));
 
     // Lock order: IW -> BD
-    final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments);
+    final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments, null);
 
     if (result.anyDeletes) {
       checkpoint();
Index: lucene/src/java/org/apache/lucene/index/IntBlocks.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/IntBlocks.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/index/IntBlocks.java	(revision 0)
@@ -0,0 +1,30 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+public class IntBlocks {
+  public int[][] buffers;
+  
+  public IntBlocks(int[][] buffers) {
+    this.buffers = buffers;
+  }
+  
+  public IntBlocks copyRef() {
+    return new IntBlocks(buffers);
+  }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/util/BitVector.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/BitVector.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/util/BitVector.java	(working copy)
@@ -73,6 +73,21 @@
     return clone;
   }
   
+  /** Clone the byte[], allowing for a new size to be set */
+  public BitVector clone(int size) {
+    int minSize = getNumBytes(size);
+    // set as the default
+    int newSizeBytes = bits.length;
+    if (minSize >= bits.length) {
+      newSizeBytes = ArrayUtil.oversize(minSize, 1);
+    }
+    byte[] newBits = new byte[newSizeBytes];
+    System.arraycopy(bits, 0, newBits, 0, bits.length);
+    BitVector clone = new BitVector(newBits, size);
+    clone.count = count;
+    return clone;
+  }
+  
   /** Sets the value of <code>bit</code> to one. */
   public final void set(int bit) {
     if (bit >= size) {
Index: lucene/src/java/org/apache/lucene/util/ByteBlockPool.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/ByteBlockPool.java	(revision 1159007)
+++ lucene/src/java/org/apache/lucene/util/ByteBlockPool.java	(working copy)
@@ -44,11 +44,12 @@
  * 
  * @lucene.internal
  **/
-public final class ByteBlockPool {
+public final class ByteBlockPool extends ByteBlocks {
   public final static int BYTE_BLOCK_SHIFT = 15;
   public final static int BYTE_BLOCK_SIZE = 1 << BYTE_BLOCK_SHIFT;
   public final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
-
+  public int byteSliceUpto = BYTE_BLOCK_SIZE;
+  
   public abstract static class Allocator {
     protected final int blockSize;
 
@@ -110,9 +111,6 @@
     
   };
 
-
-  public byte[][] buffers = new byte[10][];
-
   int bufferUpto = -1;                        // Which buffer we are upto
   public int byteUpto = BYTE_BLOCK_SIZE;             // Where we are in head buffer
 
@@ -122,6 +120,7 @@
   private final Allocator allocator;
 
   public ByteBlockPool(Allocator allocator) {
+    super(new byte[10][]);
     this.allocator = allocator;
   }
   
@@ -158,6 +157,7 @@
       bufferUpto = 0;
       byteUpto = 0;
       byteOffset = 0;
+      byteSliceUpto = 0;
       buffer = buffers[0];
     }
   }
@@ -173,15 +173,18 @@
     bufferUpto++;
 
     byteUpto = 0;
+    byteSliceUpto = 0;
     byteOffset += BYTE_BLOCK_SIZE;
   }
 
-  public int newSlice(final int size) {
+  public int newSliceByLevel(final int level) {
+    int size = levelSizeArray[level];
     if (byteUpto > BYTE_BLOCK_SIZE-size)
       nextBuffer();
     final int upto = byteUpto;
     byteUpto += size;
-    buffer[byteUpto-1] = 16;
+    byteSliceUpto = byteUpto - 3;
+    buffer[byteSliceUpto-1] = 16;
     return upto;
   }
 
@@ -192,9 +195,9 @@
   // bytes, next slice is 14 bytes, etc.
   
   public final static int[] nextLevelArray = {1, 2, 3, 4, 5, 6, 7, 8, 9, 9};
-  public final static int[] levelSizeArray = {5, 14, 20, 30, 40, 40, 80, 80, 120, 200};
+  public final static int[] levelSizeArray = {9, 14, 20, 30, 40, 40, 80, 80, 120, 200};
   public final static int FIRST_LEVEL_SIZE = levelSizeArray[0];
-
+  
   public int allocSlice(final byte[] slice, final int upto) {
 
     final int level = slice[upto] & 15;
@@ -206,45 +209,22 @@
       nextBuffer();
 
     final int newUpto = byteUpto;
-    final int offset = newUpto + byteOffset;
+    final int offset = byteUpto + byteOffset;
     byteUpto += newSize;
 
-    // Copy forward the past 3 bytes (which we are about
-    // to overwrite with the forwarding address):
-    buffer[newUpto] = slice[upto-3];
-    buffer[newUpto+1] = slice[upto-2];
-    buffer[newUpto+2] = slice[upto-1];
-
-    // Write forwarding address at end of last slice:
-    slice[upto-3] = (byte) (offset >>> 24);
-    slice[upto-2] = (byte) (offset >>> 16);
-    slice[upto-1] = (byte) (offset >>> 8);
-    slice[upto] = (byte) offset;
+    slice[upto] = (byte) (offset >>> 24);
+    slice[upto+1] = (byte) (offset >>> 16);
+    slice[upto+2] = (byte) (offset >>> 8);
+    slice[upto+3] = (byte) offset;    
+    
+    byteSliceUpto = byteUpto - 3;
         
     // Write new level:
-    buffer[byteUpto-1] = (byte) (16|newLevel);
-
-    return newUpto+3;
+    buffer[byteSliceUpto-1] = (byte) (16|newLevel);
+    
+    return newUpto;
   }
 
-  // Fill in a BytesRef from term's length & bytes encoded in
-  // byte block
-  public final BytesRef setBytesRef(BytesRef term, int textStart) {
-    final byte[] bytes = term.bytes = buffers[textStart >> BYTE_BLOCK_SHIFT];
-    int pos = textStart & BYTE_BLOCK_MASK;
-    if ((bytes[pos] & 0x80) == 0) {
-      // length is 1 byte
-      term.length = bytes[pos];
-      term.offset = pos+1;
-    } else {
-      // length is 2 bytes
-      term.length = (bytes[pos]&0x7f) + ((bytes[pos+1]&0xff)<<7);
-      term.offset = pos+2;
-    }
-    assert term.length >= 0;
-    return term;
-  }
-  
   /**
    * Copies the given {@link BytesRef} at the current positions (
    * {@link #byteUpto} across buffer boundaries
Index: lucene/src/java/org/apache/lucene/util/ByteBlocks.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/ByteBlocks.java	(revision 0)
+++ lucene/src/java/org/apache/lucene/util/ByteBlocks.java	(revision 0)
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+
+public class ByteBlocks {
+  public byte[][] buffers;
+  
+  public ByteBlocks(byte[][] buffers) {
+    this.buffers = buffers;
+  }
+  
+  public static ByteBlocks copyRef(ByteBlockPool bbp) {
+    byte[][] buffers = bbp.buffers;
+    return new ByteBlocks(buffers);
+  }
+  
+  public final BytesRef setBytesRef(BytesRef term, int textStart) {
+    final byte[] bytes = term.bytes = buffers[textStart >> ByteBlockPool.BYTE_BLOCK_SHIFT];
+    int pos = textStart & ByteBlockPool.BYTE_BLOCK_MASK;
+    if ((bytes[pos] & 0x80) == 0) {
+      // length is 1 byte
+      term.length = bytes[pos];
+      term.offset = pos + 1;
+    } else {
+      // length is 2 bytes
+      term.length = (bytes[pos] & 0x7f) + ((bytes[pos + 1] & 0xff) << 7);
+      term.offset = pos + 2;
+    }
+    assert term.length >= 0;
+    return term;
+  }
+}
\ No newline at end of file
