bufferedInputStream med meget store filer

Brug almindelig InputStream og kald read med et meget stort byte array
(f.eks. 50 milliioner).

Så får du god performance og ikke nogen in between buffer.

Styrken ved BufferedInputStream er hvis du læser mange små bidder.

Altså læser 2 bytes, læser 4 bytes etc..

Alternativt hvis du er på J2SE 1.4 kan du overveje java.nio.*
og FileChannel/MappedByteBuffer.

Jeg har aldrig prøvet dem, men de skulle være specielt lavet til
high-performance binær IO.

Nio er ikke vildt meget hurtigere, men benchmarks varierer fra maskine til maskine:

import java.nio.*;
import java.nio.channels.*;
import java.io.*;

public class NioTest {
    public static void main(String[] args) {
        try {
            File file = new File("C:/tekst.txt");

            long start = System.currentTimeMillis();
            FileChannel rwChannel = new RandomAccessFile(file, "rw").getChannel();
          ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, (int)rwChannel.size());
         
          StringBuffer buffer = new StringBuffer();
          for( int i = 0; i < wrBuf.capacity(); i++ ) {
              buffer.append((char)wrBuf.get());
          }
          String str = buffer.toString();
          System.out.println ("TIME nio: " +(System.currentTimeMillis()-start));

          /*
          long start = System.currentTimeMillis();
          RandomAccessFile rw = new RandomAccessFile(file, "rw");
          byte[] result = new byte[(int)rw.length()];
          rw.read(result);
          String str = new String(result);
          System.out.println ("TIME old " +(System.currentTimeMillis()-start));
          */
        }
        catch( FileNotFoundException fnfe ) {
            fnfe.printStackTrace();
        }
        catch( IOException ioe ) {
            ioe.printStackTrace();
        }
  }
}

Jeg prøvede lige at lave et lidt mere omfattende test program:

package test;

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;

public class ReadTest {
    final static String FILE_NAME = "C://readtest.dat";
    public static void main(String[] args) throws Exception {
        createFile();
        (new UnbufferedSmallReadTest()).doTest();
        (new UnbufferedSmallReadTest()).doTest();
        (new UnbufferedSmallReadTest()).doTest();
        (new UnbufferedLargeReadTest()).doTest();
        (new UnbufferedLargeReadTest()).doTest();
        (new UnbufferedLargeReadTest()).doTest();
        (new BufferedSmallReadTest()).doTest();
        (new BufferedSmallReadTest()).doTest();
        (new BufferedSmallReadTest()).doTest();
        (new BufferedLargeReadTest()).doTest();
        (new BufferedLargeReadTest()).doTest();
        (new BufferedLargeReadTest()).doTest();
        (new NioSmallReadTest()).doTest();
        (new NioSmallReadTest()).doTest();
        (new NioSmallReadTest()).doTest();
        (new NioLargeReadTest()).doTest();
        (new NioLargeReadTest()).doTest();
        (new NioLargeReadTest()).doTest();
    }
    private static void createFile() {
        try {
            OutputStream f = new FileOutputStream(FILE_NAME);
            for(int i = 0; i < 10000000; i++) {
                f.write((i % 256));
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

abstract class GenericReadTest {
    GenericReadTest() {
    }
    void doTest() throws Exception {
        long t1 = System.currentTimeMillis();
        readAndCheck();
        long t2 = System.currentTimeMillis();
        System.out.println(getType() + " : " + (t2 - t1));
    }
    abstract void readAndCheck() throws Exception;
    abstract String getType();
}

abstract class UnbufferedReadTest extends GenericReadTest {
    void readAndCheck() throws Exception {
        try {
            InputStream f = new FileInputStream(ReadTest.FILE_NAME);
            byte[] b = new byte[getReadSize()];
            int n;
            int ix = 0;
            while((n = f.read(b)) >= 0) {
                for(int i = 0; i < n; i++) {
                    if(b[i] != (byte)(ix % 256)) {
                        throw new Exception("Bad data read - offset " + ix + " expected " + (byte)(ix % 256) + " found " + b[i]);
                    }
                    ix++;
                }
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    abstract int getReadSize();
}

abstract class BufferedReadTest extends GenericReadTest {
    void readAndCheck() throws Exception {
        try {
            BufferedInputStream f = new BufferedInputStream(new FileInputStream(ReadTest.FILE_NAME));
            byte[] b = new byte[getReadSize()];
            int n;
            int ix = 0;
            while((n = f.read(b)) >= 0) {
                for(int i = 0; i < n; i++) {
                    if(b[i] != (byte)(ix % 256)) {
                        throw new Exception("Bad data read - offset " + ix + " expected " + (byte)(ix % 256) + " found " + b[i]);
                    }
                    ix++;
                }
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    abstract int getReadSize();
}

abstract class NioReadTest extends GenericReadTest {
    void readAndCheck() throws Exception {
        try {
            FileChannel f2 = (new FileInputStream(ReadTest.FILE_NAME)).getChannel();
            ByteBuffer f = f2.map(FileChannel.MapMode.READ_ONLY, 0, 10000000);
            byte[] b = new byte[getReadSize()];
            int ix = 0;
            while(ix < 10000000) {
                f.get(b);
                for(int i = 0; i < b.length; i++) {
                    if(b[i] != (byte)(ix % 256)) {
                        throw new Exception("Bad data read - offset " + ix + " expected " + (byte)(ix % 256) + " found " + b[i]);
                    }
                    ix++;
                }
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    abstract int getReadSize();
}

class UnbufferedSmallReadTest extends UnbufferedReadTest {
    int getReadSize() {
        return 10;
    }
    String getType() {
        return "Unbuffered small reads";
    }
}

class UnbufferedLargeReadTest extends UnbufferedReadTest {
    int getReadSize() {
        return 1000000;
    }
    String getType() {
        return "Unbuffered large reads";
    }
}

class BufferedSmallReadTest extends BufferedReadTest {
    int getReadSize() {
        return 10;
    }
    String getType() {
        return "Buffered small reads";
    }
}

class BufferedLargeReadTest extends BufferedReadTest {
    int getReadSize() {
        return 1000000;
    }
    String getType() {
        return "Buffered large reads";
    }
}

class NioSmallReadTest extends NioReadTest {
    int getReadSize() {
        return 10;
    }
    String getType() {
        return "Nio small reads";
    }
}

class NioLargeReadTest extends NioReadTest {
    int getReadSize() {
        return 1000000;
    }
    String getType() {
        return "Nio large reads";
    }
}

Resultat på min PC:

Unbuffered small reads : 1641
Unbuffered small reads : 1672
Unbuffered small reads : 1656
Unbuffered large reads : 188
Unbuffered large reads : 156
Unbuffered large reads : 203
Buffered small reads : 359
Buffered small reads : 312
Buffered small reads : 344
Buffered large reads : 188
Buffered large reads : 203
Buffered large reads : 156
Nio small reads : 594
Nio small reads : 437
Nio small reads : 407
Nio large reads : 171
Nio large reads : 125
Nio large reads : 141

Mine konklusioner:

1)  buffered large reads er ikke hurtigere end unbuffered large reads
2)  buffered small reads er meget hurtigere end unbuffered small reads
3)  large reads er altid hurtigere end small reads
4)  nio large reads er lidt hurtigere end buffered/unbuffered large reads
5)  nio small reads er lidt langsommere end buffered small reads

Jeg prøvede lige med 100 MB fil i.s.f. 10 MB fil og resultaterne er
konsistente:

Unbuffered small reads : 16500
Unbuffered small reads : 16187
Unbuffered small reads : 16094
Unbuffered large reads : 1578
Unbuffered large reads : 1563
Unbuffered large reads : 1578
Buffered small reads : 3375
Buffered small reads : 3266
Buffered small reads : 3218
Buffered large reads : 1547
Buffered large reads : 1610
Buffered large reads : 1546
Nio small reads : 4110
Nio small reads : 4015
Nio small reads : 4063
Nio large reads : 1250
Nio large reads : 1156
Nio large reads : 1172

mange tak for hjælpen arne v. Det var meget lærerigt, den test af streams vil jeg medtage i min rapport. Virkelig fedt.