[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Gzz-commits] gzz gzz/view/DefaultBinder.java lava/gzz/storm/...
|
From: |
Benja Fallenstein |
|
Subject: |
[Gzz-commits] gzz gzz/view/DefaultBinder.java lava/gzz/storm/... |
|
Date: |
Fri, 15 Nov 2002 23:38:05 -0500 |
CVSROOT: /cvsroot/gzz
Module name: gzz
Changes by: Benja Fallenstein <address@hidden> 02/11/15 23:38:04
Modified files:
gzz/view : DefaultBinder.java
lava/gzz/storm : BlockId.java BlockOutputStream.java
lava/gzz/storm/impl: AbstractPool.java DirPool.java
TransientPool.java
lava/test/gzz/storm: StormPoolTest.java
lava/test/gzz/storm/impl: DirPool.test
test/gzz/impl : BenchFS.java
test/gzz/mediaserver: TestMS.java
test/gzz/mediaserver/storage: TestZipStorer.java
Removed files:
gzz/util : TestingUtil.java
Log message:
Refactor Storm to be more efficient; rename TestingUtil to TempFileUtil
CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/gzz/view/DefaultBinder.java.diff?tr1=1.12&tr2=1.13&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/BlockId.java.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/BlockOutputStream.java.diff?tr1=1.7&tr2=1.8&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/AbstractPool.java.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/DirPool.java.diff?tr1=1.8&tr2=1.9&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/TransientPool.java.diff?tr1=1.15&tr2=1.16&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/test/gzz/storm/StormPoolTest.java.diff?tr1=1.10&tr2=1.11&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/test/gzz/storm/impl/DirPool.test.diff?tr1=1.4&tr2=1.5&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/test/gzz/impl/BenchFS.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/test/gzz/mediaserver/TestMS.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/test/gzz/mediaserver/storage/TestZipStorer.java.diff?tr1=1.1&tr2=1.2&r1=text&r2=text
Patches:
Index: gzz/gzz/view/DefaultBinder.java
diff -u gzz/gzz/view/DefaultBinder.java:1.12
gzz/gzz/view/DefaultBinder.java:1.13
--- gzz/gzz/view/DefaultBinder.java:1.12 Fri Nov 8 14:24:54 2002
+++ gzz/gzz/view/DefaultBinder.java Fri Nov 15 23:38:03 2002
@@ -44,7 +44,7 @@
/** Keybindings for the Fallback client.
*/
public class DefaultBinder implements FallbackBinder {
-public static final String rcsid = "$Id: DefaultBinder.java,v 1.12 2002/11/08
19:24:54 benja Exp $";
+public static final String rcsid = "$Id: DefaultBinder.java,v 1.13 2002/11/16
04:38:03 benja Exp $";
public static boolean dbg = false;
private static void pa(String s) { System.err.println(s); }
@@ -606,7 +606,7 @@
try {
File dir = new File(System.getProperty("user.dir"));
- tmp = gzz.util.TestingUtil.tmpFile(dir);
+ tmp = gzz.util.TempFileUtil.tmpFile(dir);
byte[] out = text.getBytes(encoding);
FileOutputStream fout = new FileOutputStream(tmp);
Index: gzz/lava/gzz/storm/BlockId.java
diff -u gzz/lava/gzz/storm/BlockId.java:1.2 gzz/lava/gzz/storm/BlockId.java:1.3
--- gzz/lava/gzz/storm/BlockId.java:1.2 Thu Nov 7 16:42:50 2002
+++ gzz/lava/gzz/storm/BlockId.java Fri Nov 15 23:38:04 2002
@@ -107,17 +107,20 @@
"Storm cannot operate without "+
"an SHA-1 implementation.");
}
-
digest.reset();
digest.update(bytes);
- byte[] hash = digest.digest();
-
- byte[] id = new byte[21];
-
+
+ byte[] id = new byte[21];
id[0] = 0x01;
- System.arraycopy(hash, 0, id, 1, 20);
-
+
+ try {
+ if(digest.digest(id, 1, 20) != 20)
+ throw new Error("SHA digest not 20 bytes long?!?");
+ } catch(DigestException e) {
+ throw new Error("DigestException: "+e);
+ }
+
return new BlockId(id);
}
}
Index: gzz/lava/gzz/storm/BlockOutputStream.java
diff -u gzz/lava/gzz/storm/BlockOutputStream.java:1.7
gzz/lava/gzz/storm/BlockOutputStream.java:1.8
--- gzz/lava/gzz/storm/BlockOutputStream.java:1.7 Thu Nov 7 17:07:00 2002
+++ gzz/lava/gzz/storm/BlockOutputStream.java Fri Nov 15 23:38:04 2002
@@ -23,16 +23,73 @@
package gzz.storm;
import gzz.storm.headers.*;
import java.io.*;
+import java.security.*;
/** An <code>OutputStream</code> for creating a new Storm block.
* The block is not actually created until <code>close()</code>
* is called on this stream. At that time, the block is added
* to the pool, and its data can be retrieved through the
* <code>getBlock()</code> and <code>getBlockId</code> methods.
+ * <p>
+ * While data is written to the stream, it is also sent
+ * to a <code>MessageDigest</code> object that is used
+ * to detemine the block's id.
+ * <p>
+ * This class works in a way similar to what
+ * <code>java.security.DigestOutputStream</code> does. However,
+ * we cannot subclass <code>DigestOutputStream</code> here,
+ * because it provides public methods that could be used to
+ * change the id generated (it is possible to write data
+ * to the stream and not send it to the digest, for example).
*/
public abstract class BlockOutputStream extends FilterOutputStream {
+ boolean dbg = false;
+ private static void p(String s) { System.out.println(s); }
+
+ protected MessageDigest digest;
+
protected BlockOutputStream(OutputStream out) {
super(out);
+ try {
+ this.digest = MessageDigest.getInstance("SHA");
+ } catch(NoSuchAlgorithmException e) {
+ throw new Error("Fatal error: The SHA-1 algorithm "+
+ "is not supported by this version "+
+ "of the Java libraries. "+
+ "Storm cannot operate without "+
+ "an SHA-1 implementation.");
+ }
+ digest.reset();
+ }
+
+ public void write(byte[] b, int off, int len) throws IOException {
+ digest.update(b, off, len);
+ if(dbg) p("<"+new String(b)+">");
+ out.write(b, off, len);
+ }
+
+ public void write(int b) throws IOException {
+ digest.update((byte)b);
+ if(dbg) p("<"+new String(new byte[] {(byte)b})+">");
+ out.write(b);
+ }
+
+ /** Construct a <code>BlockId</code> from the current state
+ * of the internal message digest. This is usefully called
+ * from the <code>close()</code> method.
+ */
+ protected BlockId makeIdFromDigest() throws IOException {
+ byte[] bytes = new byte[21];
+ bytes[0] = 0x01;
+
+ try {
+ if(digest.digest(bytes, 1, 20) != 20)
+ throw new Error("SHA digest not 20 bytes long?!?");
+ } catch(DigestException e) {
+ throw new IOException("DigestException: "+e);
+ }
+
+ return new BlockId(bytes);
}
/** Close this stream and create the new block.
Index: gzz/lava/gzz/storm/impl/AbstractPool.java
diff -u gzz/lava/gzz/storm/impl/AbstractPool.java:1.2
gzz/lava/gzz/storm/impl/AbstractPool.java:1.3
--- gzz/lava/gzz/storm/impl/AbstractPool.java:1.2 Fri Nov 15 21:16:13 2002
+++ gzz/lava/gzz/storm/impl/AbstractPool.java Fri Nov 15 23:38:04 2002
@@ -20,9 +20,15 @@
protected Header822 header;
protected Block block;
- protected AbstractBlockOutputStream(OutputStream os, Header822 header) {
+ /** Create a new AbstractBlockOutputStream object
+ * and write the given header to it.
+ */
+ protected AbstractBlockOutputStream(OutputStream os, Header822 header)
+ throws IOException {
super(os);
this.header = header;
+
+ header.writeTo(this);
}
public Block getBlock() { return block; }
Index: gzz/lava/gzz/storm/impl/DirPool.java
diff -u gzz/lava/gzz/storm/impl/DirPool.java:1.8
gzz/lava/gzz/storm/impl/DirPool.java:1.9
--- gzz/lava/gzz/storm/impl/DirPool.java:1.8 Fri Nov 15 21:15:14 2002
+++ gzz/lava/gzz/storm/impl/DirPool.java Fri Nov 15 23:38:04 2002
@@ -29,11 +29,7 @@
/** A StormPool storing blocks in individual files in a directory.
* File names have the form <code>b_</code><i>idstring</i>.
* <p>
- * Ideas for making this more efficient if needed:
- * Don't use byte arrays. When creating a new block, write it
- * to a temporary file first and multiplex to another
- * output stream to get the id; rename the file when
- * the whole block is through and the id is known.
+ * XXX check id on reading blocks in (use id-checking input stream)
*/
public class DirPool extends AbstractPool {
@@ -53,7 +49,8 @@
* corresponding to the given id.
*/
protected Header822 getFileHeader(BlockId id) throws IOException {
- InputStream is = new FileInputStream(getFile(id));
+ InputStream is =
+ new BufferedInputStream(new FileInputStream(getFile(id)));
Header822 header = Headers822.readHeader(is);
is.close();
return header;
@@ -62,21 +59,25 @@
protected class FileBlockOutputStream extends AbstractBlockOutputStream {
protected ByteArrayOutputStream baos;
+ protected File tempFile;
- protected FileBlockOutputStream(Header822 header) throws IOException {
- super(new ByteArrayOutputStream(), header);
- baos = (ByteArrayOutputStream)out;
- header.writeTo(baos);
+ protected FileBlockOutputStream(Header822 header, File tempFile)
+ throws IOException {
+ super(new BufferedOutputStream(new FileOutputStream(tempFile)),
+ header);
+ this.tempFile = tempFile;
}
public void close() throws IOException {
- byte[] bytes = baos.toByteArray();
- BlockId id = BlockId.getIdForData(bytes);
+ out.close();
+
+ BlockId id = makeIdFromDigest();
File file = getFile(id);
- OutputStream os = new FileOutputStream(file);
- os.write(bytes);
- os.close();
+ if(!tempFile.renameTo(file)) {
+ tempFile.delete();
+ throw new IOException("Could not rename temporary file");
+ }
block = new FileBlock(id);
}
@@ -91,7 +92,7 @@
}
public InputStream getRawInputStream() throws IOException {
- return new FileInputStream(file);
+ return new BufferedInputStream(new FileInputStream(file));
}
}
@@ -115,7 +116,8 @@
if(!id.equals(b.getId()))
throw new IOException("Block with bad id supplied");
- OutputStream os = new FileOutputStream(getFile(id));
+ OutputStream os =
+ new BufferedOutputStream(new FileOutputStream(getFile(id)));
os.write(bytes);
os.close();
}
@@ -136,10 +138,12 @@
throws IOException {
Header822 hdr = new UniqueHeader822();
hdr.add("Content-Type", contentType);
- return new FileBlockOutputStream(hdr);
+ File tempFile = gzz.util.TempFileUtil.tmpFile(dir);
+ return new FileBlockOutputStream(hdr, tempFile);
}
public BlockOutputStream getBlockOutputStream(Header822 hdr)
throws IOException {
- return new FileBlockOutputStream(new VerbatimHeader822(hdr));
+ File tempFile = gzz.util.TempFileUtil.tmpFile(dir);
+ return new FileBlockOutputStream(new VerbatimHeader822(hdr), tempFile);
}
}
Index: gzz/lava/gzz/storm/impl/TransientPool.java
diff -u gzz/lava/gzz/storm/impl/TransientPool.java:1.15
gzz/lava/gzz/storm/impl/TransientPool.java:1.16
--- gzz/lava/gzz/storm/impl/TransientPool.java:1.15 Fri Nov 15 21:15:15 2002
+++ gzz/lava/gzz/storm/impl/TransientPool.java Fri Nov 15 23:38:04 2002
@@ -44,12 +44,12 @@
throws IOException {
super(new ByteArrayOutputStream(), header);
baos = (ByteArrayOutputStream)out;
- header.writeTo(baos);
}
public void close() throws IOException {
- block = new TransientBlock(baos.toByteArray(), header);
- blocks.put(block.getId(),block);
+ block = new TransientBlock(makeIdFromDigest(),
+ baos.toByteArray(), header);
+ blocks.put(block.getId(), block);
}
}
@@ -57,8 +57,9 @@
protected byte[] bytes;
protected int headerLength;
- protected TransientBlock(byte[] bytes, Header822 header) throws
IOException {
- super(BlockId.getIdForData(bytes), header);
+ protected TransientBlock(BlockId id, byte[] bytes,
+ Header822 header) throws IOException {
+ super(id, header);
this.bytes = bytes;
}
@@ -81,7 +82,8 @@
Header822 header = Headers822.readHeader(is);
is.close();
- Block block = new TransientBlock(raw, header);
+ Block block = new TransientBlock(BlockId.getIdForData(raw),
+ raw, header);
BlockId id = block.getId();
if(!id.equals(b.getId()))
Index: gzz/lava/test/gzz/storm/StormPoolTest.java
diff -u gzz/lava/test/gzz/storm/StormPoolTest.java:1.10
gzz/lava/test/gzz/storm/StormPoolTest.java:1.11
--- gzz/lava/test/gzz/storm/StormPoolTest.java:1.10 Thu Nov 14 16:25:06 2002
+++ gzz/lava/test/gzz/storm/StormPoolTest.java Fri Nov 15 23:38:04 2002
@@ -207,10 +207,10 @@
id2 = bos.getBlock().getId();
if(!id.equals(id1))
- throw new Error("bos.getBlockId() does not match");
+ throw new Error("bos.getBlockId() does not match: "+id1);
if(!id.equals(id2))
- throw new Error("bos.getBlock().getId() does not match");
+ throw new Error("bos.getBlock().getId() does not match: "+id2);
}
/** Test that trying to get a nonexistent block
Index: gzz/lava/test/gzz/storm/impl/DirPool.test
diff -u gzz/lava/test/gzz/storm/impl/DirPool.test:1.4
gzz/lava/test/gzz/storm/impl/DirPool.test:1.5
--- gzz/lava/test/gzz/storm/impl/DirPool.test:1.4 Thu Nov 14 16:25:06 2002
+++ gzz/lava/test/gzz/storm/impl/DirPool.test Fri Nov 15 23:38:04 2002
@@ -18,7 +18,7 @@
import java, gzz
-dir = gzz.util.TestingUtil.tmpFile(java.io.File("."));
+dir = gzz.util.TempFileUtil.tmpFile(java.io.File("."));
dir.mkdir();
s = gzz.storm.StormPoolTest()
@@ -34,5 +34,5 @@
def testGetNonexistent(): s.testGetNonexistent(p)
def tearDown():
- gzz.util.TestingUtil.deltree(dir)
+ gzz.util.TempFileUtil.deltree(dir)
Index: gzz/test/gzz/impl/BenchFS.java
diff -u gzz/test/gzz/impl/BenchFS.java:1.3 gzz/test/gzz/impl/BenchFS.java:1.4
--- gzz/test/gzz/impl/BenchFS.java:1.3 Sun Apr 7 08:36:43 2002
+++ gzz/test/gzz/impl/BenchFS.java Fri Nov 15 23:38:04 2002
@@ -23,7 +23,7 @@
package gzz.impl;
import gzz.mediaserver.storage.*;
-import gzz.util.TestingUtil;
+import gzz.util.TempFileUtil;
import gzz.util.Bench;
import java.io.*;
@@ -33,7 +33,7 @@
public class BenchFS {
-public static final String rcsid = "$Id: BenchFS.java,v 1.3 2002/04/07
12:36:43 tjl Exp $";
+public static final String rcsid = "$Id: BenchFS.java,v 1.4 2002/11/16
04:38:04 benja Exp $";
static private void p(String s) { System.out.println(s); }
static final String key =
@@ -43,7 +43,7 @@
final byte[] array = new byte[1024];
public void runBench(Bench b) throws IOException {
- File dir = TestingUtil.tmpFile(new File("."));
+ File dir = TempFileUtil.tmpFile(new File("."));
dir.mkdir();
final Storer s = new DirStorer(dir);
write(s.store(key));
@@ -65,7 +65,7 @@
long millis = System.currentTimeMillis();
for(int i=1; i<=nfiles; i++) {
- File f = TestingUtil.tmpFile(dir);
+ File f = TempFileUtil.tmpFile(dir);
write(new FileOutputStream(f));
if(i%500 == 0)
System.out.print(i+" ");
@@ -87,7 +87,7 @@
}},
});
- TestingUtil.deltree(dir);
+ TempFileUtil.deltree(dir);
}
void read(InputStream is) throws IOException {
Index: gzz/test/gzz/mediaserver/TestMS.java
diff -u gzz/test/gzz/mediaserver/TestMS.java:1.3
gzz/test/gzz/mediaserver/TestMS.java:1.4
--- gzz/test/gzz/mediaserver/TestMS.java:1.3 Sun Jun 30 07:31:01 2002
+++ gzz/test/gzz/mediaserver/TestMS.java Fri Nov 15 23:38:04 2002
@@ -30,7 +30,7 @@
*/
public class TestMS {
-public static final String rcsid = "$Id: TestMS.java,v 1.3 2002/06/30 11:31:01
tjl Exp $";
+public static final String rcsid = "$Id: TestMS.java,v 1.4 2002/11/16 04:38:04
benja Exp $";
public static Test suite() {
TestSuite suite = new TestSuite("Mediaserver tests");
@@ -86,12 +86,12 @@
return new gzz.mediaserver.storage.DirStorer(dir);
}
public void setUp() throws IOException {
- dir = TestingUtil.tmpFile(new File("."));
+ dir = TempFileUtil.tmpFile(new File("."));
dir.mkdir();
storer = getStorer();
}
public void tearDown() {
- TestingUtil.deltree(dir);
+ TempFileUtil.deltree(dir);
}
public void testCVSDirectory() throws IOException {
new File(dir, "CVS").mkdir();
Index: gzz/test/gzz/mediaserver/storage/TestZipStorer.java
diff -u gzz/test/gzz/mediaserver/storage/TestZipStorer.java:1.1
gzz/test/gzz/mediaserver/storage/TestZipStorer.java:1.2
--- gzz/test/gzz/mediaserver/storage/TestZipStorer.java:1.1 Mon Apr 15
17:45:32 2002
+++ gzz/test/gzz/mediaserver/storage/TestZipStorer.java Fri Nov 15 23:38:04 2002
@@ -22,7 +22,7 @@
*/
package gzz.mediaserver.storage;
-import gzz.util.TestingUtil;
+import gzz.util.TempFileUtil;
import junit.framework.*;
import java.io.*;
import java.util.*;
@@ -32,7 +32,7 @@
*/
public class TestZipStorer extends TestCase {
- public static final String rcsid = "$Id: TestZipStorer.java,v 1.1
2002/04/15 21:45:32 tjl Exp $";
+ public static final String rcsid = "$Id: TestZipStorer.java,v 1.2
2002/11/16 04:38:04 benja Exp $";
public TestZipStorer(String name) { super(name); }
@@ -40,7 +40,7 @@
File tmp;
public void setUp() throws IOException {
- tmp = TestingUtil.tmpFile(new File("."));
+ tmp = TempFileUtil.tmpFile(new File("."));
ZipOutputStream os = new ZipOutputStream(new FileOutputStream(tmp));
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Gzz-commits] gzz gzz/view/DefaultBinder.java lava/gzz/storm/...,
Benja Fallenstein <=