transparent compression support

SConstruct

@@ -256,7 +256,7 @@ libcore_objects = [
 	'src/libcore/cstream.cpp', 'src/libcore/mstream.cpp', 
 	'src/libcore/sched.cpp', 'src/libcore/sched_remote.cpp',
 	'src/libcore/sshstream.cpp', 'src/libcore/wavelet.cpp',
-	'src/libcore/shvector.cpp'
+	'src/libcore/zstream.cpp', 'src/libcore/shvector.cpp'
 ]
 if sys.platform == 'darwin':
 	coreEnv_osx = coreEnv.Clone();

include/mitsuba/core/mstream.h

@@ -10,7 +10,7 @@ MTS_NAMESPACE_BEGIN
  */
 class MTS_EXPORT_CORE MemoryStream : public Stream {
 public:
-	/// Create a new console stream
+	/// Create a new memory stream
 	MemoryStream(size_t initialSize = 512);
 
 	/// Return the underlying data
@@ -40,11 +40,7 @@ public:
 protected:
 	void resize(size_t newSize);
 
-	/** \brief Virtual destructor
+	// \brief Virtual destructor
-	 *
-	 * The destructor frees all resources and closes
-	 * the socket if it is still open
-	 */
 	virtual ~MemoryStream();
 protected:
 	size_t m_capacity;

include/mitsuba/core/zstream.h

@@ -0,0 +1,51 @@
+#if !defined(__ZSTREAM_H)
+#define __ZSTREAM_H
+
+#include <mitsuba/mitsuba.h>
+#include <zlib.h>
+#define ZSTREAM_BUFSIZE 16384
+
+MTS_NAMESPACE_BEGIN
+
+/** \brief Transparent compression/decompression stream based on ZLIB.
+ */
+class MTS_EXPORT_CORE ZStream : public Stream {
+public:
+	/// Create a new compression stream
+	ZStream(Stream *childStream, int level = Z_DEFAULT_COMPRESSION);
+
+	/// Return the child stream of this compression stream
+	inline const Stream *getChildStream() const { return m_childStream.get(); }
+
+	/// Return the child stream of this compression stream
+	inline Stream *getChildStream() { return m_childStream; }
+
+	/* Stream implementation */
+	void read(void *ptr, size_t size);
+	void write(const void *ptr, size_t size);
+	void setPos(size_t pos);
+	size_t getPos() const;
+	size_t getSize() const;
+	void truncate(size_t size);
+	void flush();
+	bool canWrite() const;
+	bool canRead() const;
+
+	/// Return a string representation
+	std::string toString() const;
+
+	MTS_DECLARE_CLASS()
+protected:
+	// \brief Virtual destructor
+	virtual ~ZStream();
+private:
+	ref<Stream> m_childStream;
+	z_stream m_deflateStream, m_inflateStream;
+	uint8_t m_deflateBuffer[ZSTREAM_BUFSIZE];
+	uint8_t m_inflateBuffer[ZSTREAM_BUFSIZE];
+	bool m_didWrite;
+};
+
+MTS_NAMESPACE_END
+
+#endif /* __ZSTREAM_H */

src/libcore/fstream.cpp

@@ -85,7 +85,7 @@ void FileStream::open(const std::string &filename, EFileMode mode) {
 		Log(EError, "Unknown file mode");
 		break;
 	}
-	
+
 	m_file = CreateFile(filename.c_str(), dwDesiredAccess, 
 		FILE_SHARE_WRITE | FILE_SHARE_READ, 0, 
 		dwCreationDisposition, FILE_ATTRIBUTE_NORMAL, 0);

src/libcore/zstream.cpp

@@ -0,0 +1,148 @@
+#include <mitsuba/core/zstream.h>
+
+MTS_NAMESPACE_BEGIN
+
+ZStream::ZStream(Stream *childStream, int level) 
+		: m_childStream(childStream), m_didWrite(false) {
+	m_deflateStream.zalloc = Z_NULL;
+	m_deflateStream.zfree = Z_NULL;
+	m_deflateStream.opaque = Z_NULL;
+
+	int retval = deflateInit(&m_deflateStream, level);
+	if (retval != Z_OK)
+		Log(EError, "Could not initialize ZLIB: error code %i", retval);
+
+	m_inflateStream.zalloc = Z_NULL;
+	m_inflateStream.zfree = Z_NULL;
+	m_inflateStream.opaque = Z_NULL;
+	m_inflateStream.avail_in = 0;
+	m_inflateStream.next_in = Z_NULL;
+
+	retval = inflateInit(&m_inflateStream);
+	if (retval != Z_OK)
+		Log(EError, "Could not initialize ZLIB: error code %i", retval);
+}
+
+bool ZStream::canWrite() const {
+	return m_childStream->canWrite();
+}
+
+bool ZStream::canRead() const {
+	return m_childStream->canRead();
+}
+
+void ZStream::setPos(size_t pos) {
+	Log(EError, "setPos(): unsupported in a ZLIB stream!");
+}
+
+size_t ZStream::getPos() const {
+	Log(EError, "getPos(): unsupported in a ZLIB stream!");
+	return 0;
+}
+
+size_t ZStream::getSize() const {
+	Log(EError, "getSize(): unsupported in a ZLIB stream!");
+	return 0;
+}
+
+void ZStream::truncate(size_t size) {
+	Log(EError, "truncate(): unsupported in a ZLIB stream!");
+}
+
+void ZStream::flush() {
+	Log(EError, "flush(): not implemented!");
+}
+
+void ZStream::write(const void *ptr, size_t size) {
+	m_deflateStream.avail_in = size;
+	m_deflateStream.next_in = (uint8_t *) ptr;
+
+	int outputSize = 0;
+
+	do {
+		m_deflateStream.avail_out = sizeof(m_deflateBuffer);
+		m_deflateStream.next_out = m_deflateBuffer;
+
+		int retval = deflate(&m_deflateStream, Z_NO_FLUSH);
+		if (retval == Z_STREAM_ERROR)
+			Log(EError, "deflate(): stream error!");
+
+		outputSize = sizeof(m_deflateBuffer) - m_deflateStream.avail_out;
+
+		m_childStream->write(m_deflateBuffer, outputSize);
+	} while (outputSize != 0);
+
+	Assert(m_deflateStream.avail_in == 0);
+	m_didWrite = true;
+}
+
+void ZStream::read(void *ptr, size_t size) {
+	uint8_t *targetPtr = (uint8_t *) ptr;
+	while (size > 0) {
+		if (m_inflateStream.avail_in == 0) {
+			size_t remaining = m_childStream->getSize() - m_childStream->getPos();
+			m_inflateStream.next_in = m_inflateBuffer;
+			m_inflateStream.avail_in = std::min(remaining, sizeof(m_inflateBuffer));
+			if (m_inflateStream.avail_in == 0)
+				Log(EError, "Read less data than expected (%i more bytes required)", size);
+			m_childStream->read(m_inflateBuffer, m_inflateStream.avail_in);
+		}
+
+		m_inflateStream.avail_out = size;
+		m_inflateStream.next_out = targetPtr;
+
+		int retval = inflate(&m_inflateStream, Z_NO_FLUSH);
+		switch (retval) {
+			case Z_STREAM_ERROR:
+				Log(EError, "inflate(): stream error!");
+			case Z_NEED_DICT:
+				Log(EError, "inflate(): need dictionary!");
+			case Z_DATA_ERROR:
+				Log(EError, "inflate(): data error!");
+			case Z_MEM_ERROR:
+				Log(EError, "inflate(): memory error!");
+		};
+
+		int outputSize = size - m_inflateStream.avail_out;
+		targetPtr += outputSize;
+		size -= outputSize;
+
+		if (size > 0 && retval == Z_STREAM_END)
+			Log(EError, "inflate(): attempting to read past the end of the stream!");
+	}
+}
+
+ZStream::~ZStream() {
+	if (m_didWrite) {
+		m_deflateStream.avail_in = 0;
+		m_deflateStream.next_in = NULL;
+		int outputSize = 0;
+
+		do {
+			m_deflateStream.avail_out = sizeof(m_deflateBuffer);
+			m_deflateStream.next_out = m_deflateBuffer;
+
+			int retval = deflate(&m_deflateStream, Z_FINISH);
+			if (retval == Z_STREAM_ERROR)
+				Log(EError, "deflate(): stream error!");
+
+			outputSize = sizeof(m_deflateBuffer) - m_deflateStream.avail_out;
+
+			m_childStream->write(m_deflateBuffer, outputSize);
+		} while (outputSize != 0);
+	}
+
+	deflateEnd(&m_deflateStream);
+	inflateEnd(&m_inflateStream);
+}
+
+std::string ZStream::toString() const {
+	std::ostringstream oss;
+	oss << "ZStream[" << endl
+		<< "  childStream = " << indent(m_childStream->toString()) << endl
+		<< "]";
+	return oss.str();
+}
+
+MTS_IMPLEMENT_CLASS(ZStream, false, Stream)
+MTS_NAMESPACE_END

src/librender/trimesh.cpp

@@ -1,9 +1,11 @@
 #include <mitsuba/render/trimesh.h>
 #include <mitsuba/core/random.h>
 #include <mitsuba/core/plugin.h>
+#include <mitsuba/core/zstream.h>
 
-#define MTS_FILEFORMAT_HEADER  0x041C
+#define MTS_FILEFORMAT_HEADER 0x041C
-#define MTS_FILEFORMAT_VERSION 0x01
+#define MTS_FILEFORMAT_VERSION_V1 0x01
+#define MTS_FILEFORMAT_VERSION_V2 0x02
 
 MTS_NAMESPACE_BEGIN
 
@@ -46,7 +48,9 @@ TriMesh::TriMesh(Stream *stream, InstanceManager *manager)
 	configure();
 }
 	
-TriMesh::TriMesh(Stream *stream) : Shape(Properties()) {
+TriMesh::TriMesh(Stream *_stream) : Shape(Properties()) {
+	ref<Stream> stream = _stream;
+
 	Assert(sizeof(Vertex) == 14*sizeof(Float));
 	Assert(sizeof(Triangle) == 3*sizeof(int));	
 	if (stream->getByteOrder() != Stream::ENetworkByteOrder) 
@@ -61,9 +65,15 @@ TriMesh::TriMesh(Stream *stream) : Shape(Properties()) {
 	if (stream->readShort() != MTS_FILEFORMAT_HEADER)
 		Log(EError, "Encountered an invalid file format!");
 
-	if (stream->readShort() != MTS_FILEFORMAT_VERSION)
+	short version = stream->readShort();
+
+	if (version != MTS_FILEFORMAT_VERSION_V1 &&
+		version != MTS_FILEFORMAT_VERSION_V2)
 		Log(EError, "Encountered an incompatible file version!");
 
+	if (version == MTS_FILEFORMAT_VERSION_V2)
+		stream = new ZStream(stream);
+
 	bool fileDoublePrecision = stream->readBool();
 	m_vertexCount = (size_t) stream->readULong();
 	m_vertexBuffer = new Vertex[m_vertexCount];
@@ -291,7 +301,9 @@ void TriMesh::serialize(Stream *stream, InstanceManager *manager) const {
 		m_triangleCount * sizeof(Triangle)/sizeof(int));
 }
 
-void TriMesh::serialize(Stream *stream) const {
+void TriMesh::serialize(Stream *_stream) const {
+	ref<Stream> stream = _stream;
+
 	Assert(sizeof(Vertex) == 14*sizeof(Float));
 	Assert(sizeof(Triangle) == 3*sizeof(int));
 
@@ -306,7 +318,9 @@ void TriMesh::serialize(Stream *stream) const {
 #endif
 
 	stream->writeShort(MTS_FILEFORMAT_HEADER);
-	stream->writeShort(MTS_FILEFORMAT_VERSION);
+	stream->writeShort(MTS_FILEFORMAT_VERSION_V2);
+
+	stream = new ZStream(stream, Z_BEST_COMPRESSION);
 	stream->writeBool(doublePrecision);
 	stream->writeULong(m_vertexCount);
 	stream->writeFloatArray(reinterpret_cast<Float *>(m_vertexBuffer),