unicode readyness patch

include/mitsuba/core/fstream.h

@@ -99,6 +99,19 @@ public:
 	//! @}
 	// =============================================================
 
+	// =============================================================
+	//! @{ \name Miscellaneous
+	// =============================================================
+
+	/// Initialize the file I/O layer (unicode conversions etc.)
+	static void staticInitialization();
+
+	/// Release resources taken up by staticInitialization()
+	static void staticShutdown();
+
+	//! @}
+	// =============================================================
+
 	MTS_DECLARE_CLASS()
 protected:
 	/** \brief Virtual destructor

include/mitsuba/render/scenehandler.h

@@ -31,10 +31,11 @@
 
 XERCES_CPP_NAMESPACE_BEGIN
 class SAXParser;
+class XMLTranscoder;
 XERCES_CPP_NAMESPACE_END
 
-XERCES_CPP_NAMESPACE_USE
 MTS_NAMESPACE_BEGIN
+namespace xercesc = XERCES_CPP_NAMESPACE;
 
 #ifdef _MSC_VER
 // Disable warning 4275: non dll-interface used as base for dll-interface class
@@ -76,13 +77,14 @@ private:
  * \ingroup librender
  * \ingroup libpython
  */
-class MTS_EXPORT_RENDER SceneHandler : public HandlerBase {
+class MTS_EXPORT_RENDER SceneHandler : public xercesc::HandlerBase {
 public:
 	typedef std::map<std::string, ConfigurableObject *> NamedObjectMap;
 	typedef std::map<std::string, std::string, SimpleStringOrdering> ParameterMap;
 
-	SceneHandler(const SAXParser *parser, const ParameterMap &params,
+	SceneHandler(const xercesc::SAXParser *parser, 
-			NamedObjectMap *objects = NULL, bool isIncludedFile = false);
+			const ParameterMap &params, NamedObjectMap *objects = NULL, 
+			bool isIncludedFile = false);
 	virtual ~SceneHandler();
 
 	/// Convenience method -- load a scene from a given filename 
@@ -106,7 +108,7 @@ public:
 	virtual void endDocument();
 	virtual void startElement(
 		const XMLCh* const name,
-		AttributeList& attributes
+		xercesc::AttributeList& attributes
 	);
 	virtual void endElement(const XMLCh* const name);
 	virtual void characters(const XMLCh* const chars, const XMLSize_t length);
@@ -117,16 +119,12 @@ public:
 	// -----------------------------------------------------------------------
 	//  Implementation of the SAX ErrorHandler interface
 	// -----------------------------------------------------------------------
-	void warning(const SAXParseException& exc);
+	void warning(const xercesc::SAXParseException& exc);
-	void error(const SAXParseException& exc);
+	void error(const xercesc::SAXParseException& exc);
-	void fatalError(const SAXParseException& exc);
+	void fatalError(const xercesc::SAXParseException& exc);
 protected:
-	inline std::string transcode(const XMLCh * const xmlName) const {
+	std::string transcode(const XMLCh * input) const;
-		char *value = XMLString::transcode(xmlName);
+
-		std::string result(value);
-		XMLString::release(&value);
-		return result;
-	}
 	Float parseFloat(const std::string &name, const std::string &str,
 			Float defVal = -1) const;
 
@@ -163,7 +161,8 @@ private:
 	typedef std::pair<ETag, const Class *> TagEntry;
 	typedef boost::unordered_map<std::string, TagEntry> TagMap;
 	
-	const SAXParser *m_parser;
+	const xercesc::SAXParser *m_parser;
+	xercesc::XMLTranscoder* m_transcoder;
 	ref<Scene> m_scene;
 	ParameterMap m_params;
 	NamedObjectMap *m_namedObjects;

src/integrators/mlt/mlt.cpp

@@ -266,8 +266,8 @@ public:
 			nCores, nCores == 1 ? "core" : "cores", sampleCount);
 
 		if (m_config.workUnits <= 0)
-			m_config.workUnits = std::max((size_t) std::ceil((cropSize.x 
+			m_config.workUnits = std::max((int) std::ceil((cropSize.x 
-				* cropSize.y * sampleCount) / 200000.0f), (size_t) 1);
+				* cropSize.y * sampleCount) / 200000.0f), 1);
 
 		m_config.nMutations = (cropSize.x * cropSize.y *
 			sampleCount) / m_config.workUnits;

src/integrators/pssmlt/pssmlt.cpp

@@ -320,8 +320,8 @@ public:
 			m_config.technique == PathSampler::EBidirectional ? 100000 : 200000;
 
 		if (m_config.workUnits <= 0) 
-			m_config.workUnits = std::max((size_t) std::ceil((cropSize.x 
+			m_config.workUnits = std::max((int) std::ceil((cropSize.x 
-				* cropSize.y * sampleCount) / 200000.0f), (size_t) 1);
+				* cropSize.y * sampleCount) / 200000.0f), 1);
 
 		m_config.nMutations = (cropSize.x * cropSize.y *
 			sampleCount) / m_config.workUnits;

src/libcore/fstream.cpp

@@ -23,6 +23,7 @@
 # include <unistd.h>
 #else
 # include <windows.h>
+# include <boost/filesystem/detail/utf8_codecvt_facet.hpp>
 #endif
 
 MTS_NAMESPACE_BEGIN
@@ -359,5 +360,28 @@ bool FileStream::canWrite() const {
 	return d->write;
 }
 
+#if defined(__WINDOWS__)
+static boost::filesystem::detail::utf8_codecvt_facet *__facet = NULL;
+#endif
+
+void FileStream::staticInitialization() {
+#if defined(__WINDOWS__)
+	/* On Linux + MacOS, strings are assumed to be in UTF-8. On
+	   Windows, they still are, but fs::path is UTF-16. So we need
+	   a codecvt_facet to take care of the necessary conversions */
+	std::locale global_loc = std::locale(); 
+	__facet = new boost::filesystem::detail::utf8_codecvt_facet();
+	std::locale locale(global_loc, __facet);
+	boost::filesystem::path::imbue(locale); 
+#endif
+}
+
+void FileStream::staticShutdown() {
+#if defined(__WINDOWS__)
+	boost::filesystem::path::imbue(std::locale());
+	/* Can't delete __facet unfortunately, or we risk a crash .. oh well.. */
+#endif
+}
+
 MTS_IMPLEMENT_CLASS(FileStream, false, Stream)
 MTS_NAMESPACE_END

src/librender/scenehandler.cpp

@@ -23,14 +23,19 @@
 #if defined(Assert)
 # undef Assert
 #endif
+
 #include <xercesc/parsers/SAXParser.hpp>
 #include <xercesc/framework/MemBufInputSource.hpp>
+#include <xercesc/util/TransService.hpp>
 #include <mitsuba/render/scenehandler.h>
 #include <mitsuba/core/fresolver.h>
 #include <mitsuba/render/scene.h>
 #include <boost/algorithm/string.hpp>
 
 MTS_NAMESPACE_BEGIN
+XERCES_CPP_NAMESPACE_USE
+
+#define TRANSCODE_BLOCKSIZE 2048
 
 #if !defined(__OSX__)
 	#define XMLLog(level, fmt, ...) Thread::getThread()->getLogger()->log(\
@@ -94,9 +99,15 @@ SceneHandler::SceneHandler(const SAXParser *parser,
 	m_tags["transform"]  = TagEntry(ETransform,  (Class *) NULL);
 	m_tags["include"]    = TagEntry(EInclude,    (Class *) NULL);
 	m_tags["alias"]      = TagEntry(EAlias,      (Class *) NULL);
+
+
+	XMLTransService::Codes failReason; 
+	m_transcoder = XMLPlatformUtils::fgTransService->makeNewTranscoderFor(
+			"UTF-8", failReason, TRANSCODE_BLOCKSIZE);
 }
 
 SceneHandler::~SceneHandler() {
+	delete m_transcoder;
 	clear();
 	if (!m_isIncludedFile)
 		delete m_namedObjects;
@@ -112,6 +123,28 @@ void SceneHandler::clear() {
 	}
 }
 
+std::string SceneHandler::transcode(const XMLCh * input) const {
+	XMLSize_t charsToBeConsumed = XMLString::stringLen(input);
+	char output[TRANSCODE_BLOCKSIZE + 4]; 
+	XMLSize_t totalCharsConsumed = 0;
+	std::string result;
+
+	while (totalCharsConsumed < charsToBeConsumed) {
+		XMLSize_t charsConsumed = 0;
+		XMLSize_t charsProduced = m_transcoder->transcodeTo(input,
+			std::min((XMLSize_t) 2048, charsToBeConsumed - totalCharsConsumed), 
+			(XMLByte *) output, TRANSCODE_BLOCKSIZE, charsConsumed,
+			XMLTranscoder::UnRep_RepChar);
+
+		totalCharsConsumed += charsConsumed;
+		output[charsProduced] = '\0';
+		input += charsConsumed;
+		result += output;
+	}
+
+	return result;
+}
+
 // -----------------------------------------------------------------------
 //  Implementation of the SAX DocumentHandler interface
 // -----------------------------------------------------------------------

src/mitsuba/mitsuba.cpp

@@ -27,6 +27,7 @@
 #include <mitsuba/core/sched_remote.h>
 #include <mitsuba/core/sstream.h>
 #include <mitsuba/core/fresolver.h>
+#include <mitsuba/core/fstream.h>
 #include <mitsuba/core/appender.h>
 #include <mitsuba/core/sshstream.h>
 #include <mitsuba/core/shvector.h>
@@ -43,6 +44,8 @@
 #include <signal.h>
 #endif
 
+XERCES_CPP_NAMESPACE_USE
+
 using namespace mitsuba;
 
 void help() {
@@ -402,6 +405,7 @@ int main(int argc, char **argv) {
 	Statistics::staticInitialization();
 	Thread::staticInitialization();
 	Logger::staticInitialization();
+	FileStream::staticInitialization();
 	Spectrum::staticInitialization();
 	Bitmap::staticInitialization();
 	Scheduler::staticInitialization();
@@ -430,6 +434,7 @@ int main(int argc, char **argv) {
 	Scheduler::staticShutdown();
 	Bitmap::staticShutdown();
 	Spectrum::staticShutdown();
+	FileStream::staticShutdown();
 	Logger::staticShutdown();
 	Thread::staticShutdown();
 	Statistics::staticShutdown();

src/mitsuba/mtssrv.cpp

@@ -21,6 +21,7 @@
 #include <mitsuba/core/sstream.h>
 #include <mitsuba/core/statistics.h>
 #include <mitsuba/core/sshstream.h>
+#include <mitsuba/core/fstream.h>
 #include <mitsuba/core/shvector.h>
 #include <mitsuba/core/appender.h>
 #include <mitsuba/core/version.h>
@@ -389,6 +390,7 @@ int main(int argc, char **argv) {
 	Statistics::staticInitialization();
 	Thread::staticInitialization();
 	Logger::staticInitialization();
+	FileStream::staticInitialization();
 	Spectrum::staticInitialization();
 	Bitmap::staticInitialization();
 	Scheduler::staticInitialization();
@@ -414,6 +416,7 @@ int main(int argc, char **argv) {
 	Scheduler::staticShutdown();
 	Bitmap::staticShutdown();
 	Spectrum::staticShutdown();
+	FileStream::staticShutdown();
 	Logger::staticShutdown();
 	Thread::staticShutdown();
 	Statistics::staticShutdown();

src/mitsuba/mtsutil.cpp

@@ -30,6 +30,7 @@
 #include <mitsuba/core/shvector.h>
 #include <mitsuba/core/statistics.h>
 #include <mitsuba/core/fresolver.h>
+#include <mitsuba/core/fstream.h>
 #include <mitsuba/core/version.h>
 #include <mitsuba/core/appender.h>
 #include <mitsuba/render/util.h>
@@ -390,6 +391,7 @@ int mts_main(int argc, char **argv) {
 	Statistics::staticInitialization();
 	Thread::staticInitialization();
 	Logger::staticInitialization();
+	FileStream::staticInitialization();
 	Spectrum::staticInitialization();
 	Bitmap::staticInitialization();
 	Scheduler::staticInitialization();
@@ -418,6 +420,7 @@ int mts_main(int argc, char **argv) {
 	Scheduler::staticShutdown();
 	Bitmap::staticShutdown();
 	Spectrum::staticShutdown();
+	FileStream::staticShutdown();
 	Logger::staticShutdown();
 	Thread::staticShutdown();
 	Statistics::staticShutdown();

src/mtsgui/common.h

@@ -230,6 +230,8 @@ public:
 	}
 };
 
+#define QStringToUTF8(str) str.toUtf8().data()
+
 inline QString fromFsPath(const fs::path &path) {
 #if defined(__WINDOWS__)
 # if defined(_MSC_VER) && _MSC_VER >= 1600
@@ -237,7 +239,7 @@ inline QString fromFsPath(const fs::path &path) {
 # endif
 	return QString(reinterpret_cast<const QChar*>(path.c_str()));
 #else
-	return QString(path.c_str());
+	return QString::fromUtf8(path.c_str());
 #endif
 }
 
@@ -245,7 +247,7 @@ inline fs::path toFsPath(const QString &str) {
 #if defined(__WINDOWS__)
 	return fs::path(reinterpret_cast<const fs::path::value_type*>(str.constData()));
 #else
-	return fs::path(qPrintable(str));
+	return fs::path(QStringToUTF8(str));
 #endif
 }
 

src/mtsgui/logwidget.h

@@ -37,7 +37,7 @@ public:
 
     void append(ELogLevel level, const std::string &message) {
 		if (!m_ignoreMessages) {
-			emit textMessage(level, QString::fromLatin1(message.c_str()));
+			emit textMessage(level, QString::fromUtf8(message.c_str()));
 			if (level >= EWarn)
 				floodCheck();
 		}
@@ -47,8 +47,8 @@ public:
 		const std::string &formatted, const std::string &eta,
 		const void *ptr) {
 		emit progressMessage((RenderJob *) ptr, 
-			QString::fromLatin1(name.c_str()), (float) progress,
+			QString::fromUtf8(name.c_str()), (float) progress,
-			QString::fromLatin1(eta.c_str()));
+			QString::fromUtf8(eta.c_str()));
 	}
 
 	void floodCheck() {

src/mtsgui/main.cpp

@@ -23,6 +23,7 @@
 #include <mitsuba/core/sched.h>
 #include <mitsuba/core/plugin.h>
 #include <mitsuba/core/fresolver.h>
+#include <mitsuba/core/fstream.h>
 #include <mitsuba/core/appender.h>
 #include <mitsuba/core/statistics.h>
 #include <mitsuba/render/scenehandler.h>
@@ -168,6 +169,7 @@ int main(int argc, char *argv[]) {
 	Statistics::staticInitialization();
 	Thread::staticInitialization();
 	Logger::staticInitialization();
+	FileStream::staticInitialization();
 	Thread::initializeOpenMP(getCoreCount());
 	Spectrum::staticInitialization();
 	Bitmap::staticInitialization();
@@ -312,6 +314,7 @@ int main(int argc, char *argv[]) {
 	Scheduler::staticShutdown();
 	Bitmap::staticShutdown();
 	Spectrum::staticShutdown();
+	FileStream::staticShutdown();
 	Logger::staticShutdown();
 	Thread::staticShutdown();
 	Statistics::staticShutdown();

src/mtsgui/sceneloader.cpp

@@ -29,6 +29,8 @@
 #include <mitsuba/core/fstream.h>
 #include <boost/algorithm/string.hpp>
 
+XERCES_CPP_NAMESPACE_USE
+
 SceneLoader::SceneLoader(FileResolver *resolver, const fs::path &filename,
 	const std::map<std::string, std::string, SimpleStringOrdering> &parameters) 
 	: Thread("load"), m_resolver(resolver), m_filename(fromFsPath(filename)),
@@ -100,11 +102,11 @@ void SceneLoader::run() {
 				filePath = fs::absolute(filename).parent_path(),
 				baseName = filename.stem();
 
-			SLog(EInfo, "Parsing scene description from \"%s\" ..", qPrintable(m_filename));
+			SLog(EInfo, "Parsing scene description from \"%s\" ..", filename.string().c_str());
 
 			if (!fs::exists(filename))
 				SLog(EError, "Unable to load scene \"%s\": file not found!",
-					qPrintable(m_filename));
+					filename.string().c_str());
 
 			try {
 				parser->parse(filename.c_str());
@@ -134,7 +136,7 @@ void SceneLoader::run() {
 			/* Also generate a DOM representation for the Qt-based GUI */
 			QFile file(m_filename);
 			if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
-				Log(EError, "Unable to open the file \"%s\"", qPrintable(m_filename));
+				Log(EError, "Unable to open the file \"%s\"", filename.string().c_str());
 			QString errorMsg;
 			int line, column;
 			if (!m_result->doc.setContent(&file, &errorMsg, &line, &column))

src/mtsgui/upgrade.cpp

@@ -86,7 +86,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers
 	if (backupFilename.endsWith(".xml", Qt::CaseInsensitive))
 		backupFilename.replace(backupFilename.length()-4, 4, ".bak");
 	SLog(EInfo, "Saving a backup copy of \"%s\" as \"%s\"",
-		qPrintable(filename), qPrintable(backupFilename));
+		QStringToUTF8(filename), QStringToUTF8(backupFilename));
 
 	QFile file(filename), backupFile(backupFilename);
 	if (backupFile.exists())
@@ -97,7 +97,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers
 		
 	if (!file.open(QIODevice::ReadOnly))
 		SLog(EError, "Unable to open \"%s\" with read access -- stopping "
-			"the upgrade operation.", qPrintable(filename));
+			"the upgrade operation.", QStringToUTF8(filename));
 	QByteArray inputArray = file.readAll(), outputArray;
 	file.close();
 		
@@ -119,7 +119,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers
 		QFile trafoFile(trafoFilename);
 		if (!trafoFile.open(QIODevice::ReadOnly | QIODevice::Text))
 			SLog(EError, "Unable to open the stylesheet \"%s\" -- stopping "
-				"the upgrade operation.", qPrintable(trafoFilename));
+				"the upgrade operation.", QStringToUTF8(trafoFilename));
 		
 		QXmlQuery query(QXmlQuery::XSLT20);
 		query.setMessageHandler(&handler);
@@ -127,7 +127,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers
 		query.setQuery(&trafoFile);
 		if (!query.isValid())
 			SLog(EError, "Unable to parse the stylesheet \"%s\" -- stopping "
-				"the upgrade operation.", qPrintable(trafoFilename));
+				"the upgrade operation.", QStringToUTF8(trafoFilename));
 
 		SLog(EInfo, "Transformation is ready, running it now..");
 		QXmlFormatter formatter(query, &outputBuffer);
@@ -152,11 +152,11 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers
 	
 	/* Done, write back to disk */
 	SLog(EInfo, "Successfully applied %i transformations, writing the result "
-		"to \"%s\" ..", nTransformations, qPrintable(filename));
+		"to \"%s\" ..", nTransformations, QStringToUTF8(filename));
 
 	if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
 		SLog(EError, "Unable to open \"%s\" with write access -- stopping "
-			"the upgrade operation.", qPrintable(filename));
+			"the upgrade operation.", QStringToUTF8(filename));
 	}
 
 	int line, column;
@@ -164,7 +164,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers
 	QDomDocument doc;
 	if (!doc.setContent(inputArray, &errorMsg, &line, &column))
 		SLog(EError, "Unable to parse file: error %s at line %i, colum %i",
-			qPrintable(errorMsg), line, column);
+			QStringToUTF8(errorMsg), line, column);
 	QDomElement root = doc.documentElement();
 
 	/* Search for include nodes */