From 68c265cfdc3dde9469079db2e3ebf5fb1308feb3 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Mon, 8 Oct 2012 23:02:27 -0700 Subject: [PATCH] unicode readyness patch --- include/mitsuba/core/fstream.h | 13 +++++++++++ include/mitsuba/render/scenehandler.h | 29 ++++++++++++----------- src/integrators/mlt/mlt.cpp | 4 ++-- src/integrators/pssmlt/pssmlt.cpp | 4 ++-- src/libcore/fstream.cpp | 24 +++++++++++++++++++ src/librender/scenehandler.cpp | 33 +++++++++++++++++++++++++++ src/mitsuba/mitsuba.cpp | 5 ++++ src/mitsuba/mtssrv.cpp | 3 +++ src/mitsuba/mtsutil.cpp | 3 +++ src/mtsgui/common.h | 6 +++-- src/mtsgui/logwidget.h | 6 ++--- src/mtsgui/main.cpp | 3 +++ src/mtsgui/sceneloader.cpp | 8 ++++--- src/mtsgui/upgrade.cpp | 14 ++++++------ 14 files changed, 121 insertions(+), 34 deletions(-) diff --git a/include/mitsuba/core/fstream.h b/include/mitsuba/core/fstream.h index 19b00452..b54993b3 100644 --- a/include/mitsuba/core/fstream.h +++ b/include/mitsuba/core/fstream.h @@ -99,6 +99,19 @@ public: //! @} // ============================================================= + // ============================================================= + //! @{ \name Miscellaneous + // ============================================================= + + /// Initialize the file I/O layer (unicode conversions etc.) + static void staticInitialization(); + + /// Release resources taken up by staticInitialization() + static void staticShutdown(); + + //! @} + // ============================================================= + MTS_DECLARE_CLASS() protected: /** \brief Virtual destructor diff --git a/include/mitsuba/render/scenehandler.h b/include/mitsuba/render/scenehandler.h index cbc782e1..08d6ac8d 100644 --- a/include/mitsuba/render/scenehandler.h +++ b/include/mitsuba/render/scenehandler.h @@ -31,10 +31,11 @@ XERCES_CPP_NAMESPACE_BEGIN class SAXParser; +class XMLTranscoder; XERCES_CPP_NAMESPACE_END -XERCES_CPP_NAMESPACE_USE MTS_NAMESPACE_BEGIN +namespace xercesc = XERCES_CPP_NAMESPACE; #ifdef _MSC_VER // Disable warning 4275: non dll-interface used as base for dll-interface class @@ -76,13 +77,14 @@ private: * \ingroup librender * \ingroup libpython */ -class MTS_EXPORT_RENDER SceneHandler : public HandlerBase { +class MTS_EXPORT_RENDER SceneHandler : public xercesc::HandlerBase { public: typedef std::map NamedObjectMap; typedef std::map ParameterMap; - SceneHandler(const SAXParser *parser, const ParameterMap ¶ms, - NamedObjectMap *objects = NULL, bool isIncludedFile = false); + SceneHandler(const xercesc::SAXParser *parser, + const ParameterMap ¶ms, NamedObjectMap *objects = NULL, + bool isIncludedFile = false); virtual ~SceneHandler(); /// Convenience method -- load a scene from a given filename @@ -106,7 +108,7 @@ public: virtual void endDocument(); virtual void startElement( const XMLCh* const name, - AttributeList& attributes + xercesc::AttributeList& attributes ); virtual void endElement(const XMLCh* const name); virtual void characters(const XMLCh* const chars, const XMLSize_t length); @@ -117,16 +119,12 @@ public: // ----------------------------------------------------------------------- // Implementation of the SAX ErrorHandler interface // ----------------------------------------------------------------------- - void warning(const SAXParseException& exc); - void error(const SAXParseException& exc); - void fatalError(const SAXParseException& exc); + void warning(const xercesc::SAXParseException& exc); + void error(const xercesc::SAXParseException& exc); + void fatalError(const xercesc::SAXParseException& exc); protected: - inline std::string transcode(const XMLCh * const xmlName) const { - char *value = XMLString::transcode(xmlName); - std::string result(value); - XMLString::release(&value); - return result; - } + std::string transcode(const XMLCh * input) const; + Float parseFloat(const std::string &name, const std::string &str, Float defVal = -1) const; @@ -163,7 +161,8 @@ private: typedef std::pair TagEntry; typedef boost::unordered_map TagMap; - const SAXParser *m_parser; + const xercesc::SAXParser *m_parser; + xercesc::XMLTranscoder* m_transcoder; ref m_scene; ParameterMap m_params; NamedObjectMap *m_namedObjects; diff --git a/src/integrators/mlt/mlt.cpp b/src/integrators/mlt/mlt.cpp index 69728335..7e40250b 100644 --- a/src/integrators/mlt/mlt.cpp +++ b/src/integrators/mlt/mlt.cpp @@ -266,8 +266,8 @@ public: nCores, nCores == 1 ? "core" : "cores", sampleCount); if (m_config.workUnits <= 0) - m_config.workUnits = std::max((size_t) std::ceil((cropSize.x - * cropSize.y * sampleCount) / 200000.0f), (size_t) 1); + m_config.workUnits = std::max((int) std::ceil((cropSize.x + * cropSize.y * sampleCount) / 200000.0f), 1); m_config.nMutations = (cropSize.x * cropSize.y * sampleCount) / m_config.workUnits; diff --git a/src/integrators/pssmlt/pssmlt.cpp b/src/integrators/pssmlt/pssmlt.cpp index 98619b8b..12539619 100644 --- a/src/integrators/pssmlt/pssmlt.cpp +++ b/src/integrators/pssmlt/pssmlt.cpp @@ -320,8 +320,8 @@ public: m_config.technique == PathSampler::EBidirectional ? 100000 : 200000; if (m_config.workUnits <= 0) - m_config.workUnits = std::max((size_t) std::ceil((cropSize.x - * cropSize.y * sampleCount) / 200000.0f), (size_t) 1); + m_config.workUnits = std::max((int) std::ceil((cropSize.x + * cropSize.y * sampleCount) / 200000.0f), 1); m_config.nMutations = (cropSize.x * cropSize.y * sampleCount) / m_config.workUnits; diff --git a/src/libcore/fstream.cpp b/src/libcore/fstream.cpp index fe94a97a..072335b6 100644 --- a/src/libcore/fstream.cpp +++ b/src/libcore/fstream.cpp @@ -23,6 +23,7 @@ # include #else # include +# include #endif MTS_NAMESPACE_BEGIN @@ -359,5 +360,28 @@ bool FileStream::canWrite() const { return d->write; } +#if defined(__WINDOWS__) +static boost::filesystem::detail::utf8_codecvt_facet *__facet = NULL; +#endif + +void FileStream::staticInitialization() { +#if defined(__WINDOWS__) + /* On Linux + MacOS, strings are assumed to be in UTF-8. On + Windows, they still are, but fs::path is UTF-16. So we need + a codecvt_facet to take care of the necessary conversions */ + std::locale global_loc = std::locale(); + __facet = new boost::filesystem::detail::utf8_codecvt_facet(); + std::locale locale(global_loc, __facet); + boost::filesystem::path::imbue(locale); +#endif +} + +void FileStream::staticShutdown() { +#if defined(__WINDOWS__) + boost::filesystem::path::imbue(std::locale()); + /* Can't delete __facet unfortunately, or we risk a crash .. oh well.. */ +#endif +} + MTS_IMPLEMENT_CLASS(FileStream, false, Stream) MTS_NAMESPACE_END diff --git a/src/librender/scenehandler.cpp b/src/librender/scenehandler.cpp index 60ad3ff3..bb82a207 100644 --- a/src/librender/scenehandler.cpp +++ b/src/librender/scenehandler.cpp @@ -23,14 +23,19 @@ #if defined(Assert) # undef Assert #endif + #include #include +#include #include #include #include #include MTS_NAMESPACE_BEGIN +XERCES_CPP_NAMESPACE_USE + +#define TRANSCODE_BLOCKSIZE 2048 #if !defined(__OSX__) #define XMLLog(level, fmt, ...) Thread::getThread()->getLogger()->log(\ @@ -94,9 +99,15 @@ SceneHandler::SceneHandler(const SAXParser *parser, m_tags["transform"] = TagEntry(ETransform, (Class *) NULL); m_tags["include"] = TagEntry(EInclude, (Class *) NULL); m_tags["alias"] = TagEntry(EAlias, (Class *) NULL); + + + XMLTransService::Codes failReason; + m_transcoder = XMLPlatformUtils::fgTransService->makeNewTranscoderFor( + "UTF-8", failReason, TRANSCODE_BLOCKSIZE); } SceneHandler::~SceneHandler() { + delete m_transcoder; clear(); if (!m_isIncludedFile) delete m_namedObjects; @@ -112,6 +123,28 @@ void SceneHandler::clear() { } } +std::string SceneHandler::transcode(const XMLCh * input) const { + XMLSize_t charsToBeConsumed = XMLString::stringLen(input); + char output[TRANSCODE_BLOCKSIZE + 4]; + XMLSize_t totalCharsConsumed = 0; + std::string result; + + while (totalCharsConsumed < charsToBeConsumed) { + XMLSize_t charsConsumed = 0; + XMLSize_t charsProduced = m_transcoder->transcodeTo(input, + std::min((XMLSize_t) 2048, charsToBeConsumed - totalCharsConsumed), + (XMLByte *) output, TRANSCODE_BLOCKSIZE, charsConsumed, + XMLTranscoder::UnRep_RepChar); + + totalCharsConsumed += charsConsumed; + output[charsProduced] = '\0'; + input += charsConsumed; + result += output; + } + + return result; +} + // ----------------------------------------------------------------------- // Implementation of the SAX DocumentHandler interface // ----------------------------------------------------------------------- diff --git a/src/mitsuba/mitsuba.cpp b/src/mitsuba/mitsuba.cpp index 26540846..b515b7aa 100644 --- a/src/mitsuba/mitsuba.cpp +++ b/src/mitsuba/mitsuba.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,8 @@ #include #endif +XERCES_CPP_NAMESPACE_USE + using namespace mitsuba; void help() { @@ -402,6 +405,7 @@ int main(int argc, char **argv) { Statistics::staticInitialization(); Thread::staticInitialization(); Logger::staticInitialization(); + FileStream::staticInitialization(); Spectrum::staticInitialization(); Bitmap::staticInitialization(); Scheduler::staticInitialization(); @@ -430,6 +434,7 @@ int main(int argc, char **argv) { Scheduler::staticShutdown(); Bitmap::staticShutdown(); Spectrum::staticShutdown(); + FileStream::staticShutdown(); Logger::staticShutdown(); Thread::staticShutdown(); Statistics::staticShutdown(); diff --git a/src/mitsuba/mtssrv.cpp b/src/mitsuba/mtssrv.cpp index ee597b04..e9d57fa6 100644 --- a/src/mitsuba/mtssrv.cpp +++ b/src/mitsuba/mtssrv.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -389,6 +390,7 @@ int main(int argc, char **argv) { Statistics::staticInitialization(); Thread::staticInitialization(); Logger::staticInitialization(); + FileStream::staticInitialization(); Spectrum::staticInitialization(); Bitmap::staticInitialization(); Scheduler::staticInitialization(); @@ -414,6 +416,7 @@ int main(int argc, char **argv) { Scheduler::staticShutdown(); Bitmap::staticShutdown(); Spectrum::staticShutdown(); + FileStream::staticShutdown(); Logger::staticShutdown(); Thread::staticShutdown(); Statistics::staticShutdown(); diff --git a/src/mitsuba/mtsutil.cpp b/src/mitsuba/mtsutil.cpp index f7b55b68..4f9df2a3 100644 --- a/src/mitsuba/mtsutil.cpp +++ b/src/mitsuba/mtsutil.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -390,6 +391,7 @@ int mts_main(int argc, char **argv) { Statistics::staticInitialization(); Thread::staticInitialization(); Logger::staticInitialization(); + FileStream::staticInitialization(); Spectrum::staticInitialization(); Bitmap::staticInitialization(); Scheduler::staticInitialization(); @@ -418,6 +420,7 @@ int mts_main(int argc, char **argv) { Scheduler::staticShutdown(); Bitmap::staticShutdown(); Spectrum::staticShutdown(); + FileStream::staticShutdown(); Logger::staticShutdown(); Thread::staticShutdown(); Statistics::staticShutdown(); diff --git a/src/mtsgui/common.h b/src/mtsgui/common.h index 9550ad41..39b0d430 100644 --- a/src/mtsgui/common.h +++ b/src/mtsgui/common.h @@ -230,6 +230,8 @@ public: } }; +#define QStringToUTF8(str) str.toUtf8().data() + inline QString fromFsPath(const fs::path &path) { #if defined(__WINDOWS__) # if defined(_MSC_VER) && _MSC_VER >= 1600 @@ -237,7 +239,7 @@ inline QString fromFsPath(const fs::path &path) { # endif return QString(reinterpret_cast(path.c_str())); #else - return QString(path.c_str()); + return QString::fromUtf8(path.c_str()); #endif } @@ -245,7 +247,7 @@ inline fs::path toFsPath(const QString &str) { #if defined(__WINDOWS__) return fs::path(reinterpret_cast(str.constData())); #else - return fs::path(qPrintable(str)); + return fs::path(QStringToUTF8(str)); #endif } diff --git a/src/mtsgui/logwidget.h b/src/mtsgui/logwidget.h index 68408809..53fc73d9 100644 --- a/src/mtsgui/logwidget.h +++ b/src/mtsgui/logwidget.h @@ -37,7 +37,7 @@ public: void append(ELogLevel level, const std::string &message) { if (!m_ignoreMessages) { - emit textMessage(level, QString::fromLatin1(message.c_str())); + emit textMessage(level, QString::fromUtf8(message.c_str())); if (level >= EWarn) floodCheck(); } @@ -47,8 +47,8 @@ public: const std::string &formatted, const std::string &eta, const void *ptr) { emit progressMessage((RenderJob *) ptr, - QString::fromLatin1(name.c_str()), (float) progress, - QString::fromLatin1(eta.c_str())); + QString::fromUtf8(name.c_str()), (float) progress, + QString::fromUtf8(eta.c_str())); } void floodCheck() { diff --git a/src/mtsgui/main.cpp b/src/mtsgui/main.cpp index 1f880b49..e315e59c 100644 --- a/src/mtsgui/main.cpp +++ b/src/mtsgui/main.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -168,6 +169,7 @@ int main(int argc, char *argv[]) { Statistics::staticInitialization(); Thread::staticInitialization(); Logger::staticInitialization(); + FileStream::staticInitialization(); Thread::initializeOpenMP(getCoreCount()); Spectrum::staticInitialization(); Bitmap::staticInitialization(); @@ -312,6 +314,7 @@ int main(int argc, char *argv[]) { Scheduler::staticShutdown(); Bitmap::staticShutdown(); Spectrum::staticShutdown(); + FileStream::staticShutdown(); Logger::staticShutdown(); Thread::staticShutdown(); Statistics::staticShutdown(); diff --git a/src/mtsgui/sceneloader.cpp b/src/mtsgui/sceneloader.cpp index c9771199..d52af8be 100644 --- a/src/mtsgui/sceneloader.cpp +++ b/src/mtsgui/sceneloader.cpp @@ -29,6 +29,8 @@ #include #include +XERCES_CPP_NAMESPACE_USE + SceneLoader::SceneLoader(FileResolver *resolver, const fs::path &filename, const std::map ¶meters) : Thread("load"), m_resolver(resolver), m_filename(fromFsPath(filename)), @@ -100,11 +102,11 @@ void SceneLoader::run() { filePath = fs::absolute(filename).parent_path(), baseName = filename.stem(); - SLog(EInfo, "Parsing scene description from \"%s\" ..", qPrintable(m_filename)); + SLog(EInfo, "Parsing scene description from \"%s\" ..", filename.string().c_str()); if (!fs::exists(filename)) SLog(EError, "Unable to load scene \"%s\": file not found!", - qPrintable(m_filename)); + filename.string().c_str()); try { parser->parse(filename.c_str()); @@ -134,7 +136,7 @@ void SceneLoader::run() { /* Also generate a DOM representation for the Qt-based GUI */ QFile file(m_filename); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) - Log(EError, "Unable to open the file \"%s\"", qPrintable(m_filename)); + Log(EError, "Unable to open the file \"%s\"", filename.string().c_str()); QString errorMsg; int line, column; if (!m_result->doc.setContent(&file, &errorMsg, &line, &column)) diff --git a/src/mtsgui/upgrade.cpp b/src/mtsgui/upgrade.cpp index e0e60a3d..869a4612 100644 --- a/src/mtsgui/upgrade.cpp +++ b/src/mtsgui/upgrade.cpp @@ -86,7 +86,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers if (backupFilename.endsWith(".xml", Qt::CaseInsensitive)) backupFilename.replace(backupFilename.length()-4, 4, ".bak"); SLog(EInfo, "Saving a backup copy of \"%s\" as \"%s\"", - qPrintable(filename), qPrintable(backupFilename)); + QStringToUTF8(filename), QStringToUTF8(backupFilename)); QFile file(filename), backupFile(backupFilename); if (backupFile.exists()) @@ -97,7 +97,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers if (!file.open(QIODevice::ReadOnly)) SLog(EError, "Unable to open \"%s\" with read access -- stopping " - "the upgrade operation.", qPrintable(filename)); + "the upgrade operation.", QStringToUTF8(filename)); QByteArray inputArray = file.readAll(), outputArray; file.close(); @@ -119,7 +119,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers QFile trafoFile(trafoFilename); if (!trafoFile.open(QIODevice::ReadOnly | QIODevice::Text)) SLog(EError, "Unable to open the stylesheet \"%s\" -- stopping " - "the upgrade operation.", qPrintable(trafoFilename)); + "the upgrade operation.", QStringToUTF8(trafoFilename)); QXmlQuery query(QXmlQuery::XSLT20); query.setMessageHandler(&handler); @@ -127,7 +127,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers query.setQuery(&trafoFile); if (!query.isValid()) SLog(EError, "Unable to parse the stylesheet \"%s\" -- stopping " - "the upgrade operation.", qPrintable(trafoFilename)); + "the upgrade operation.", QStringToUTF8(trafoFilename)); SLog(EInfo, "Transformation is ready, running it now.."); QXmlFormatter formatter(query, &outputBuffer); @@ -152,11 +152,11 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers /* Done, write back to disk */ SLog(EInfo, "Successfully applied %i transformations, writing the result " - "to \"%s\" ..", nTransformations, qPrintable(filename)); + "to \"%s\" ..", nTransformations, QStringToUTF8(filename)); if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { SLog(EError, "Unable to open \"%s\" with write access -- stopping " - "the upgrade operation.", qPrintable(filename)); + "the upgrade operation.", QStringToUTF8(filename)); } int line, column; @@ -164,7 +164,7 @@ void UpgradeManager::performUpgrade(const QString &filename, const Version &vers QDomDocument doc; if (!doc.setContent(inputArray, &errorMsg, &line, &column)) SLog(EError, "Unable to parse file: error %s at line %i, colum %i", - qPrintable(errorMsg), line, column); + QStringToUTF8(errorMsg), line, column); QDomElement root = doc.documentElement(); /* Search for include nodes */