From 299b2d7a563b061d0150b07613523fedb1be3970 Mon Sep 17 00:00:00 2001 From: dehnert Date: Fri, 14 Oct 2016 13:47:13 +0200 Subject: [PATCH] some start on JIT-based model builder Former-commit-id: b0bffd4908891485c61b7716de9759270d538c79 [formerly 95829c4970e911a9992d9bbd86929f5a1fbf0644] Former-commit-id: 8e98da5dd43fdcab0cd0b4dc0f4cd760e2597ca6 --- CMakeLists.txt | 4 +- resources/3rdparty/CMakeLists.txt | 19 +- resources/3rdparty/cpptemplate/cpptempl.h | 21 +- resources/3rdparty/include_cpptemplate.cmake | 19 + .../3rdparty/utf8_v2_3_4/doc/ReleaseNotes | 12 + .../3rdparty/utf8_v2_3_4/doc/utf8cpp.html | 1789 +++++++++++++++++ resources/3rdparty/utf8_v2_3_4/source/utf8.h | 34 + .../utf8_v2_3_4/source/utf8/checked.h | 327 +++ .../3rdparty/utf8_v2_3_4/source/utf8/core.h | 329 +++ .../utf8_v2_3_4/source/utf8/unchecked.h | 228 +++ src/builder/ExplicitJitJaniModelBuilder.cpp | 165 ++ src/builder/ExplicitJitJaniModelBuilder.h | 50 + src/builder/JitModelBuilderInterface.h | 19 + .../ExplicitJitJaniModelBuilderTest.cpp | 12 + 14 files changed, 3023 insertions(+), 5 deletions(-) create mode 100644 resources/3rdparty/include_cpptemplate.cmake create mode 100644 resources/3rdparty/utf8_v2_3_4/doc/ReleaseNotes create mode 100644 resources/3rdparty/utf8_v2_3_4/doc/utf8cpp.html create mode 100644 resources/3rdparty/utf8_v2_3_4/source/utf8.h create mode 100644 resources/3rdparty/utf8_v2_3_4/source/utf8/checked.h create mode 100644 resources/3rdparty/utf8_v2_3_4/source/utf8/core.h create mode 100644 resources/3rdparty/utf8_v2_3_4/source/utf8/unchecked.h create mode 100644 src/builder/ExplicitJitJaniModelBuilder.cpp create mode 100644 src/builder/ExplicitJitJaniModelBuilder.h create mode 100644 src/builder/JitModelBuilderInterface.h create mode 100644 test/functional/builder/ExplicitJitJaniModelBuilderTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 645f0649d..1b532ddd9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,7 +123,7 @@ set(CONVERSIONHELPER_TARGET "${PROJECT_SOURCE_DIR}/src/utility/ConversionHelper. if(CMAKE_COMPILER_IS_GNUCC) set(STORM_COMPILED_BY "GCC") # Set standard flags for GCC - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -funroll-loops") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -funroll-loops --fast-math -fno-finite-math-only") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -funroll-loops") # TODO: remove forcing the old version of optional as soon as the related Spirit bug is fixed: @@ -165,7 +165,7 @@ else(CLANG) # As CLANG is not set as a variable, we need to set it in case we have not matched another compiler. set (CLANG ON) # Set standard flags for clang - set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -funroll-loops -O3") + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -funroll-loops -ffast-math -fno-finite-math-only") if(UNIX AND NOT APPLE) if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.2) diff --git a/resources/3rdparty/CMakeLists.txt b/resources/3rdparty/CMakeLists.txt index 07e60829f..e2236b621 100644 --- a/resources/3rdparty/CMakeLists.txt +++ b/resources/3rdparty/CMakeLists.txt @@ -64,7 +64,7 @@ set(Boost_USE_STATIC_LIBS ${USE_BOOST_STATIC_LIBRARIES}) set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) -find_package(Boost 1.57.0 QUIET REQUIRED) +find_package(Boost 1.57.0 QUIET REQUIRED COMPONENTS filesystem) if ((NOT Boost_LIBRARY_DIRS) OR ("${Boost_LIBRARY_DIRS}" STREQUAL "")) set(Boost_LIBRARY_DIRS "${Boost_INCLUDE_DIRS}/stage/lib") @@ -97,6 +97,16 @@ include_directories("${PROJECT_SOURCE_DIR}/resources/3rdparty/exprtk") message (STATUS "StoRM - Including Sparsepp") include_directories("${PROJECT_SOURCE_DIR}/resources/3rdparty/sparsepp") +############################################################# +## +## utf8cpp +## +############################################################# + +# Use the shipped version of utf8cpp +message (STATUS "StoRM - Including utf8cpp") +include_directories("${PROJECT_SOURCE_DIR}/resources/3rdparty/utf8_v2_3_4/source") + ############################################################# ## ## ModernJSON @@ -160,6 +170,13 @@ endif() ############################################################# include(${STORM_3RDPARTY_SOURCE_DIR}/include_cudd.cmake) +############################################################# +## +## cpptemplate +## +############################################################# +include(${STORM_3RDPARTY_SOURCE_DIR}/include_cpptemplate.cmake) + ############################################################# ## ## CLN diff --git a/resources/3rdparty/cpptemplate/cpptempl.h b/resources/3rdparty/cpptemplate/cpptempl.h index d6ae2ed4f..e3c153979 100644 --- a/resources/3rdparty/cpptemplate/cpptempl.h +++ b/resources/3rdparty/cpptemplate/cpptempl.h @@ -49,6 +49,7 @@ Example: #include #ifndef _MSC_VER #include +#include #else #include #include "windows.h" @@ -122,7 +123,15 @@ namespace cpptempl // convenience functions for recoding utf8 string to wstring and back inline std::wstring utf8_to_wide(const std::string& text) { #ifndef _MSC_VER - return boost::locale::conv::to_utf(text, "UTF-8"); + std::wstring result; + if (sizeof(wchar_t) == 2) { + utf8::utf8to16(text.begin(), text.end(), std::back_inserter(result)); + } else { + assert(sizeof(wchar_t) == 4); + utf8::utf8to32(text.begin(), text.end(), std::back_inserter(result)); + } + return result; + //return boost::locale::conv::to_utf(text, "UTF-8"); #else // Calculate the required length of the buffer const size_t len_needed = ::MultiByteToWideChar(CP_UTF8, 0, text.c_str(), (UINT)(text.length()) , NULL, 0 ); @@ -134,7 +143,15 @@ namespace cpptempl inline std::string wide_to_utf8(const std::wstring& text) { #ifndef _MSC_VER - return boost::locale::conv::from_utf<>(text, "UTF-8"); + std::string result; + if (sizeof(wchar_t) == 2) { + utf8::utf16to8(text.begin(), text.end(), std::back_inserter(result)); + } else { + assert(sizeof(wchar_t) == 4); + utf8::utf32to8(text.begin(), text.end(), std::back_inserter(result)); + } + return result; + //return boost::locale::conv::from_utf<>(text, "UTF-8"); #else const size_t len_needed = ::WideCharToMultiByte(CP_UTF8, 0, text.c_str(), (UINT)(text.length()) , NULL, 0, NULL, NULL) ; boost::scoped_array buff(new char[len_needed+1]) ; diff --git a/resources/3rdparty/include_cpptemplate.cmake b/resources/3rdparty/include_cpptemplate.cmake new file mode 100644 index 000000000..f70c1660d --- /dev/null +++ b/resources/3rdparty/include_cpptemplate.cmake @@ -0,0 +1,19 @@ +ExternalProject_Add( + cpptemplate + DOWNLOAD_COMMAND "" + SOURCE_DIR ${STORM_3RDPARTY_SOURCE_DIR}/cpptemplate + PREFIX ${STORM_3RDPARTY_BINARY_DIR}/cpptemplate + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMAKE_CXX_COMPILER} -std=c++14 -stdlib=libc++ -O3 -I${Boost_INCLUDE_DIRS} -I${STORM_3RDPARTY_SOURCE_DIR}/utf8_v2_3_4/source -shared ${STORM_3RDPARTY_SOURCE_DIR}/cpptemplate/cpptempl.cpp -o ${STORM_3RDPARTY_BINARY_DIR}/cpptemplate/cpptemplate${DYNAMIC_EXT} + INSTALL_COMMAND "" + BUILD_IN_SOURCE 0 + LOG_BUILD ON +) + +set(CPPTEMPLATE_INCLUDE_DIR ${STORM_3RDPARTY_SOURCE_DIR}/cpptemplate) +set(CPPTEMPLATE_SHARED_LIBRARY ${STORM_3RDPARTY_BINARY_DIR}/cpptemplate/cpptemplate${DYNAMIC_EXT}) +list(APPEND STORM_LINK_LIBRARIES ${CPPTEMPLATE_SHARED_LIBRARY}) +add_dependencies(resources cpptemplate) + +message(STATUS "StoRM - Linking with cpptemplate") +include_directories(${CPPTEMPLATE_INCLUDE_DIR}) diff --git a/resources/3rdparty/utf8_v2_3_4/doc/ReleaseNotes b/resources/3rdparty/utf8_v2_3_4/doc/ReleaseNotes new file mode 100644 index 000000000..364411a23 --- /dev/null +++ b/resources/3rdparty/utf8_v2_3_4/doc/ReleaseNotes @@ -0,0 +1,12 @@ +utf8 cpp library +Release 2.3.4 + +A minor bug fix release. Thanks to all who reported bugs. + +Note: Version 2.3.3 contained a regression, and therefore was removed. + +Changes from version 2.3.2 +- Bug fix [39]: checked.h Line 273 and unchecked.h Line 182 have an extra ';' +- Bug fix [36]: replace_invalid() only works with back_inserter + +Files included in the release: utf8.h, core.h, checked.h, unchecked.h, utf8cpp.html, ReleaseNotes diff --git a/resources/3rdparty/utf8_v2_3_4/doc/utf8cpp.html b/resources/3rdparty/utf8_v2_3_4/doc/utf8cpp.html new file mode 100644 index 000000000..6f2aacbe7 --- /dev/null +++ b/resources/3rdparty/utf8_v2_3_4/doc/utf8cpp.html @@ -0,0 +1,1789 @@ + + + + + + + + + UTF8-CPP: UTF-8 with C++ in a Portable Way + + + + +

+ UTF8-CPP: UTF-8 with C++ in a Portable Way +

+

+ The Sourceforge project page +

+ +

+ Introduction +

+

+ Many C++ developers miss an easy and portable way of handling Unicode encoded + strings. The original C++ Standard (known as C++98 or C++03) is Unicode agnostic. + C++11 provides some support for Unicode on core language and library level: + u8, u, and U character and string literals, char16_t and char32_t character types, + u16string and u32string library classes, and codecvt support for conversions + between Unicode encoding forms. + In the meantime, developers use third party libraries like ICU, OS specific capabilities, or simply + roll out their own solutions. +

+

+ In order to easily handle UTF-8 encoded Unicode strings, I came up with a small + generic library. For anybody used to work with STL algorithms and iterators, it should be + easy and natural to use. The code is freely available for any purpose - check out + the license at the beginning of the utf8.h file. If you run into + bugs or performance issues, please let me know and I'll do my best to address them. +

+

+ The purpose of this article is not to offer an introduction to Unicode in general, + and UTF-8 in particular. If you are not familiar with Unicode, be sure to check out + Unicode Home Page or some other source of + information for Unicode. Also, it is not my aim to advocate the use of UTF-8 + encoded strings in C++ programs; if you want to handle UTF-8 encoded strings from + C++, I am sure you have good reasons for it. +

+

+ Examples of use +

+

+ Introductionary Sample +

+

+ To illustrate the use of the library, let's start with a small but complete program + that opens a file containing UTF-8 encoded text, reads it line by line, checks each line + for invalid UTF-8 byte sequences, and converts it to UTF-16 encoding and back to UTF-8: +

+
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+#include "utf8.h"
+using namespace std;
+int main(int argc, char** argv)
+{
+    if (argc != 2) {
+        cout << "\nUsage: docsample filename\n";
+        return 0;
+    }
+
+    const char* test_file_path = argv[1];
+    // Open the test file (contains UTF-8 encoded text)
+    ifstream fs8(test_file_path);
+    if (!fs8.is_open()) {
+    cout << "Could not open " << test_file_path << endl;
+    return 0;
+    }
+
+    unsigned line_count = 1;
+    string line;
+    // Play with all the lines in the file
+    while (getline(fs8, line)) {
+       // check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
+        string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
+        if (end_it != line.end()) {
+            cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
+            cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
+        }
+
+        // Get the line length (at least for the valid part)
+        int length = utf8::distance(line.begin(), end_it);
+        cout << "Length of line " << line_count << " is " << length <<  "\n";
+
+        // Convert it to utf-16
+        vector<unsigned short> utf16line;
+        utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
+
+        // And back to utf-8
+        string utf8line; 
+        utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
+
+        // Confirm that the conversion went OK:
+        if (utf8line != string(line.begin(), end_it))
+            cout << "Error in UTF-16 conversion at line: " << line_count << "\n";        
+
+        line_count++;
+    }
+    return 0;
+}
+
+

+ In the previous code sample, for each line we performed + a detection of invalid UTF-8 sequences with find_invalid; the number + of characters (more precisely - the number of Unicode code points, including the end + of line and even BOM if there is one) in each line was + determined with a use of utf8::distance; finally, we have converted + each line to UTF-16 encoding with utf8to16 and back to UTF-8 with + utf16to8. +

+

Checking if a file contains valid UTF-8 text

+

+Here is a function that checks whether the content of a file is valid UTF-8 encoded text without +reading the content into the memory: +

+
    
+bool valid_utf8_file(iconst char* file_name)
+{
+    ifstream ifs(file_name);
+    if (!ifs)
+        return false; // even better, throw here
+
+    istreambuf_iterator<char> it(ifs.rdbuf());
+    istreambuf_iterator<char> eos;
+
+    return utf8::is_valid(it, eos);
+}
+
+

+Because the function utf8::is_valid() works with input iterators, we were able +to pass an istreambuf_iterator to it and read the content of the file directly +without loading it to the memory first.

+

+Note that other functions that take input iterator arguments can be used in a similar way. For +instance, to read the content of a UTF-8 encoded text file and convert the text to UTF-16, just +do something like: +

+
+    utf8::utf8to16(it, eos, back_inserter(u16string));
+
+

Ensure that a string contains valid UTF-8 text

+

+If we have some text that "probably" contains UTF-8 encoded text and we want to +replace any invalid UTF-8 sequence with a replacement character, something like +the following function may be used: +

+
+void fix_utf8_string(std::string& str)
+{
+    std::string temp;
+    utf8::replace_invalid(str.begin(), str.end(), back_inserter(temp));
+    str = temp;
+}
+
+

The function will replace any invalid UTF-8 sequence with a Unicode replacement character. +There is an overloaded function that enables the caller to supply their own replacement character. +

+

+ Reference +

+

+ Functions From utf8 Namespace +

+

+ utf8::append +

+

+ Available in version 1.0 and later. +

+

+ Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequence + to a UTF-8 string. +

+
+template <typename octet_iterator>
+octet_iterator append(uint32_t cp, octet_iterator result);
+   
+
+

+ octet_iterator: an output iterator.
+ cp: a 32 bit integer representing a code point to append to the + sequence.
+ result: an output iterator to the place in the sequence where to + append the code point.
+ Return value: an iterator pointing to the place + after the newly appended sequence. +

+

+ Example of use: +

+
+unsigned char u[5] = {0,0,0,0,0};
+unsigned char* end = append(0x0448, u);
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+

+ Note that append does not allocate any memory - it is the burden of + the caller to make sure there is enough memory allocated for the operation. To make + things more interesting, append can add anywhere between 1 and 4 + octets to the sequence. In practice, you would most often want to use + std::back_inserter to ensure that the necessary memory is allocated. +

+

+ In case of an invalid code point, a utf8::invalid_code_point exception + is thrown. +

+

+ utf8::next +

+

+ Available in version 1.0 and later. +

+

+ Given the iterator to the beginning of the UTF-8 sequence, it returns the code + point and moves the iterator to the next position. +

+
+template <typename octet_iterator> 
+uint32_t next(octet_iterator& it, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + beginning of the next code point.
+ end: end of the UTF-8 sequence to be processed. If it + gets equal to end during the extraction of a code point, an + utf8::not_enough_room exception is thrown.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = next(w, twochars + 6);
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+
+

+ This function is typically used to iterate through a UTF-8 encoded string. +

+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. +

+

+ utf8::peek_next +

+

+ Available in version 2.1 and later. +

+

+ Given the iterator to the beginning of the UTF-8 sequence, it returns the code + point for the following sequence without changing the value of the iterator. +

+
+template <typename octet_iterator> 
+uint32_t peek_next(octet_iterator it, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ it: an iterator pointing to the beginning of an UTF-8 + encoded code point.
+ end: end of the UTF-8 sequence to be processed. If it + gets equal to end during the extraction of a code point, an + utf8::not_enough_room exception is thrown.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = peek_next(w, twochars + 6);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. +

+

+ utf8::prior +

+

+ Available in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 sequence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator> 
+uint32_t prior(octet_iterator& it, octet_iterator start);
+   
+
+

+ octet_iterator: a bidirectional iterator.
+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ start: an iterator to the beginning of the sequence where the search + for the beginning of a code point is performed. It is a + safety measure to prevent passing the beginning of the string in the search for a + UTF-8 lead octet.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+unsigned char* w = twochars + 3;
+int cp = prior (w, twochars);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ This function has two purposes: one is two iterate backwards through a UTF-8 + encoded string. Note that it is usually a better idea to iterate forward instead, + since utf8::next is faster. The second purpose is to find a beginning + of a UTF-8 sequence if we have a random position within a string. Note that in that + case utf8::prior may not detect an invalid UTF-8 sequence in some scenarios: + for instance if there are superfluous trail octets, it will just skip them. +

+

+ it will typically point to the beginning of + a code point, and start will point to the + beginning of the string to ensure we don't go backwards too far. it is + decreased until it points to a lead UTF-8 octet, and then the UTF-8 sequence + beginning with that octet is decoded to a 32 bit representation and returned. +

+

+ In case start is reached before a UTF-8 lead octet is hit, or if an + invalid UTF-8 sequence is started by the lead octet, an invalid_utf8 + exception is thrown. +

+

In case start equals it, a not_enough_room + exception is thrown. +

+ utf8::previous +

+

+ Deprecated in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator> 
+uint32_t previous(octet_iterator& it, octet_iterator pass_start);
+   
+
+

+ octet_iterator: a random access iterator.
+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ pass_start: an iterator to the point in the sequence where the search + for the beginning of a code point is aborted if no result was reached. It is a + safety measure to prevent passing the beginning of the string in the search for a + UTF-8 lead octet.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+unsigned char* w = twochars + 3;
+int cp = previous (w, twochars - 1);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ utf8::previous is deprecated, and utf8::prior should + be used instead, although the existing code can continue using this function. + The problem is the parameter pass_start that points to the position + just before the beginning of the sequence. Standard containers don't have the + concept of "pass start" and the function can not be used with their iterators. +

+

+ it will typically point to the beginning of + a code point, and pass_start will point to the octet just before the + beginning of the string to ensure we don't go backwards too far. it is + decreased until it points to a lead UTF-8 octet, and then the UTF-8 sequence + beginning with that octet is decoded to a 32 bit representation and returned. +

+

+ In case pass_start is reached before a UTF-8 lead octet is hit, or if an + invalid UTF-8 sequence is started by the lead octet, an invalid_utf8 + exception is thrown +

+

+ utf8::advance +

+

+ Available in version 1.0 and later. +

+

+ Advances an iterator by the specified number of code points within an UTF-8 + sequence. +

+
+template <typename octet_iterator, typename distance_type> 
+void advance (octet_iterator& it, distance_type n, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ distance_type: an integral type convertible to octet_iterator's difference type.
+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + nth following code point.
+ n: a positive integer that shows how many code points we want to + advance.
+ end: end of the UTF-8 sequence to be processed. If it + gets equal to end during the extraction of a code point, an + utf8::not_enough_room exception is thrown.
+

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+unsigned char* w = twochars;
+advance (w, 2, twochars + 6);
+assert (w == twochars + 5);
+
+

+ This function works only "forward". In case of a negative n, there is + no effect. +

+

+ In case of an invalid code point, a utf8::invalid_code_point exception + is thrown. +

+

+ utf8::distance +

+

+ Available in version 1.0 and later. +

+

+ Given the iterators to two UTF-8 encoded code points in a seqence, returns the + number of code points between them. +

+
+template <typename octet_iterator> 
+typename std::iterator_traits<octet_iterator>::difference_type distance (octet_iterator first, octet_iterator last);
+   
+
+

+ octet_iterator: an input iterator.
+ first: an iterator to a beginning of a UTF-8 encoded code point.
+ last: an iterator to a "post-end" of the last UTF-8 encoded code + point in the sequence we are trying to determine the length. It can be the + beginning of a new code point, or not.
+ Return value the distance between the iterators, + in code points. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+size_t dist = utf8::distance(twochars, twochars + 5);
+assert (dist == 2);
+
+

+ This function is used to find the length (in code points) of a UTF-8 encoded + string. The reason it is called distance, rather than, say, + length is mainly because developers are used that length is an + O(1) function. Computing the length of an UTF-8 string is a linear operation, and + it looked better to model it after std::distance algorithm. +

+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. If last does not point to the past-of-end of a UTF-8 seqence, + a utf8::not_enough_room exception is thrown. +

+

+ utf8::utf16to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-16 encoded string to UTF-8. +

+
+template <typename u16bit_iterator, typename octet_iterator>
+octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result);
+   
+
+

+ u16bit_iterator: an input iterator.
+ octet_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-16 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-16 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+assert (utf8result.size() == 10);    
+
+

+ In case of invalid UTF-16 sequence, a utf8::invalid_utf16 exception is + thrown. +

+

+ utf8::utf8to16 +

+

+ Available in version 1.0 and later. +

+

+ Converts an UTF-8 encoded string to UTF-16 +

+
+template <typename u16bit_iterator, typename octet_iterator>
+u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result);
+   
+
+

+ octet_iterator: an input iterator.
+ u16bit_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert. < br /> end: an iterator pointing to + pass-the-end of the UTF-8 encoded string to convert.
+ result: an output iterator to the place in the UTF-16 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-16 string. +

+

+ Example of use: +

+
+char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
+vector <unsigned short> utf16result;
+utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+
+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. If end does not point to the past-of-end of a UTF-8 seqence, a + utf8::not_enough_room exception is thrown. +

+

+ utf8::utf32to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-32 encoded string to UTF-8. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result);
+   
+
+

+ octet_iterator: an output iterator.
+ u32bit_iterator: an input iterator.
+ start: an iterator pointing to the beginning of the UTF-32 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-32 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+vector<unsigned char> utf8result;
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+assert (utf8result.size() == 9);
+
+

+ In case of invalid UTF-32 string, a utf8::invalid_code_point exception + is thrown. +

+

+ utf8::utf8to32 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-8 encoded string to UTF-32. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result);
+   
+
+

+ octet_iterator: an input iterator.
+ u32bit_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-8 encoded string + to convert.
+ result: an output iterator to the place in the UTF-32 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-32 string. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+vector<int> utf32result;
+utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+assert (utf32result.size() == 2);
+
+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. If end does not point to the past-of-end of a UTF-8 seqence, a + utf8::not_enough_room exception is thrown. +

+

+ utf8::find_invalid +

+

+ Available in version 1.0 and later. +

+

+ Detects an invalid sequence within a UTF-8 string. +

+
+template <typename octet_iterator> 
+octet_iterator find_invalid(octet_iterator start, octet_iterator end);
+
+

+ octet_iterator: an input iterator.
+ start: an iterator pointing to the beginning of the UTF-8 string to + test for validity.
+ end: an iterator pointing to pass-the-end of the UTF-8 string to test + for validity.
+ Return value: an iterator pointing to the first + invalid octet in the UTF-8 string. In case none were found, equals + end. +

+

+ Example of use: +

+
+char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
+char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
+assert (invalid == utf_invalid + 5);
+
+

+ This function is typically used to make sure a UTF-8 string is valid before + processing it with other functions. It is especially important to call it if before + doing any of the unchecked operations on it. +

+

+ utf8::is_valid +

+

+ Available in version 1.0 and later. +

+

+ Checks whether a sequence of octets is a valid UTF-8 string. +

+
+template <typename octet_iterator> 
+bool is_valid(octet_iterator start, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ start: an iterator pointing to the beginning of the UTF-8 string to + test for validity.
+ end: an iterator pointing to pass-the-end of the UTF-8 string to test + for validity.
+ Return value: true if the sequence + is a valid UTF-8 string; false if not. +

+ Example of use: +
+char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
+bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
+assert (bvalid == false);
+
+

+ is_valid is a shorthand for find_invalid(start, end) == + end;. You may want to use it to make sure that a byte seqence is a valid + UTF-8 string without the need to know where it fails if it is not valid. +

+

+ utf8::replace_invalid +

+

+ Available in version 2.0 and later. +

+

+ Replaces all invalid UTF-8 sequences within a string with a replacement marker. +

+
+template <typename octet_iterator, typename output_iterator>
+output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement);
+template <typename octet_iterator, typename output_iterator>
+output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out);
+   
+
+

+ octet_iterator: an input iterator.
+ output_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-8 string to + look for invalid UTF-8 sequences.
+ end: an iterator pointing to pass-the-end of the UTF-8 string to look + for invalid UTF-8 sequences.
+ out: An output iterator to the range where the result of replacement + is stored.
+ replacement: A Unicode code point for the replacement marker. The + version without this parameter assumes the value 0xfffd
+ Return value: An iterator pointing to the place + after the UTF-8 string with replaced invalid sequences. +

+

+ Example of use: +

+
+char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
+vector<char> replace_invalid_result;
+replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), back_inserter(replace_invalid_result), '?');
+bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
+assert (bvalid);
+char* fixed_invalid_sequence = "a????z";
+assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.end(), fixed_invalid_sequence));
+
+

+ replace_invalid does not perform in-place replacement of invalid + sequences. Rather, it produces a copy of the original string with the invalid + sequences replaced with a replacement marker. Therefore, out must not + be in the [start, end] range. +

+

+ If end does not point to the past-of-end of a UTF-8 sequence, a + utf8::not_enough_room exception is thrown. +

+

+ utf8::starts_with_bom +

+

+ Available in version 2.3 and later. Relaces deprecated is_bom() function. +

+

+ Checks whether an octet sequence starts with a UTF-8 byte order mark (BOM) +

+
+template <typename octet_iterator> 
+bool starts_with_bom (octet_iterator it, octet_iterator end);
+
+

+ octet_iterator: an input iterator.
+ it: beginning of the octet sequence to check
+ end: pass-end of the sequence to check
+ Return value: true if the sequence + starts with a UTF-8 byte order mark; false if not. +

+

+ Example of use: +

+
+unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
+assert (bbom == true);
+
+

+ The typical use of this function is to check the first three bytes of a file. If + they form the UTF-8 BOM, we want to skip them before processing the actual UTF-8 + encoded text. +

+

+ utf8::is_bom +

+

+ Available in version 1.0 and later. Deprecated in version 2.3. starts_with_bom() should be used + instead. +

+

+ Checks whether a sequence of three octets is a UTF-8 byte order mark (BOM) +

+
+template <typename octet_iterator> 
+bool is_bom (octet_iterator it);  // Deprecated
+
+

+ octet_iterator: an input iterator.
+ it: beginning of the 3-octet sequence to check
+ Return value: true if the sequence + is UTF-8 byte order mark; false if not. +

+

+ Example of use: +

+
+unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+bool bbom = is_bom(byte_order_mark);
+assert (bbom == true);
+
+

+ The typical use of this function is to check the first three bytes of a file. If + they form the UTF-8 BOM, we want to skip them before processing the actual UTF-8 + encoded text. +

+

+ If a sequence is + shorter than three bytes, an invalid iterator will be dereferenced. Therefore, this function is deprecated + in favor of starts_with_bom()that takes the end of sequence as an argument. +

+

+ Types From utf8 Namespace +

+

utf8::exception +

+

+ Available in version 2.3 and later. +

+

+ Base class for the exceptions thrown by UTF CPP library functions. +

+
+class exception : public std::exception {};
+
+

+ Example of use: +

+
+try {
+  code_that_uses_utf_cpp_library();
+}
+catch(const utf8::exception& utfcpp_ex) {
+  cerr << utfcpp_ex.what();
+}
+
+ +

utf8::invalid_code_point +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP functions such as advance and next if an UTF-8 sequence represents and invalid code point. +

+ +
+class invalid_code_point : public exception {
+public: 
+    uint32_t code_point() const;
+};
+
+
+

+ Member function code_point() can be used to determine the invalid code point that + caused the exception to be thrown. +

+

utf8::invalid_utf8 +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP functions such as next and prior if an invalid UTF-8 sequence + is detected during decoding. +

+ +
+class invalid_utf8 : public exception {
+public: 
+    uint8_t utf8_octet() const;
+};
+
+ +

+ Member function utf8_octet() can be used to determine the beginning of the byte + sequence that caused the exception to be thrown. +

+ +

utf8::invalid_utf16 +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP function utf16to8 if an invalid UTF-16 sequence + is detected during decoding. +

+ +
+class invalid_utf16 : public exception {
+public: 
+    uint16_t utf16_word() const;
+};
+
+ +

+ Member function utf16_word() can be used to determine the UTF-16 code unit + that caused the exception to be thrown. +

+

utf8::not_enough_room +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP functions such as next if the end of the decoded UTF-8 sequence + was reached before the code point was decoded. +

+ +
+class not_enough_room : public exception {};
+
+

+ utf8::iterator +

+

+ Available in version 2.0 and later. +

+

+ Adapts the underlying octet iterator to iterate over the sequence of code points, + rather than raw octets. +

+
+template <typename octet_iterator>
+class iterator;
+
+ +
Member functions
+
+
iterator();
the deafult constructor; the underlying octet_iterator is + constructed with its default constructor. +
explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end);
a constructor + that initializes the underlying octet_iterator with octet_it + and sets the range in which the iterator is considered valid. +
octet_iterator base () const;
returns the + underlying octet_iterator. +
uint32_t operator * () const;
decodes the utf-8 sequence + the underlying octet_iterator is pointing to and returns the code point. +
bool operator == (const iterator& rhs) + const;
returns true + if the two underlaying iterators are equal. +
bool operator != (const iterator& rhs) + const;
returns true + if the two underlaying iterators are not equal. +
iterator& operator ++ ();
the prefix increment - moves + the iterator to the next UTF-8 encoded code point. +
iterator operator ++ (int);
+ the postfix increment - moves the iterator to the next UTF-8 encoded code point and returns the current one. +
iterator& operator -- ();
the prefix decrement - moves + the iterator to the previous UTF-8 encoded code point. +
iterator operator -- (int);
+ the postfix decrement - moves the iterator to the previous UTF-8 encoded code point and returns the current one. +
+

+ Example of use: +

+
+char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
+utf8::iterator<char*> it(threechars, threechars, threechars + 9);
+utf8::iterator<char*> it2 = it;
+assert (it2 == it);
+assert (*it == 0x10346);
+assert (*(++it) == 0x65e5);
+assert ((*it++) == 0x65e5);
+assert (*it == 0x0448);
+assert (it != it2);
+utf8::iterator<char*> endit (threechars + 9, threechars, threechars + 9);  
+assert (++it == endit);
+assert (*(--it) == 0x0448);
+assert ((*it--) == 0x0448);
+assert (*it == 0x65e5);
+assert (--it == utf8::iterator<char*>(threechars, threechars, threechars + 9));
+assert (*it == 0x10346);
+
+

+ The purpose of utf8::iterator adapter is to enable easy iteration as well as the use of STL + algorithms with UTF-8 encoded strings. Increment and decrement operators are implemented in terms of + utf8::next() and utf8::prior() functions. +

+

+ Note that utf8::iterator adapter is a checked iterator. It operates on the range specified in + the constructor; any attempt to go out of that range will result in an exception. Even the comparison operators + require both iterator object to be constructed against the same range - otherwise an exception is thrown. Typically, + the range will be determined by sequence container functions begin and end, i.e.: +

+
+std::string s = "example";
+utf8::iterator i (s.begin(), s.begin(), s.end());
+
+

+ Functions From utf8::unchecked Namespace +

+

+ utf8::unchecked::append +

+

+ Available in version 1.0 and later. +

+

+ Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequence + to a UTF-8 string. +

+
+template <typename octet_iterator>
+octet_iterator append(uint32_t cp, octet_iterator result);
+   
+
+

+ cp: A 32 bit integer representing a code point to append to the + sequence.
+ result: An output iterator to the place in the sequence where to + append the code point.
+ Return value: An iterator pointing to the place + after the newly appended sequence. +

+

+ Example of use: +

+
+unsigned char u[5] = {0,0,0,0,0};
+unsigned char* end = unchecked::append(0x0448, u);
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+

+ This is a faster but less safe version of utf8::append. It does not + check for validity of the supplied code point, and may produce an invalid UTF-8 + sequence. +

+

+ utf8::unchecked::next +

+

+ Available in version 1.0 and later. +

+

+ Given the iterator to the beginning of a UTF-8 sequence, it returns the code point + and moves the iterator to the next position. +

+
+template <typename octet_iterator>
+uint32_t next(octet_iterator& it);
+   
+
+

+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + beginning of the next code point.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = unchecked::next(w);
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+
+

+ This is a faster but less safe version of utf8::next. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::peek_next +

+

+ Available in version 2.1 and later. +

+

+ Given the iterator to the beginning of a UTF-8 sequence, it returns the code point. +

+
+template <typename octet_iterator>
+uint32_t peek_next(octet_iterator it);
+   
+
+

+ it: an iterator pointing to the beginning of an UTF-8 + encoded code point.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = unchecked::peek_next(w);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ This is a faster but less safe version of utf8::peek_next. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::prior +

+

+ Available in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator>
+uint32_t prior(octet_iterator& it);
+   
+
+

+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars + 3;
+int cp = unchecked::prior (w);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ This is a faster but less safe version of utf8::prior. It does not + check for validity of the supplied UTF-8 sequence and offers no boundary checking. +

+

+ utf8::unchecked::previous (deprecated, see utf8::unchecked::prior) +

+

+ Deprecated in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator>
+uint32_t previous(octet_iterator& it);
+   
+
+

+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars + 3;
+int cp = unchecked::previous (w);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ The reason this function is deprecated is just the consistency with the "checked" + versions, where prior should be used instead of previous. + In fact, unchecked::previous behaves exactly the same as + unchecked::prior +

+

+ This is a faster but less safe version of utf8::previous. It does not + check for validity of the supplied UTF-8 sequence and offers no boundary checking. +

+

+ utf8::unchecked::advance +

+

+ Available in version 1.0 and later. +

+

+ Advances an iterator by the specified number of code points within an UTF-8 + sequence. +

+
+template <typename octet_iterator, typename distance_type>
+void advance (octet_iterator& it, distance_type n);
+   
+
+

+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + nth following code point.
+ n: a positive integer that shows how many code points we want to + advance.
+

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+unchecked::advance (w, 2);
+assert (w == twochars + 5);
+
+

+ This function works only "forward". In case of a negative n, there is + no effect. +

+

+ This is a faster but less safe version of utf8::advance. It does not + check for validity of the supplied UTF-8 sequence and offers no boundary checking. +

+

+ utf8::unchecked::distance +

+

+ Available in version 1.0 and later. +

+

+ Given the iterators to two UTF-8 encoded code points in a seqence, returns the + number of code points between them. +

+
+template <typename octet_iterator>
+typename std::iterator_traits<octet_iterator>::difference_type distance (octet_iterator first, octet_iterator last);
+
+

+ first: an iterator to a beginning of a UTF-8 encoded code point.
+ last: an iterator to a "post-end" of the last UTF-8 encoded code + point in the sequence we are trying to determine the length. It can be the + beginning of a new code point, or not.
+ Return value the distance between the iterators, + in code points. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+size_t dist = utf8::unchecked::distance(twochars, twochars + 5);
+assert (dist == 2);
+
+

+ This is a faster but less safe version of utf8::distance. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::utf16to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-16 encoded string to UTF-8. +

+
+template <typename u16bit_iterator, typename octet_iterator>
+octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-16 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-16 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+assert (utf8result.size() == 10);    
+
+

+ This is a faster but less safe version of utf8::utf16to8. It does not + check for validity of the supplied UTF-16 sequence. +

+

+ utf8::unchecked::utf8to16 +

+

+ Available in version 1.0 and later. +

+

+ Converts an UTF-8 encoded string to UTF-16 +

+
+template <typename u16bit_iterator, typename octet_iterator>
+u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert. < br /> end: an iterator pointing to + pass-the-end of the UTF-8 encoded string to convert.
+ result: an output iterator to the place in the UTF-16 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-16 string. +

+

+ Example of use: +

+
+char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
+vector <unsigned short> utf16result;
+unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+
+

+ This is a faster but less safe version of utf8::utf8to16. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::utf32to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-32 encoded string to UTF-8. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-32 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-32 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+int utf32string[] = {0x448, 0x65e5, 0x10346, 0};
+vector<unsigned char> utf8result;
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+assert (utf8result.size() == 9);
+
+

+ This is a faster but less safe version of utf8::utf32to8. It does not + check for validity of the supplied UTF-32 sequence. +

+

+ utf8::unchecked::utf8to32 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-8 encoded string to UTF-32. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-8 encoded string + to convert.
+ result: an output iterator to the place in the UTF-32 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-32 string. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+vector<int> utf32result;
+unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+assert (utf32result.size() == 2);
+
+

+ This is a faster but less safe version of utf8::utf8to32. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ Types From utf8::unchecked Namespace +

+

+ utf8::iterator +

+

+ Available in version 2.0 and later. +

+

+ Adapts the underlying octet iterator to iterate over the sequence of code points, + rather than raw octets. +

+
+template <typename octet_iterator>
+class iterator;
+
+ +
Member functions
+
+
iterator();
the deafult constructor; the underlying octet_iterator is + constructed with its default constructor. +
explicit iterator (const octet_iterator& octet_it); +
a constructor + that initializes the underlying octet_iterator with octet_it +
octet_iterator base () const;
returns the + underlying octet_iterator. +
uint32_t operator * () const;
decodes the utf-8 sequence + the underlying octet_iterator is pointing to and returns the code point. +
bool operator == (const iterator& rhs) + const;
returns true + if the two underlaying iterators are equal. +
bool operator != (const iterator& rhs) + const;
returns true + if the two underlaying iterators are not equal. +
iterator& operator ++ ();
the prefix increment - moves + the iterator to the next UTF-8 encoded code point. +
iterator operator ++ (int);
+ the postfix increment - moves the iterator to the next UTF-8 encoded code point and returns the current one. +
iterator& operator -- ();
the prefix decrement - moves + the iterator to the previous UTF-8 encoded code point. +
iterator operator -- (int);
+ the postfix decrement - moves the iterator to the previous UTF-8 encoded code point and returns the current one. +
+

+ Example of use: +

+
+char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
+utf8::unchecked::iterator<char*> un_it(threechars);
+utf8::unchecked::iterator<char*> un_it2 = un_it;
+assert (un_it2 == un_it);
+assert (*un_it == 0x10346);
+assert (*(++un_it) == 0x65e5);
+assert ((*un_it++) == 0x65e5);
+assert (*un_it == 0x0448);
+assert (un_it != un_it2);
+utf8::::unchecked::iterator<char*> un_endit (threechars + 9);  
+assert (++un_it == un_endit);
+assert (*(--un_it) == 0x0448);
+assert ((*un_it--) == 0x0448);
+assert (*un_it == 0x65e5);
+assert (--un_it == utf8::unchecked::iterator<char*>(threechars));
+assert (*un_it == 0x10346);
+
+

+ This is an unchecked version of utf8::iterator. It is faster in many cases, but offers + no validity or range checks. +

+

+ Points of interest +

+

+ Design goals and decisions +

+

+ The library was designed to be: +

+
    +
  1. + Generic: for better or worse, there are many C++ string classes out there, and + the library should work with as many of them as possible. +
    2. +
  2. + Portable: the library should be portable both accross different platforms and + compilers. The only non-portable code is a small section that declares unsigned + integers of different sizes: three typedefs. They can be changed by the users of + the library if they don't match their platform. The default setting should work + for Windows (both 32 and 64 bit), and most 32 bit and 64 bit Unix derivatives. +
    3. +
  3. + Lightweight: follow the "pay only for what you use" guideline. +
    4. +
  4. + Unintrusive: avoid forcing any particular design or even programming style on the + user. This is a library, not a framework. +
    5. +
+

+ Alternatives +

+

+ In case you want to look into other means of working with UTF-8 strings from C++, + here is the list of solutions I am aware of: +

+
    +
  1. + ICU Library. It is very powerful, + complete, feature-rich, mature, and widely used. Also big, intrusive, + non-generic, and doesn't play well with the Standard Library. I definitelly + recommend looking at ICU even if you don't plan to use it. +
    2. +
  2. + C++11 language and library features. Still far from complete, and not widely + supported by compiler vendors. +
    3. +
  3. + Glib::ustring. + A class specifically made to work with UTF-8 strings, and also feel like + std::string. If you prefer to have yet another string class in your + code, it may be worth a look. Be aware of the licensing issues, though. +
    4. +
  4. + Platform dependent solutions: Windows and POSIX have functions to convert strings + from one encoding to another. That is only a subset of what my library offers, + but if that is all you need it may be good enough. +
    5. +
+ +
    +
  1. + The Unicode Consortium. +
    2. +
  2. + ICU Library. +
    3. +
  3. + UTF-8 at Wikipedia +
    4. +
  4. + UTF-8 and Unicode FAQ for + Unix/Linux +
    5. +
+ + diff --git a/resources/3rdparty/utf8_v2_3_4/source/utf8.h b/resources/3rdparty/utf8_v2_3_4/source/utf8.h new file mode 100644 index 000000000..4e4451403 --- /dev/null +++ b/resources/3rdparty/utf8_v2_3_4/source/utf8.h @@ -0,0 +1,34 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#endif // header guard diff --git a/resources/3rdparty/utf8_v2_3_4/source/utf8/checked.h b/resources/3rdparty/utf8_v2_3_4/source/utf8/checked.h new file mode 100644 index 000000000..133115513 --- /dev/null +++ b/resources/3rdparty/utf8_v2_3_4/source/utf8/checked.h @@ -0,0 +1,327 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + /// Deprecated in versions that include "prior" + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return utf8::next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start != end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end) : + it(octet_it), range_start(range_start), range_end(range_end) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + + diff --git a/resources/3rdparty/utf8_v2_3_4/source/utf8/core.h b/resources/3rdparty/utf8_v2_3_4/source/utf8/core.h new file mode 100644 index 000000000..693d388c0 --- /dev/null +++ b/resources/3rdparty/utf8_v2_3_4/source/utf8/core.h @@ -0,0 +1,329 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } + + //Deprecated in release 2.3 + template + inline bool is_bom (octet_iterator it) + { + return ( + (utf8::internal::mask8(*it++)) == bom[0] && + (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2] + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/resources/3rdparty/utf8_v2_3_4/source/utf8/unchecked.h b/resources/3rdparty/utf8_v2_3_4/source/utf8/unchecked.h new file mode 100644 index 000000000..cb2427166 --- /dev/null +++ b/resources/3rdparty/utf8_v2_3_4/source/utf8/unchecked.h @@ -0,0 +1,228 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) + template + inline uint32_t previous(octet_iterator& it) + { + return utf8::unchecked::prior(it); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + diff --git a/src/builder/ExplicitJitJaniModelBuilder.cpp b/src/builder/ExplicitJitJaniModelBuilder.cpp new file mode 100644 index 000000000..fc042874e --- /dev/null +++ b/src/builder/ExplicitJitJaniModelBuilder.cpp @@ -0,0 +1,165 @@ +#include "src/builder/ExplicitJitJaniModelBuilder.h" + +#include +#include + +#include "cpptempl.h" + +#include "src/utility/macros.h" +#include "src/exceptions/InvalidStateException.h" + +namespace storm { + namespace builder { + + static const std::string CXX_COMPILER = "clang++"; + static const std::string DYLIB_EXTENSION = ".dylib"; + static const std::string COMPILER_FLAGS = "-std=c++11 -stdlib=libc++ -fPIC -O3 -shared"; + static const std::string STORM_ROOT = "/Users/chris/work/storm2"; + static const std::string BOOST_ROOT = "/usr/local/Cellar/boost/1.61.0_1/include"; + + template + ExplicitJitJaniModelBuilder::ExplicitJitJaniModelBuilder(storm::jani::Model const& model) : model(model) { + // Intentionally left empty. + } + + template + std::string ExplicitJitJaniModelBuilder::createSourceCode() { + std::string sourceTemplate = R"( + + #include + #include + #include + + #include "src/builder/JitModelBuilderInterface.h" + #include "resources/3rdparty/sparsepp/sparsepp.h" + + namespace storm { + namespace builder { + + struct StateType { + {% for stateVariable in stateVariables %}int64_t {$stateVariable.name} : {$stateVariable.bitwidth};{% endfor %} + }; + + class JitBuilder : public JitModelBuilderInterface { + public: + JitBuilder() { + // Intentionally left empty. + } + + virtual void build() override { + std::cout << "building in progress" << std::endl; + } + + static JitModelBuilderInterface* create() { + return new JitBuilder(); + } + + private: + spp::sparse_hash_map stateIds; + }; + + BOOST_DLL_ALIAS(storm::builder::JitBuilder::create, create_builder) + + } + } + )"; + + cpptempl::data_map modelData; + return cpptempl::parse(sourceTemplate, modelData); + } + + template + boost::optional ExplicitJitJaniModelBuilder::execute(std::string command) { + char buffer[128]; + std::stringstream output; + command += " 2>&1"; + + std::cout << "executing " << command << std::endl; + + std::unique_ptr pipe(popen(command.c_str(), "r")); + STORM_LOG_THROW(pipe, storm::exceptions::InvalidStateException, "Call to popen failed."); + + while (!feof(pipe.get())) { + if (fgets(buffer, 128, pipe.get()) != nullptr) + output << buffer; + } + int result = pclose(pipe.get()); + pipe.release(); + + if (WEXITSTATUS(result) == 0) { + return boost::none; + } else { + return "Executing command failed. Got response: " + output.str(); + } + } + + template + void ExplicitJitJaniModelBuilder::createBuilder(boost::filesystem::path const& dynamicLibraryPath) { + jitBuilderGetFunction = boost::dll::import_alias::CreateFunctionType>(dynamicLibraryPath, "create_builder"); + builder = std::unique_ptr>(jitBuilderGetFunction()); + } + + template + boost::filesystem::path ExplicitJitJaniModelBuilder::writeSourceToTemporaryFile(std::string const& source) { + boost::filesystem::path temporaryFile = boost::filesystem::unique_path("%%%%-%%%%-%%%%-%%%%.cpp"); + std::ofstream out(temporaryFile.native()); + out << source << std::endl; + out.close(); + return temporaryFile; + } + + template + boost::filesystem::path ExplicitJitJaniModelBuilder::compileSourceToSharedLibrary(boost::filesystem::path const& sourceFile) { + std::string sourceFilename = boost::filesystem::absolute(sourceFile).string(); + auto dynamicLibraryPath = sourceFile; + dynamicLibraryPath += DYLIB_EXTENSION; + std::string dynamicLibraryFilename = boost::filesystem::absolute(dynamicLibraryPath).string(); + + std::string command = CXX_COMPILER + " " + sourceFilename + " " + COMPILER_FLAGS + " -I" + STORM_ROOT + " -I" + BOOST_ROOT + " -o " + dynamicLibraryFilename; + boost::optional error = execute(command); + + if (error) { + boost::filesystem::remove(sourceFile); + STORM_LOG_THROW(false, storm::exceptions::InvalidStateException, "Compiling shared library failed. Error: " << error.get()); + } + + return dynamicLibraryPath; + } + + template + std::shared_ptr>> ExplicitJitJaniModelBuilder::build() { + + // (1) generate the source code of the shared library + std::string source = createSourceCode(); + std::cout << "created source code: " << source << std::endl; + + // (2) write the source code to a temporary file + boost::filesystem::path temporarySourceFile = writeSourceToTemporaryFile(source); + std::cout << "wrote source to file " << temporarySourceFile.native() << std::endl; + + // (3) compile the shared library + boost::filesystem::path dynamicLibraryPath = compileSourceToSharedLibrary(temporarySourceFile); + std::cout << "successfully compiled shared library" << std::endl; + + // (4) remove the source code we just compiled + boost::filesystem::remove(temporarySourceFile); + + // (5) create the loader from the shared library + createBuilder(dynamicLibraryPath); + + // (6) execute the function in the shared lib + builder->build(); + + // (7) use result to build the model + + // (8) delete the shared library + boost::filesystem::remove(dynamicLibraryPath); + + // FIXME + return nullptr; + } + + template class ExplicitJitJaniModelBuilder; + + } +} diff --git a/src/builder/ExplicitJitJaniModelBuilder.h b/src/builder/ExplicitJitJaniModelBuilder.h new file mode 100644 index 000000000..dbb28fd18 --- /dev/null +++ b/src/builder/ExplicitJitJaniModelBuilder.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include +#include +#include + +#include "src/storage/jani/Model.h" + +#include "src/builder/JitModelBuilderInterface.h" + +namespace storm { + namespace models { + namespace sparse { + template + class Model; + + template + class StandardRewardModel; + } + } + + namespace builder { + + template + class ExplicitJitJaniModelBuilder { + public: + typedef JitModelBuilderInterface* (CreateFunctionType)(); + typedef boost::function ImportFunctionType; + + ExplicitJitJaniModelBuilder(storm::jani::Model const& model); + + std::shared_ptr>> build(); + + private: + void createBuilder(boost::filesystem::path const& dynamicLibraryPath); + std::string createSourceCode(); + boost::filesystem::path writeSourceToTemporaryFile(std::string const& source); + boost::filesystem::path compileSourceToSharedLibrary(boost::filesystem::path const& sourceFile); + + static boost::optional execute(std::string command); + + storm::jani::Model const& model; + typename ExplicitJitJaniModelBuilder::ImportFunctionType jitBuilderGetFunction; + std::unique_ptr> builder; + }; + + } +} diff --git a/src/builder/JitModelBuilderInterface.h b/src/builder/JitModelBuilderInterface.h new file mode 100644 index 000000000..d88dcfc82 --- /dev/null +++ b/src/builder/JitModelBuilderInterface.h @@ -0,0 +1,19 @@ +#pragma once + +namespace storm { + namespace builder { + + template + class JitModelBuilderInterface { + public: + virtual ~JitModelBuilderInterface() { + // Intentionally left empty. + } + + virtual void build() { + // Intentionally left empty. + } + }; + + } +} diff --git a/test/functional/builder/ExplicitJitJaniModelBuilderTest.cpp b/test/functional/builder/ExplicitJitJaniModelBuilderTest.cpp new file mode 100644 index 000000000..b2f06a905 --- /dev/null +++ b/test/functional/builder/ExplicitJitJaniModelBuilderTest.cpp @@ -0,0 +1,12 @@ +#include "gtest/gtest.h" +#include "storm-config.h" +#include "src/parser/PrismParser.h" +#include "src/storage/jani/Model.h" +#include "src/builder/ExplicitJitJaniModelBuilder.h" + +TEST(ExplicitJitJaniModelBuilderTest, Dtmc) { + storm::prism::Program program = storm::parser::PrismParser::parse(STORM_CPP_TESTS_BASE_PATH "/functional/builder/die.pm"); + storm::jani::Model janiModel = program.toJani(); + + storm::builder::ExplicitJitJaniModelBuilder(janiModel).build(); +}