version 0.3.25
authorSomeDude <SomeDude@NuBL7aaJ6Cn4fB7GXFb9Zfi8w1FhPyW3oKgU9TweZMw>
Tue, 9 Dec 2008 17:45:00 +0000 (18:45 +0100)
committerDavid ‘Bombe’ Roden <bombe@freenetproject.org>
Tue, 9 Dec 2008 17:45:00 +0000 (18:45 +0100)
CMakeLists.txt
include/global.h
include/unicode/ConvertUTF.h [new file with mode: 0644]
include/unicode/unicodeformatter.h [new file with mode: 0644]
include/unicode/utfconversion.h [new file with mode: 0644]
readme.txt
src/http/pages/forumcreatepostpage.cpp
src/http/pages/forumviewthreadpage.cpp
src/unicode/ConvertUTF.c [new file with mode: 0644]
src/unicode/unicodeformatter.cpp [new file with mode: 0644]
src/unicode/utfconversion.cpp [new file with mode: 0644]

index 4585bbc..e4251a9 100644 (file)
@@ -107,6 +107,9 @@ src/nntp/mime/MimeChar.cpp
 src/nntp/mime/MimeCode.cpp\r
 src/nntp/mime/MimeType.cpp\r
 src/threadwrapper/threadedexector.cpp\r
+src/unicode/ConvertUTF.c\r
+src/unicode/unicodeformatter.cpp\r
+src/unicode/utfconversion.cpp\r
 )\r
 \r
 SET(FMS_PLATFORM_SRC )\r
index eb9d33d..f305cfb 100644 (file)
@@ -7,10 +7,10 @@
 \r
 #define VERSION_MAJOR          "0"\r
 #define VERSION_MINOR          "3"\r
-#define VERSION_RELEASE                "24"\r
+#define VERSION_RELEASE                "25"\r
 #define FMS_VERSION                    VERSION_MAJOR"."VERSION_MINOR"."VERSION_RELEASE\r
-#define FMS_FREESITE_USK       "USK@0npnMrqZNKRCRoGojZV93UNHCMN-6UU3rRSAmP6jNLE,~BG-edFtdCC1cSH4O3BWdeIYa8Sw5DfyrSV-TKdO5ec,AQACAAE/fms/85/"\r
-#define FMS_VERSION_EDITION    "26"\r
+#define FMS_FREESITE_USK       "USK@0npnMrqZNKRCRoGojZV93UNHCMN-6UU3rRSAmP6jNLE,~BG-edFtdCC1cSH4O3BWdeIYa8Sw5DfyrSV-TKdO5ec,AQACAAE/fms/86/"\r
+#define FMS_VERSION_EDITION    "27"\r
 \r
 typedef Poco::ScopedLock<Poco::FastMutex> Guard;\r
 \r
diff --git a/include/unicode/ConvertUTF.h b/include/unicode/ConvertUTF.h
new file mode 100644 (file)
index 0000000..e264915
--- /dev/null
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
+
+    Several funtions are included here, forming a complete set of
+    conversions between the three formats.  UTF-7 is not included
+    here, but is handled in a separate source file.
+
+    Each of these routines takes pointers to input buffers and output
+    buffers.  The input buffers are const.
+
+    Each routine converts the text between *sourceStart and sourceEnd,
+    putting the result into the buffer between *targetStart and
+    targetEnd. Note: the end pointers are *after* the last item: e.g. 
+    *(sourceEnd - 1) is the last item.
+
+    The return result indicates whether the conversion was successful,
+    and if not, whether the problem was in the source or target buffers.
+    (Only the first encountered problem is indicated.)
+
+    After the conversion, *sourceStart and *targetStart are both
+    updated to point to the end of last text successfully converted in
+    the respective buffers.
+
+    Input parameters:
+       sourceStart - pointer to a pointer to the source buffer.
+               The contents of this are modified on return so that
+               it points at the next thing to be converted.
+       targetStart - similarly, pointer to pointer to the target buffer.
+       sourceEnd, targetEnd - respectively pointers to the ends of the
+               two buffers, for overflow checking only.
+
+    These conversion functions take a ConversionFlags argument. When this
+    flag is set to strict, both irregular sequences and isolated surrogates
+    will cause an error.  When the flag is set to lenient, both irregular
+    sequences and isolated surrogates are converted.
+
+    Whether the flag is strict or lenient, all illegal sequences will cause
+    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
+    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
+    must check for illegal sequences.
+
+    When the flag is set to lenient, characters over 0x10FFFF are converted
+    to the replacement character; otherwise (when the flag is set to strict)
+    they constitute an error.
+
+    Output parameters:
+       The value "sourceIllegal" is returned from some routines if the input
+       sequence is malformed.  When "sourceIllegal" is returned, the source
+       value will point to the illegal value that caused the problem. E.g.,
+       in UTF-8 when a sequence is malformed, it points to the start of the
+       malformed sequence.  
+
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+                Fixes & updates, Sept 2001.
+
+------------------------------------------------------------------------ */
+
+/* ---------------------------------------------------------------------
+    The following 4 definitions are compiler-specific.
+    The C standard does not guarantee that wchar_t has at least
+    16 bits, so wchar_t is no less portable than unsigned short!
+    All should be unsigned values to avoid sign extension during
+    bit mask & shift operations.
+------------------------------------------------------------------------ */
+
+typedef unsigned long  UTF32;  /* at least 32 bits */
+typedef unsigned short UTF16;  /* at least 16 bits */
+typedef unsigned char  UTF8;   /* typically 8 bits */
+typedef unsigned char  Boolean; /* 0 or 1 */
+
+/* Some fundamental constants */
+#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
+#define UNI_MAX_BMP (UTF32)0x0000FFFF
+#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
+#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
+#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
+
+typedef enum {
+       conversionOK,           /* conversion successful */
+       sourceExhausted,        /* partial character in source, but hit end */
+       targetExhausted,        /* insuff. room in target for conversion */
+       sourceIllegal           /* source sequence is illegal/malformed */
+} ConversionResult;
+
+typedef enum {
+       strictConversion = 0,
+       lenientConversion
+} ConversionFlags;
+
+/* This is for C++ and does no harm in C */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ConversionResult ConvertUTF8toUTF16 (
+               const UTF8** sourceStart, const UTF8* sourceEnd, 
+               UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF16toUTF8 (
+               const UTF16** sourceStart, const UTF16* sourceEnd, 
+               UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+               
+ConversionResult ConvertUTF8toUTF32 (
+               const UTF8** sourceStart, const UTF8* sourceEnd, 
+               UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF8 (
+               const UTF32** sourceStart, const UTF32* sourceEnd, 
+               UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+               
+ConversionResult ConvertUTF16toUTF32 (
+               const UTF16** sourceStart, const UTF16* sourceEnd, 
+               UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF16 (
+               const UTF32** sourceStart, const UTF32* sourceEnd, 
+               UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
+
+#ifdef __cplusplus
+}
+#endif
+
+/* --------------------------------------------------------------------- */
diff --git a/include/unicode/unicodeformatter.h b/include/unicode/unicodeformatter.h
new file mode 100644 (file)
index 0000000..1f2d31c
--- /dev/null
@@ -0,0 +1,19 @@
+#ifndef _unicode_formatter_\r
+#define _unicode_formatter_\r
+\r
+#include <string>\r
+\r
+class UnicodeFormatter\r
+{\r
+public:\r
+\r
+       static const bool LineWrap(const std::string &utf8input, const int linelength, const std::string &ignorechars, std::string &utf8output);\r
+\r
+private:\r
+\r
+       static std::wstring m_unicodenewline;\r
+       static wchar_t m_unicodewhitespace[];\r
+\r
+};\r
+\r
+#endif // _unicode_formatter_\r
diff --git a/include/unicode/utfconversion.h b/include/unicode/utfconversion.h
new file mode 100644 (file)
index 0000000..2ebe864
--- /dev/null
@@ -0,0 +1,25 @@
+#ifndef _UTF_conversion_\r
+#define _UTF_conversion_\r
+\r
+/*\r
+       Thanks to http://www.codeproject.com/KB/string/UtfConverter.aspx for the basic idea\r
+       http://www.unicode.org/Public/PROGRAMS/CVTUTF/\r
+*/\r
+\r
+#include <string>\r
+#include <vector>\r
+\r
+#include "ConvertUTF.h"\r
+\r
+namespace UTFConversion\r
+{\r
+\r
+// UTF-8 byte sequence to UTF-16 or UTF-32, depending on size of wchar_t\r
+const bool FromUTF8(const std::vector<char> &utf8string, std::wstring &wcstring);\r
+const bool FromUTF8(const std::string &utf8string, std::wstring &wcstring);\r
+// UTF-16 or UTF-32 to UTF-8 byte sequence\r
+const bool ToUTF8(const std::wstring &wcstring, std::string &utf8string);\r
+\r
+}      // namespace\r
+\r
+#endif // _UTF_conversion_\r
index a919c4d..4968c47 100644 (file)
@@ -17,10 +17,10 @@ This option requires the FreeImage library to be installed.
 \r
 UPGRADING\r
 ---------\r
-It is always a good idea to make a copy of your current FMS installation before\r
-continuing.  First shut down FMS, make a copy of the directory, and then\r
-replace all files except the database with those from the new version.  You may\r
-keep the same database unless otherwise noted in the release information.\r
+*ALWAYS* make a copy of your current FMS installation before continuing.  First\r
+shut down FMS, make a copy of the directory, and then replace all files except\r
+the database with those from the new version.  You may keep the same database\r
+unless otherwise noted in the release information.\r
 \r
 INSTALLATION\r
 ------------\r
index 1896143..4fd81c9 100644 (file)
@@ -1,6 +1,7 @@
 #include "../../../include/http/pages/forumcreatepostpage.h"\r
 #include "../../../include/stringfunctions.h"\r
 #include "../../../include/message.h"\r
+#include "../../../include/unicode/unicodeformatter.h"\r
 \r
 #ifdef XMEM\r
        #include <xmem.h>\r
@@ -61,6 +62,7 @@ const std::string ForumCreatePostPage::GeneratePage(const std::string &method, c
                {\r
                        body=(*queryvars.find("body")).second;\r
                        body=StringFunctions::Replace(body,"\r\n","\n");\r
+                       UnicodeFormatter::LineWrap(body,80,">",body);\r
                }\r
                else\r
                {\r
@@ -148,6 +150,7 @@ const std::string ForumCreatePostPage::GeneratePage(const std::string &method, c
                                        }\r
                                        body+="\n";\r
                                }\r
+\r
                        }\r
                }\r
        }\r
index d99768d..a680bfe 100644 (file)
@@ -1,5 +1,6 @@
 #include "../../../include/http/pages/forumviewthreadpage.h"\r
 #include "../../../include/stringfunctions.h"\r
+#include "../../../include/unicode/unicodeformatter.h"\r
 \r
 #ifdef XMEM\r
        #include <xmem.h>\r
@@ -12,6 +13,8 @@ const std::string ForumViewThreadPage::FixBody(const std::string &body)
 \r
        output=StringFunctions::Replace(output,"\r\n","\n");\r
 \r
+       UnicodeFormatter::LineWrap(output,80,"",output);\r
+       /*\r
        // put \n after 80 contiguous characters in the body\r
        std::string::size_type prevpos=0;\r
        std::string::size_type pos=output.find_first_of(whitespace);\r
@@ -30,6 +33,7 @@ const std::string ForumViewThreadPage::FixBody(const std::string &body)
                output.insert(prevpos+80,"\n");\r
                prevpos+=81;\r
        }\r
+       */\r
 \r
        output=StringFunctions::Replace(output,"<","&lt;");\r
        output=StringFunctions::Replace(output,">","&gt;");\r
diff --git a/src/unicode/ConvertUTF.c b/src/unicode/ConvertUTF.c
new file mode 100644 (file)
index 0000000..1567f64
--- /dev/null
@@ -0,0 +1,539 @@
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+    Sept 2001: fixed const & error conditions per
+       mods suggested by S. Parent & A. Lillich.
+    June 2002: Tim Dodd added detection and handling of incomplete
+       source sequences, enhanced error detection, added casts
+       to eliminate compiler warnings.
+    July 2003: slight mods to back out aggressive FFFE detection.
+    Jan 2004: updated switches in from-UTF8 conversions.
+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
+
+    See the header file "ConvertUTF.h" for complete documentation.
+
+------------------------------------------------------------------------ */
+
+
+#include "../../include/unicode/ConvertUTF.h"
+#ifdef CVTUTF_DEBUG
+#include <stdio.h>
+#endif
+
+static const int halfShift  = 10; /* used for shifting by 10 bits */
+
+static const UTF32 halfBase = 0x0010000UL;
+static const UTF32 halfMask = 0x3FFUL;
+
+#define UNI_SUR_HIGH_START  (UTF32)0xD800
+#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
+#define UNI_SUR_LOW_START   (UTF32)0xDC00
+#define UNI_SUR_LOW_END     (UTF32)0xDFFF
+#define false     0
+#define true       1
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF16 (
+       const UTF32** sourceStart, const UTF32* sourceEnd, 
+       UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+       UTF32 ch;
+       if (target >= targetEnd) {
+           result = targetExhausted; break;
+       }
+       ch = *source++;
+       if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+           /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
+           if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+               if (flags == strictConversion) {
+                   --source; /* return to the illegal value itself */
+                   result = sourceIllegal;
+                   break;
+               } else {
+                   *target++ = UNI_REPLACEMENT_CHAR;
+               }
+           } else {
+               *target++ = (UTF16)ch; /* normal case */
+           }
+       } else if (ch > UNI_MAX_LEGAL_UTF32) {
+           if (flags == strictConversion) {
+               result = sourceIllegal;
+           } else {
+               *target++ = UNI_REPLACEMENT_CHAR;
+           }
+       } else {
+           /* target is a character in range 0xFFFF - 0x10FFFF. */
+           if (target + 1 >= targetEnd) {
+               --source; /* Back up source pointer! */
+               result = targetExhausted; break;
+           }
+           ch -= halfBase;
+           *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+           *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+       }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF32 (
+       const UTF16** sourceStart, const UTF16* sourceEnd, 
+       UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF32* target = *targetStart;
+    UTF32 ch, ch2;
+    while (source < sourceEnd) {
+       const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
+       ch = *source++;
+       /* If we have a surrogate pair, convert to UTF32 first. */
+       if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+           /* If the 16 bits following the high surrogate are in the source buffer... */
+           if (source < sourceEnd) {
+               ch2 = *source;
+               /* If it's a low surrogate, convert to UTF32. */
+               if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                   ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                       + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                   ++source;
+               } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                   --source; /* return to the illegal value itself */
+                   result = sourceIllegal;
+                   break;
+               }
+           } else { /* We don't have the 16 bits following the high surrogate. */
+               --source; /* return to the high surrogate */
+               result = sourceExhausted;
+               break;
+           }
+       } else if (flags == strictConversion) {
+           /* UTF-16 surrogate values are illegal in UTF-32 */
+           if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+               --source; /* return to the illegal value itself */
+               result = sourceIllegal;
+               break;
+           }
+       }
+       if (target >= targetEnd) {
+           source = oldSource; /* Back up source pointer! */
+           result = targetExhausted; break;
+       }
+       *target++ = ch;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+#ifdef CVTUTF_DEBUG
+if (result == sourceIllegal) {
+    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
+    fflush(stderr);
+}
+#endif
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
+ * left as-is for anyone who may want to do such conversion, which was
+ * allowed in earlier algorithms.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
+};
+
+/*
+ * Magic values subtracted from a buffer value during UTF8 conversion.
+ * This table contains as many values as there might be trailing bytes
+ * in a UTF-8 sequence.
+ */
+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
+                    0x03C82080UL, 0xFA082080UL, 0x82082080UL };
+
+/*
+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
+ * into the first byte, depending on how many bytes follow.  There are
+ * as many entries in this table as there are UTF-8 sequence types.
+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
+ * for *legal* UTF-8 will be 4 or fewer bytes total.
+ */
+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+/* --------------------------------------------------------------------- */
+
+/* The interface converts a whole buffer to avoid function-call overhead.
+ * Constants have been gathered. Loops & conditionals have been removed as
+ * much as possible for efficiency, in favor of drop-through switches.
+ * (See "Note A" at the bottom of the file for equivalent code.)
+ * If your compiler supports it, the "isLegalUTF8" call can be turned
+ * into an inline function.
+ */
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF8 (
+       const UTF16** sourceStart, const UTF16* sourceEnd, 
+       UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+       UTF32 ch;
+       unsigned short bytesToWrite = 0;
+       const UTF32 byteMask = 0xBF;
+       const UTF32 byteMark = 0x80; 
+       const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
+       ch = *source++;
+       /* If we have a surrogate pair, convert to UTF32 first. */
+       if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+           /* If the 16 bits following the high surrogate are in the source buffer... */
+           if (source < sourceEnd) {
+               UTF32 ch2 = *source;
+               /* If it's a low surrogate, convert to UTF32. */
+               if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                   ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                       + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                   ++source;
+               } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                   --source; /* return to the illegal value itself */
+                   result = sourceIllegal;
+                   break;
+               }
+           } else { /* We don't have the 16 bits following the high surrogate. */
+               --source; /* return to the high surrogate */
+               result = sourceExhausted;
+               break;
+           }
+       } else if (flags == strictConversion) {
+           /* UTF-16 surrogate values are illegal in UTF-32 */
+           if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+               --source; /* return to the illegal value itself */
+               result = sourceIllegal;
+               break;
+           }
+       }
+       /* Figure out how many bytes the result will require */
+       if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+       } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+       } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+       } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
+       } else {                            bytesToWrite = 3;
+                                           ch = UNI_REPLACEMENT_CHAR;
+       }
+
+       target += bytesToWrite;
+       if (target > targetEnd) {
+           source = oldSource; /* Back up source pointer! */
+           target -= bytesToWrite; result = targetExhausted; break;
+       }
+       switch (bytesToWrite) { /* note: everything falls through. */
+           case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+           case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+           case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+           case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
+       }
+       target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
+ * This must be called with the length pre-determined by the first byte.
+ * If not calling this from ConvertUTF8to*, then the length can be set by:
+ *  length = trailingBytesForUTF8[*source]+1;
+ * and the sequence is illegal right away if there aren't that many bytes
+ * available.
+ * If presented with a length > 4, this returns false.  The Unicode
+ * definition of UTF-8 goes up to 4-byte sequences.
+ */
+
+static Boolean isLegalUTF8(const UTF8 *source, int length) {
+    UTF8 a;
+    const UTF8 *srcptr = source+length;
+    switch (length) {
+    default: return false;
+       /* Everything else falls through when "true"... */
+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
+
+       switch (*source) {
+           /* no fall-through in this inner switch */
+           case 0xE0: if (a < 0xA0) return false; break;
+           case 0xED: if (a > 0x9F) return false; break;
+           case 0xF0: if (a < 0x90) return false; break;
+           case 0xF4: if (a > 0x8F) return false; break;
+           default:   if (a < 0x80) return false;
+       }
+
+    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+    }
+    if (*source > 0xF4) return false;
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 sequence is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
+    int length = trailingBytesForUTF8[*source]+1;
+    if (source+length > sourceEnd) {
+       return false;
+    }
+    return isLegalUTF8(source, length);
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF16 (
+       const UTF8** sourceStart, const UTF8* sourceEnd, 
+       UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+       UTF32 ch = 0;
+       unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+       if (source + extraBytesToRead >= sourceEnd) {
+           result = sourceExhausted; break;
+       }
+       /* Do this check whether lenient or strict */
+       if (! isLegalUTF8(source, extraBytesToRead+1)) {
+           result = sourceIllegal;
+           break;
+       }
+       /*
+        * The cases all fall through. See "Note A" below.
+        */
+       switch (extraBytesToRead) {
+           case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+           case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+           case 3: ch += *source++; ch <<= 6;
+           case 2: ch += *source++; ch <<= 6;
+           case 1: ch += *source++; ch <<= 6;
+           case 0: ch += *source++;
+       }
+       ch -= offsetsFromUTF8[extraBytesToRead];
+
+       if (target >= targetEnd) {
+           source -= (extraBytesToRead+1); /* Back up source pointer! */
+           result = targetExhausted; break;
+       }
+       if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+           /* UTF-16 surrogate values are illegal in UTF-32 */
+           if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+               if (flags == strictConversion) {
+                   source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                   result = sourceIllegal;
+                   break;
+               } else {
+                   *target++ = UNI_REPLACEMENT_CHAR;
+               }
+           } else {
+               *target++ = (UTF16)ch; /* normal case */
+           }
+       } else if (ch > UNI_MAX_UTF16) {
+           if (flags == strictConversion) {
+               result = sourceIllegal;
+               source -= (extraBytesToRead+1); /* return to the start */
+               break; /* Bail out; shouldn't continue */
+           } else {
+               *target++ = UNI_REPLACEMENT_CHAR;
+           }
+       } else {
+           /* target is a character in range 0xFFFF - 0x10FFFF. */
+           if (target + 1 >= targetEnd) {
+               source -= (extraBytesToRead+1); /* Back up source pointer! */
+               result = targetExhausted; break;
+           }
+           ch -= halfBase;
+           *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+           *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+       }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF8 (
+       const UTF32** sourceStart, const UTF32* sourceEnd, 
+       UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+       UTF32 ch;
+       unsigned short bytesToWrite = 0;
+       const UTF32 byteMask = 0xBF;
+       const UTF32 byteMark = 0x80; 
+       ch = *source++;
+       if (flags == strictConversion ) {
+           /* UTF-16 surrogate values are illegal in UTF-32 */
+           if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+               --source; /* return to the illegal value itself */
+               result = sourceIllegal;
+               break;
+           }
+       }
+       /*
+        * Figure out how many bytes the result will require. Turn any
+        * illegally large UTF32 things (> Plane 17) into replacement chars.
+        */
+       if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+       } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+       } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+       } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
+       } else {                            bytesToWrite = 3;
+                                           ch = UNI_REPLACEMENT_CHAR;
+                                           result = sourceIllegal;
+       }
+       
+       target += bytesToWrite;
+       if (target > targetEnd) {
+           --source; /* Back up source pointer! */
+           target -= bytesToWrite; result = targetExhausted; break;
+       }
+       switch (bytesToWrite) { /* note: everything falls through. */
+           case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+           case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+           case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+           case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
+       }
+       target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF32 (
+       const UTF8** sourceStart, const UTF8* sourceEnd, 
+       UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF32* target = *targetStart;
+    while (source < sourceEnd) {
+       UTF32 ch = 0;
+       unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+       if (source + extraBytesToRead >= sourceEnd) {
+           result = sourceExhausted; break;
+       }
+       /* Do this check whether lenient or strict */
+       if (! isLegalUTF8(source, extraBytesToRead+1)) {
+           result = sourceIllegal;
+           break;
+       }
+       /*
+        * The cases all fall through. See "Note A" below.
+        */
+       switch (extraBytesToRead) {
+           case 5: ch += *source++; ch <<= 6;
+           case 4: ch += *source++; ch <<= 6;
+           case 3: ch += *source++; ch <<= 6;
+           case 2: ch += *source++; ch <<= 6;
+           case 1: ch += *source++; ch <<= 6;
+           case 0: ch += *source++;
+       }
+       ch -= offsetsFromUTF8[extraBytesToRead];
+
+       if (target >= targetEnd) {
+           source -= (extraBytesToRead+1); /* Back up the source pointer! */
+           result = targetExhausted; break;
+       }
+       if (ch <= UNI_MAX_LEGAL_UTF32) {
+           /*
+            * UTF-16 surrogate values are illegal in UTF-32, and anything
+            * over Plane 17 (> 0x10FFFF) is illegal.
+            */
+           if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+               if (flags == strictConversion) {
+                   source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                   result = sourceIllegal;
+                   break;
+               } else {
+                   *target++ = UNI_REPLACEMENT_CHAR;
+               }
+           } else {
+               *target++ = ch;
+           }
+       } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
+           result = sourceIllegal;
+           *target++ = UNI_REPLACEMENT_CHAR;
+       }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* ---------------------------------------------------------------------
+
+    Note A.
+    The fall-through switches in UTF-8 reading code save a
+    temp variable, some decrements & conditionals.  The switches
+    are equivalent to the following loop:
+       {
+           int tmpBytesToRead = extraBytesToRead+1;
+           do {
+               ch += *source++;
+               --tmpBytesToRead;
+               if (tmpBytesToRead) ch <<= 6;
+           } while (tmpBytesToRead > 0);
+       }
+    In UTF-8 writing code, the switches on "bytesToWrite" are
+    similarly unrolled loops.
+
+   --------------------------------------------------------------------- */
diff --git a/src/unicode/unicodeformatter.cpp b/src/unicode/unicodeformatter.cpp
new file mode 100644 (file)
index 0000000..c89cc7c
--- /dev/null
@@ -0,0 +1,78 @@
+#include "../../include/unicode/unicodeformatter.h"\r
+#include "../../include/unicode/utfconversion.h"\r
+\r
+std::wstring UnicodeFormatter::m_unicodenewline=L"\n";\r
+wchar_t UnicodeFormatter::m_unicodewhitespace[]={0x0009,0x000A,0x000B,0x000C,0x000D,\r
+                                                                                               0x0020,0x0085,0x00A0,0x1680,0x180E,\r
+                                                                                               0x2000,0x2001,0x2002,0x2003,0x2004,\r
+                                                                                               0x2005,0x2006,0x2007,0x2008,0x2009,\r
+                                                                                               0x200A,0x200B,0x2029,0x202F,0x205F,\r
+                                                                                               0x3000,0xFEFF};\r
+\r
+const bool UnicodeFormatter::LineWrap(const std::string &utf8input, const int linelength, const std::string &ignorechars, std::string &utf8output)\r
+{\r
+       std::wstring wcstring;\r
+       std::wstring wcignorechars;\r
+\r
+       if(UTFConversion::FromUTF8(utf8input,wcstring) && UTFConversion::FromUTF8(ignorechars,wcignorechars))\r
+       {\r
+\r
+               std::wstring::size_type currentpos=0;\r
+               std::wstring::size_type lastnewlinepos=0;\r
+               std::wstring::size_type whitespacepos=0;\r
+\r
+               while(currentpos+linelength<wcstring.length())\r
+               {\r
+                       if(ignorechars.size()==0 || wcstring.find_first_of(wcignorechars,currentpos)!=currentpos)\r
+                       {\r
+                               lastnewlinepos=wcstring.rfind(m_unicodenewline,currentpos+linelength);\r
+                               // newline found within line length - we don't need to wrap\r
+                               if(lastnewlinepos!=std::wstring::npos && lastnewlinepos>=currentpos)\r
+                               {\r
+                                       currentpos=lastnewlinepos+1;\r
+                               }\r
+                               // newline doesn't exist at all - force one in\r
+                               else if(lastnewlinepos==std::wstring::npos)\r
+                               {\r
+                                       wcstring.insert(currentpos+linelength,m_unicodenewline);\r
+                                       currentpos+=linelength+m_unicodenewline.length();\r
+                               }\r
+                               else\r
+                               {\r
+                                       whitespacepos=wcstring.find_last_of(m_unicodewhitespace,currentpos+linelength);\r
+                                       // whitespace found within line length - erase whitespace and insert newline\r
+                                       if(whitespacepos!=std::wstring::npos && whitespacepos>=currentpos)\r
+                                       {\r
+                                               wcstring.erase(whitespacepos,1);\r
+                                               wcstring.insert(whitespacepos,m_unicodenewline);\r
+                                               currentpos=whitespacepos+m_unicodenewline.length();\r
+                                       }\r
+                                       // whitespace not found within line length - force newline at line length\r
+                                       else\r
+                                       {\r
+                                               wcstring.insert(currentpos+linelength,m_unicodenewline);\r
+                                               currentpos+=linelength+m_unicodenewline.length();\r
+                                       }\r
+                               }\r
+                       }\r
+                       else\r
+                       {\r
+                               currentpos=wcstring.find(m_unicodenewline,currentpos+1);\r
+                               if(currentpos==std::string::npos)\r
+                               {\r
+                                       currentpos=wcstring.size();\r
+                               }\r
+                               currentpos++;\r
+                       }\r
+               }\r
+\r
+               if(UTFConversion::ToUTF8(wcstring,utf8output))\r
+               {\r
+                       return true;\r
+               }\r
+\r
+       }\r
+\r
+       return false;\r
+\r
+}\r
diff --git a/src/unicode/utfconversion.cpp b/src/unicode/utfconversion.cpp
new file mode 100644 (file)
index 0000000..112ed17
--- /dev/null
@@ -0,0 +1,122 @@
+#include "../../include/unicode/utfconversion.h"\r
+\r
+namespace UTFConversion\r
+{\r
+\r
+const bool FromUTF8(const std::vector<char> &utf8string, std::wstring &wcstring)\r
+{\r
+       if(utf8string.size()==0)\r
+       {\r
+               wcstring.assign(L"");\r
+               return true;\r
+       }\r
+\r
+       std::vector<wchar_t> dest(utf8string.size(),0);         // dest will never be bigger than the input but could be smaller\r
+       \r
+       const UTF8 *sourcestart=reinterpret_cast<const UTF8 *>(&utf8string[0]);\r
+       const UTF8 *sourceend=sourcestart+utf8string.size();\r
+       \r
+       if(sizeof(wchar_t)==2)\r
+       {       \r
+               UTF16 *deststart=reinterpret_cast<UTF16 *>(&dest[0]);\r
+               UTF16 *destend=deststart+dest.size();\r
+               \r
+               ConversionResult rval=ConvertUTF8toUTF16(&sourcestart,sourceend,&deststart,destend,lenientConversion);\r
+               \r
+               if(rval!=conversionOK)\r
+               {\r
+                       return false;   \r
+               }\r
+               \r
+               wcstring.assign(dest.begin(),dest.end()-(destend-deststart));\r
+               \r
+       }\r
+       else if(sizeof(wchar_t)==4)\r
+       {\r
+               UTF32 *deststart=reinterpret_cast<UTF32 *>(&dest[0]);\r
+               UTF32 *destend=deststart+dest.size();\r
+               \r
+               ConversionResult rval=ConvertUTF8toUTF32(&sourcestart,sourceend,&deststart,destend,lenientConversion);\r
+\r
+               if(rval!=conversionOK)\r
+               {\r
+                       return false;   \r
+               }\r
+               \r
+               wcstring.assign(dest.begin(),dest.end()-(destend-deststart));\r
+               \r
+       }\r
+       else\r
+       {\r
+               return false;   \r
+       }\r
+\r
+       return true;\r
+}\r
+\r
+const bool FromUTF8(const std::string &utf8string, std::wstring &wcstring)\r
+{\r
+\r
+       return FromUTF8(std::vector<char>(utf8string.begin(),utf8string.end()),wcstring);\r
+\r
+}\r
+\r
+const bool ToUTF8(const std::wstring &wcstring, std::string &utf8string)\r
+{\r
+       if(wcstring.size()==0)\r
+       {\r
+               utf8string.assign("");\r
+               return true;\r
+       }\r
+\r
+       std::vector<wchar_t> source(wcstring.begin(),wcstring.end());\r
+       \r
+       if(sizeof(wchar_t)==2)\r
+       {\r
+               std::vector<char> dest(wcstring.size()*2,0);\r
+               \r
+               const UTF16 *sourcestart=reinterpret_cast<const UTF16 *>(&source[0]);\r
+               const UTF16 *sourceend=sourcestart+source.size();\r
+               \r
+               UTF8 *deststart=reinterpret_cast<UTF8 *>(&dest[0]);\r
+               UTF8 *destend=deststart+dest.size();\r
+               \r
+               ConversionResult rval=ConvertUTF16toUTF8(&sourcestart,sourceend,&deststart,destend,lenientConversion);\r
+               \r
+               if(rval!=conversionOK)\r
+               {\r
+                       return false;   \r
+               }\r
+               \r
+               utf8string.assign(dest.begin(),dest.end()-(destend-deststart));\r
+               \r
+       }\r
+       else if(sizeof(wchar_t)==4)\r
+       {\r
+               std::vector<char> dest(wcstring.size()*4,0);\r
+               \r
+               const UTF32 *sourcestart=reinterpret_cast<const UTF32 *>(&source[0]);\r
+               const UTF32 *sourceend=sourcestart+source.size();\r
+               \r
+               UTF8 *deststart=reinterpret_cast<UTF8 *>(&dest[0]);\r
+               UTF8 *destend=deststart+dest.size();\r
+               \r
+               ConversionResult rval=ConvertUTF32toUTF8(&sourcestart,sourceend,&deststart,destend,lenientConversion);\r
+               \r
+               if(rval!=conversionOK)\r
+               {\r
+                       return false;   \r
+               }\r
+               \r
+               utf8string.assign(dest.begin(),dest.end()-(destend-deststart));\r
+               \r
+       }\r
+       else\r
+       {\r
+               return false;\r
+       }\r
+\r
+       return true;\r
+}\r
+\r
+}      // namespace\r