// bdlde_charconvertucs2.h                                            -*-C++-*-
#ifndef INCLUDED_BDLDE_CHARCONVERTUCS2
#define INCLUDED_BDLDE_CHARCONVERTUCS2

#include <bsls_ident.h>

BSLS_IDENT("$Id: $")

//@PURPOSE: Provide efficient conversions between UTF-8 and UCS-2 encodings.
//
//@CLASSES:
//  bdlde::CharConvertUcs2: namespace for conversions between UTF-8 and UCS-2
//
//@DESCRIPTION: This component provides a suite of pure procedures supporting
// the *fast* conversion of *valid* UTF-8 encoded "C" strings to *valid* UCS-2
// 16-bit character arrays and vice versa.  In order to provide the fastest
// possible implementation, some error checking is deliberately omitted, and
// the input strings are required to be null-terminated; however, all C-style
// functions will honor `strlcpy` semantics and null-terminate any output
// buffer having a non-zero length.
//
///History and Motivation
///----------------------
// UTF-8 is a character encoding that allows 32-bit character sets like Unicode
// to be represented using null-terminated (8-bit) byte strings (NTBS), while
// allowing "standard ASCII" strings to be used "as-is".  Note that UTF-8 is
// described in detail in RFC 2279 (http://tools.ietf.org/html/rfc2279).
//
// UCS-2 is a 16-bit character encoding with no support for "higher-order"
// character encodings.  UCS-2 is equivalent to UTF-16 in the Basic
// Multilingual Plane (BMP) of Unicode (the first 65536 character points,
// excluding the "surrogate code points" U+D800-U+DFFF, which do not map to
// Unicode characters).  If the characters being represented are within the
// BMP, then UCS-2 can be thought of as "the Windows encoding" for
// international characters.  Historically, UCS-2 was the only "wide char"
// representation for Windows versions prior to Windows 2000.  UTF-16 was
// adopted instead for Windows 2000, and has been used ever since.
//
// Most conversion routines strive for correctness at the cost of performance.
// The `glib` conversion routines are *much* slower than the functions
// implemented here because the `glib` functions first compute the number of
// output characters required, allocate the memory for them, and then perform
// the conversion, validating the input characters.  The C-style methods of
// `bdlde::CharConvertUcs2`, on the other hand, assume that the user-provided
// output buffer is wide enough, make a "best effort" to convert into it, and
// return an error code if not enough space was provided.  The C++-style
// methods are more forgiving, since the output `bsl::string` or
// `bsl::vector<unsigned short>` is resized as needed.  No attempt is made to
// validate whether the character codes correspond to valid Unicode code
// points, nor is validation performed to check for overlong UTF-8 encodings
// (where characters that could be expressed in one octet are encoded using two
// octets).
//
///Usage
///-----
// This section illustrates intended use of this component.
//
///Example 1: C-Style Interface
/// - - - - - - - - - - - - - -
// The following snippet of code illustrates a typical use of the
// `bdlde::CharConvertUcs2` struct's C-style utility functions, converting a
// simple UTF-8 string to UCS-2.
// ```
// void testCFunction1()
// {
//     unsigned short buffer[256];  // arbitrary "wide-enough" size
//     bsl::size_t    buffSize = sizeof buffer / sizeof *buffer;
//     bsl::size_t    charsWritten;
//
//     int retVal =
//               BloombergLP::bdlde::CharConvertUcs2::utf8ToUcs2(buffer,
//                                                              buffSize,
//                                                              "Hello",
//                                                              &charsWritten);
//
//     assert( 0  == retVal);
//     assert('H' == buffer[0]);
//     assert('e' == buffer[1]);
//     assert('l' == buffer[2]);
//     assert('l' == buffer[3]);
//     assert('o' == buffer[4]);
//     assert( 0  == buffer[5]);
//     assert( 6  == charsWritten);
// }
// ```
//
///Example 2: C-Style Round-Trip
///- - - - - - - - - - - - - - -
// The following snippet of code illustrates another typical use of the
// `bdlde::CharConvertUcs2` struct's C-style utility functions, converting a
// simple UTF-8 string to UCS-2, then converting the UCS-2 back and making sure
// the round-trip conversion results in the input.
// ```
// void testCFunction2()
// {
//     unsigned short buffer[256];  // arbitrary "wide-enough" size
//     bsl::size_t    buffSize = sizeof buffer / sizeof *buffer;
//     bsl::size_t    charsWritten;
//
//     // "&Eacute;cole", the French word for School.  '&Eacute;' is the HTML
//     // entity equivalent to "Unicode-E WITH ACUTE, LATIN CAPITAL LETTER".
//     int retVal =
//           BloombergLP::bdlde::CharConvertUcs2::utf8ToUcs2(buffer,
//                                                          buffSize,
//                                                          "\xc3\x89" "cole",
//                                                          &charsWritten);
//
//     assert( 0   == retVal);
//     assert(0xc9 == buffer[0]); // Unicode-E WITH ACUTE, LATIN CAPITAL LETTER
//     assert('c'  == buffer[1]);
//     assert('o'  == buffer[2]);
//     assert('l'  == buffer[3]);
//     assert('e'  == buffer[4]);
//     assert( 0   == buffer[5]);
//     assert( 6   == charsWritten);
//
//     char           buffer2[256];  // arbitrary "wide-enough" size
//     bsl::size_t    buffer2Size  = sizeof buffer2 / sizeof *buffer2;
//     bsl::size_t    bytesWritten = 0;
//
//     // Reversing the conversion returns the original string:
//     retVal =
//           BloombergLP::bdlde::CharConvertUcs2::ucs2ToUtf8(buffer2,
//                                                          buffer2Size,
//                                                          buffer,
//                                                          &charsWritten,
//                                                          &bytesWritten);
//
//     assert( 0 == retVal);
//     assert( 0 == bsl::strcmp(buffer2, "\xc3\x89" "cole"));
//
//     // 6 characters written, but 7 bytes, since the first character takes 2
//     // octets.
//
//     assert( 6 == charsWritten);
//     assert( 7 == bytesWritten);
// }
// ```
// In this example, a UTF-8 input string is converted then passed to another
// function, which expects a UCS-2 buffer.
//
// First, we define a utility *strlen* replacement for UCS-2:
// ```
// int wideStrlen(const unsigned short *str)
// {
//     int len = 0;
//
//     while (*str++) {
//         ++len;
//     }
//
//     return len;
// }
// ```
// Now, some arbitrary function that calls `wideStrlen`:
// ```
// void functionRequiringUcs2(const unsigned short *str, bsl::size_t strLen)
// {
//     // Would probably do something more reasonable here.
//
//     assert(wideStrlen(str) + 1 == static_cast<int>(strLen));
// }
// ```
// Finally, we can take some UTF-8 as an input and call
// `functionRequiringUcs2`:
// ```
// void processUtf8(const char *strU8)
// {
//     unsigned short buffer[1024];  // some "large enough" size
//     bsl::size_t    buffSize     = sizeof buffer / sizeof *buffer;
//     bsl::size_t    charsWritten = 0;
//
//     int result =
//               BloombergLP::bdlde::CharConvertUcs2::utf8ToUcs2(buffer,
//                                                              buffSize,
//                                                              strU8,
//                                                              &charsWritten);
//
//     if (0 == result) {
//         functionRequiringUcs2(buffer, charsWritten);
//     }
// }
// ```
//
///Example 3: C++-Style Interface
/// - - - - - - - - - - - - - - -
// The following snippet of code illustrates a typical use of the
// `bdlde::CharConvertUcs2` struct's C++-style utility functions, converting a
// simple UTF-8 string to UCS-2.
// ```
// void loadUCS2Hello(bsl::vector<unsigned short> *result)
// {
//     int retVal =
//               BloombergLP::bdlde::CharConvertUcs2::utf8ToUcs2(result,
//                                                              "Hello");
//
//     assert( 0  == retVal);
//     assert('H' == (*result)[0]);
//     assert('e' == (*result)[1]);
//     assert('l' == (*result)[2]);
//     assert('l' == (*result)[3]);
//     assert('o' == (*result)[4]);
//     assert( 0  == (*result)[5]);
//     assert( 6  == result->size());
// }
// ```
// The following snippet of code illustrates another typical use of the
// `bdlde::CharConvertUcs2` struct's C++-style utility functions, first
// converting from UTF-8 to UCS-2, and then converting back to make sure the
// round trip returns the same value.
// ```
// void checkCppRoundTrip()
// {
//     bsl::vector<unsigned short> result;
//
//     // "&Eacute;cole", the French word for School.  &Eacute; is the HTML
//     // entity corresponding to "Unicode-E WITH ACUTE, LATIN CAPITAL LETTER".
//     int retVal =
//           BloombergLP::bdlde::CharConvertUcs2::utf8ToUcs2(&result,
//                                                          "\xc3\x89" "cole");
//
//     assert( 0   == retVal);
//     assert(0xc9 == result[0]); // Unicode-E WITH ACUTE, LATIN CAPITAL LETTER
//     assert('c'  == result[1]);
//     assert('o'  == result[2]);
//     assert('l'  == result[3]);
//     assert('e'  == result[4]);
//     assert( 0   == result[5]);
//     assert( 6   == result.size());
//
//     bsl::string    result2;
//     bsl::size_t    charsWritten = 0;
//
//     // Reversing the conversion returns the original string:
//     retVal =
//           BloombergLP::bdlde::CharConvertUcs2::ucs2ToUtf8(&result2,
//                                                          &result.front(),
//                                                          &charsWritten);
//
//     assert( 0 == retVal);
//     assert( result2 == "\xc3\x89" "cole");
//
//     // 6 characters written (including the null-terminator), and 6 bytes,
//     // since the first character takes 2 octets and the null-terminator is
//     // not counted in "length()".
//     assert( 6 == charsWritten);
//     assert( 6 == result2.length());
// }
// ```
// In this example, a UTF-8 input string is converted then returned.
// ```
// bsl::vector<unsigned short> processUtf8(const bsl::string& strU8)
// {
//     bsl::vector<unsigned short> result;
//
//     BloombergLP::bdlde::CharConvertUcs2::utf8ToUcs2(&result, strU8.c_str());
//
//     return result;
// }
// ```

#include <bdlscm_version.h>

#include <bdlde_charconvertstatus.h>

#include <bsl_cstddef.h>            // 'bsl::size_t'
#include <bsl_string.h>
#include <bsl_vector.h>

#include <bsls_libraryfeatures.h>

#include <string>                  // 'std::string', 'std::pmr::string'
#include <vector>                  // 'std::vector', 'std::pmr::vector'

namespace BloombergLP {

namespace bdlde {
                           // ======================
                           // struct CharConvertUcs2
                           // ======================

/// This `struct` provides a namespace for a suite of pure procedures to
/// convert character buffers between UTF-8 and UCS-2.  UCS-2 conversions
/// are performed to/from the full `2 ^ 16` bit space (the "UTF-16" hole
/// U+D800-U+DFFF is not treated as a special case).  Note that all C-style
/// routines in this component honor *strlcpy* semantics, meaning that all
/// returned C-style strings will be null-terminated as long as the return
/// buffer size is positive (i.e., `dstCapacity > 0`).  Note that since all
/// UCS-2 operations take place as `unsigned short`s, byte order is not
/// taken into consideration, and Byte Order Mark (BOM) characters are not
/// generated.  If a BOM is present in the input, it will be translated into
/// the output.
struct CharConvertUcs2 {

    // CLASS METHODS

    /// Load, into the specified `dstBuffer` of the specified `dstCapacity`,
    /// the result of converting the specified null-terminated UTF-8
    /// `srcString` to its UCS-2 equivalent.  Optionally specify
    /// `numCharsWritten` which (if non-zero) indicates the modifiable
    /// integer into which the number of characters written (including the
    /// null terminator) is to be loaded.  Optionally specify
    /// `errorCharacter` to be substituted for invalid (i.e., not
    /// convertible to UCS-2) input characters.  If `errorCharacter` is 0,
    /// invalid input characters are ignored (i.e., produce no corresponding
    /// output characters).  Return 0 on success and a bitwise-or of the
    /// masks specified by `CharConvertStatus::Enum` otherwise, with
    /// `CharConvertStatus::k_INVALID_INPUT_BIT` set to indicate that at
    /// least one invalid input sequence was encountered, and
    /// `CharConvertStatus::k_OUT_OF_SPACE_BIT` set to indicate that
    /// `dstCapacity` was insufficient to accommodate the output.  If
    /// `dstCapacity` was insufficient, the maximal null-terminated prefix
    /// of the properly converted result string is loaded into `dstBuffer`,
    /// and (unless null) `*numCharsWritten` is set to `dstCapacity`.  The
    /// behavior is undefined unless `0 <= dstCapacity`, `dstBuffer` refers
    /// to an array of at least `dstCapacity` elements, and `srcString` is
    /// null-terminated.  Note that if `dstCapacity` is 0, this function
    /// returns exactly 2 and `*numCharsWritten` (if specified) is loaded
    /// with 0 (since there is insufficient space for the null terminator
    /// even for an empty input string).
    static int utf8ToUcs2(unsigned short *dstBuffer,
                          bsl::size_t     dstCapacity,
                          const char     *srcString,
                          bsl::size_t    *numCharsWritten = 0,
                          unsigned short  errorCharacter  = '?');

    static int utf8ToUcs2(bsl::vector<unsigned short> *result,
                          const char                  *srcString,
                          unsigned short               errorCharacter  = '?');
    static int utf8ToUcs2(std::vector<unsigned short> *result,
                          const char                  *srcString,
                          unsigned short               errorCharacter  = '?');
#ifdef BSLS_LIBRARYFEATURES_HAS_CPP17_PMR
    static int utf8ToUcs2(
                      std::pmr::vector<unsigned short> *result,
                      const char                       *srcString,
                      unsigned short                    errorCharacter  = '?');
#endif
        // Load into the specified 'result' the conversion of the specified
        // null-terminated UTF-8 'srcString' to its null-terminated UCS-2
        // equivalent.  Optionally specify 'errorCharacter' to be substituted
        // for invalid (i.e., not convertible to UCS-2) input characters.  If
        // 'errorCharacter' is 0, invalid input characters are ignored (i.e.,
        // produce no corresponding output characters).  Return 0 on success
        // and 'CharConvertStatus::k_INVALILD_CHARS_BIT' otherwise, meaning
        // that at least one sequence of characters was encountered that could
        // not be translated to UCS-2.  If 'result & 1' is non-zero, one or
        // more input characters are invalid (in which case the conversion
        // continues).  The behavior is undefined unless 'srcString' is
        // null-terminated.  Note that the null-terminating word counts towards
        // 'result->size()'.

    /// Load, into the specified `dstBuffer` of the specified `dstCapacity`,
    /// the result of converting the specified null-terminated UCS-2
    /// `srcString` to its UTF-8 equivalent.  Optionally specify
    /// `numCharsWritten` which (if not 0) indicates the modifiable integer
    /// into which the number of *UTF-8 characters* written (including the
    /// null terminator) is to be loaded.  Optionally specify
    /// `numBytesWritten` which (if not 0) indicates the modifiable integer
    /// into which the number of *bytes* written (including the null
    /// terminator) is to be loaded.  Return 0 on success and a bitwise-or
    /// of the masks specified by `CharConvertStatus::Enum` otherwise,
    /// with `CharConvertStatus::k_INVALID_INPUT_BIT` set to indicate that
    /// at least one invalid input sequence was encountered, and
    /// `CharConvertStatus::k_OUT_OF_SPACE_BIT` set to indicate that
    /// `dstCapacity` was insufficient to accommodate the output.  If
    /// `dstCapacity` was insufficient, the maximal null-terminated prefix
    /// of the properly converted result string is loaded into `dstBuffer`.
    /// The behavior is undefined unless `0 <= dstCapacity`, `dstBuffer`
    /// refers to an array of at least `dstCapacity` elements, and
    /// `srcString` is null-terminated.  Note that if `dstCapacity` is 0,
    /// this function returns exactly 2 and `*numCharsWritten` and
    /// `*numBytesWritten` (if not null) are loaded with 0 (since there is
    /// insufficient space for the null terminator even for an empty input
    /// string).  Also note that since UTF-8 is a variable-length encoding,
    /// it is possible for `numBytesWritten` to be greater than
    /// `numCharsWritten`, and therefore that an input `srcString` of
    /// `dstCapacity - 1` *characters* may not fit into `dstBuffer`.
    static int ucs2ToUtf8(char                 *dstBuffer,
                          bsl::size_t           dstCapacity,
                          const unsigned short *srcString,
                          bsl::size_t          *numCharsWritten = 0,
                          bsl::size_t          *numBytesWritten = 0);

    static int ucs2ToUtf8(bsl::string          *result,
                          const unsigned short *srcString,
                          bsl::size_t          *numCharsWritten = 0);
    static int ucs2ToUtf8(std::string          *result,
                          const unsigned short *srcString,
                          bsl::size_t          *numCharsWritten = 0);
#ifdef BSLS_LIBRARYFEATURES_HAS_CPP17_PMR_STRING
    static int ucs2ToUtf8(std::pmr::string     *result,
                          const unsigned short *srcString,
                          bsl::size_t          *numCharsWritten = 0);
#endif
        // Load, into the specified 'result', the conversion of the specified
        // null-terminated UCS-2 'srcString' to its UTF-8 equivalent.
        // Optionally specify 'numCharsWritten' which (if not 0) indicates the
        // modifiable integer into which the number of *characters* written
        // (including the null terminator) is to be loaded.  Return 0 on
        // success and 'CharConvertStatus::k_INVALILD_CHARS_BIT' otherwise,
        // meaning that at least one sequence of characters was encountered
        // that could not be translated to UTF-8.  The behavior is undefined
        // unless 'srcString' is null-terminated.  Note that the
        // null-terminating character is not counted in 'result->length()'.
        // Also note that this function does not currently implement failure
        // modes; however, this could change if UTF-8 input validation is
        // added.
};

}  // close package namespace
}  // close enterprise namespace

#endif

// ----------------------------------------------------------------------------
// Copyright 2015 Bloomberg Finance L.P.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------- END-OF-FILE ----------------------------------
