Mirror-Tools
/
shaka-packager
mirror of https://github.com/shaka-project/shaka-packager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
shaka-packager/base/i18n/icu_string_conversions.cc

293 lines
11 KiB
C++
Raw Blame History

// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "base/i18n/icu_string_conversions.h"
#include <vector>
#include "base/basictypes.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/strings/string_util.h"
#include "base/strings/utf_string_conversions.h"
#include "third_party/icu/source/common/unicode/ucnv.h"
#include "third_party/icu/source/common/unicode/ucnv_cb.h"
#include "third_party/icu/source/common/unicode/ucnv_err.h"
#include "third_party/icu/source/common/unicode/unorm.h"
#include "third_party/icu/source/common/unicode/ustring.h"
namespace base {
namespace {
// ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
// in source/common/ucnv_err.c.
// Copyright (c) 1995-2006 International Business Machines Corporation
// and others
//
// All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, and/or
// sell copies of the Software, and to permit persons to whom the Software
// is furnished to do so, provided that the above copyright notice(s) and
// this permission notice appear in all copies of the Software and that
// both the above copyright notice(s) and this permission notice appear in
// supporting documentation.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
// OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
// INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
// OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
// OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
// OR PERFORMANCE OF THIS SOFTWARE.
//
// Except as contained in this notice, the name of a copyright holder
// shall not be used in advertising or otherwise to promote the sale, use
// or other dealings in this Software without prior written authorization
// of the copyright holder.
// ___________________________________________________________________________
//
// All trademarks and registered trademarks mentioned herein are the property
// of their respective owners.
void ToUnicodeCallbackSubstitute(const void* context,
UConverterToUnicodeArgs *to_args,
const char* code_units,
int32_t length,
UConverterCallbackReason reason,
UErrorCode * err) {
static const UChar kReplacementChar = 0xFFFD;
if (reason <= UCNV_IRREGULAR) {
if (context == NULL ||
(*(reinterpret_cast<const char*>(context)) == 'i' &&
reason == UCNV_UNASSIGNED)) {
*err = U_ZERO_ERROR;
ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
}
// else the caller must have set the error code accordingly.
}
// else ignore the reset, close and clone calls.
}
bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src,
int uchar_len, OnStringConversionError::Type on_error,
std::string* encoded) {
int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len,
ucnv_getMaxCharSize(converter));
encoded->resize(encoded_max_length);
UErrorCode status = U_ZERO_ERROR;
// Setup our error handler.
switch (on_error) {
case OnStringConversionError::FAIL:
ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, 0,
NULL, NULL, &status);
break;
case OnStringConversionError::SKIP:
case OnStringConversionError::SUBSTITUTE:
ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, 0,
NULL, NULL, &status);
break;
default:
NOTREACHED();
}
// ucnv_fromUChars returns size not including terminating null
int actual_size = ucnv_fromUChars(converter, &(*encoded)[0],
encoded_max_length, uchar_src, uchar_len, &status);
encoded->resize(actual_size);
ucnv_close(converter);
if (U_SUCCESS(status))
return true;
encoded->clear(); // Make sure the output is empty on error.
return false;
}
// Set up our error handler for ToUTF-16 converters
void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
UConverter* converter, UErrorCode* status) {
switch (on_error) {
case OnStringConversionError::FAIL:
ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, 0,
NULL, NULL, status);
break;
case OnStringConversionError::SKIP:
ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, 0,
NULL, NULL, status);
break;
case OnStringConversionError::SUBSTITUTE:
ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, 0,
NULL, NULL, status);
break;
default:
NOTREACHED();
}
}
inline UConverterType utf32_platform_endian() {
#if U_IS_BIG_ENDIAN
return UCNV_UTF32_BigEndian;
#else
return UCNV_UTF32_LittleEndian;
#endif
}
} // namespace
// Codepage <-> Wide/UTF-16 ---------------------------------------------------
bool UTF16ToCodepage(const string16& utf16,
const char* codepage_name,
OnStringConversionError::Type on_error,
std::string* encoded) {
encoded->clear();
UErrorCode status = U_ZERO_ERROR;
UConverter* converter = ucnv_open(codepage_name, &status);
if (!U_SUCCESS(status))
return false;
return ConvertFromUTF16(converter, utf16.c_str(),
static_cast<int>(utf16.length()), on_error, encoded);
}
bool CodepageToUTF16(const std::string& encoded,
const char* codepage_name,
OnStringConversionError::Type on_error,
string16* utf16) {
utf16->clear();
UErrorCode status = U_ZERO_ERROR;
UConverter* converter = ucnv_open(codepage_name, &status);
if (!U_SUCCESS(status))
return false;
// Even in the worst case, the maximum length in 2-byte units of UTF-16
// output would be at most the same as the number of bytes in input. There
// is no single-byte encoding in which a character is mapped to a
// non-BMP character requiring two 2-byte units.
//
// Moreover, non-BMP characters in legacy multibyte encodings
// (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
// BOCU and SCSU, but we don't care about them.
size_t uchar_max_length = encoded.length() + 1;
SetUpErrorHandlerForToUChars(on_error, converter, &status);
scoped_ptr<char16[]> buffer(new char16[uchar_max_length]);
int actual_size = ucnv_toUChars(converter, buffer.get(),
static_cast<int>(uchar_max_length), encoded.data(),
static_cast<int>(encoded.length()), &status);
ucnv_close(converter);
if (!U_SUCCESS(status)) {
utf16->clear(); // Make sure the output is empty on error.
return false;
}
utf16->assign(buffer.get(), actual_size);
return true;
}
bool WideToCodepage(const std::wstring& wide,
const char* codepage_name,
OnStringConversionError::Type on_error,
std::string* encoded) {
#if defined(WCHAR_T_IS_UTF16)
return UTF16ToCodepage(wide, codepage_name, on_error, encoded);
#elif defined(WCHAR_T_IS_UTF32)
encoded->clear();
UErrorCode status = U_ZERO_ERROR;
UConverter* converter = ucnv_open(codepage_name, &status);
if (!U_SUCCESS(status))
return false;
int utf16_len;
// When wchar_t is wider than UChar (16 bits), transform |wide| into a
// UChar* string. Size the UChar* buffer to be large enough to hold twice
// as many UTF-16 code units (UChar's) as there are Unicode code points,
// in case each code points translates to a UTF-16 surrogate pair,
// and leave room for a NUL terminator.
std::vector<UChar> utf16(wide.length() * 2 + 1);
u_strFromUTF32(&utf16[0], utf16.size(), &utf16_len,
reinterpret_cast<const UChar32*>(wide.c_str()),
wide.length(), &status);
DCHECK(U_SUCCESS(status)) << "failed to convert wstring to UChar*";
return ConvertFromUTF16(converter, &utf16[0], utf16_len, on_error, encoded);
#endif // defined(WCHAR_T_IS_UTF32)
}
bool CodepageToWide(const std::string& encoded,
const char* codepage_name,
OnStringConversionError::Type on_error,
std::wstring* wide) {
#if defined(WCHAR_T_IS_UTF16)
return CodepageToUTF16(encoded, codepage_name, on_error, wide);
#elif defined(WCHAR_T_IS_UTF32)
wide->clear();
UErrorCode status = U_ZERO_ERROR;
UConverter* converter = ucnv_open(codepage_name, &status);
if (!U_SUCCESS(status))
return false;
// The maximum length in 4 byte unit of UTF-32 output would be
// at most the same as the number of bytes in input. In the worst
// case of GB18030 (excluding escaped-based encodings like ISO-2022-JP),
// this can be 4 times larger than actually needed.
size_t wchar_max_length = encoded.length() + 1;
SetUpErrorHandlerForToUChars(on_error, converter, &status);
scoped_ptr<wchar_t[]> buffer(new wchar_t[wchar_max_length]);
int actual_size = ucnv_toAlgorithmic(utf32_platform_endian(), converter,
reinterpret_cast<char*>(buffer.get()),
static_cast<int>(wchar_max_length) * sizeof(wchar_t), encoded.data(),
static_cast<int>(encoded.length()), &status);
ucnv_close(converter);
if (!U_SUCCESS(status)) {
wide->clear(); // Make sure the output is empty on error.
return false;
}
// actual_size is # of bytes.
wide->assign(buffer.get(), actual_size / sizeof(wchar_t));
return true;
#endif // defined(WCHAR_T_IS_UTF32)
}
bool ConvertToUtf8AndNormalize(const std::string& text,
const std::string& charset,
std::string* result) {
result->clear();
string16 utf16;
if (!CodepageToUTF16(
text, charset.c_str(), OnStringConversionError::FAIL, &utf16))
return false;
UErrorCode status = U_ZERO_ERROR;
size_t max_length = utf16.length() + 1;
string16 normalized_utf16;
scoped_ptr<char16[]> buffer(new char16[max_length]);
int actual_length = unorm_normalize(
utf16.c_str(), utf16.length(), UNORM_NFC, 0,
buffer.get(), static_cast<int>(max_length), &status);
if (!U_SUCCESS(status))
return false;
normalized_utf16.assign(buffer.get(), actual_length);
return UTF16ToUTF8(normalized_utf16.data(),
normalized_utf16.length(), result);
}
} // namespace base
Reference in New Issue View Git Blame Copy Permalink