diff --git a/CMakeLists.txt b/CMakeLists.txt index d605f6c3e..fc815b8e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,9 +127,6 @@ add_feature_info(tiff WITH_TIFF "build TIFF support") option(WITH_MNG "Build MNG support" ON) add_feature_info(mng WITH_MNG "build MNG support") -option(WITH_ICONV "Build Iconv support" ON) -add_feature_info(iconv WITH_ICONV "build Iconv support") - option(WITH_DBUS "Build DBus support" ON) add_feature_info(dbus WITH_DBUS "build DBus support") @@ -349,14 +346,6 @@ set_package_properties(MNG PROPERTIES TYPE RECOMMENDED ) -find_package(Iconv) -set_package_properties(Iconv PROPERTIES - PURPOSE "Iconv support" - DESCRIPTION "iconv() implementation, for use on systems which don't have one or lack Unicode support" - URL "http://www.gnu.org/software/libiconv/" - TYPE RECOMMENDED -) - find_package(Cups) set_package_properties(Cups PROPERTIES PURPOSE "Required for printing support" @@ -497,11 +486,6 @@ if(NOT WITH_ACCESSIBILITY) katie_definition(-DQT_NO_ACCESSIBILITY) endif() -# optional packages with interface definition only for convenience -if(NOT WITH_ICONV OR NOT ICONV_FOUND) - katie_definition(-DQT_NO_ICONV) -endif() - # keep in sync with the gui component CMake file, only definitions setup is needed here foreach(x11ext Xshape Xinerama Xrandr Xrender Xfixes Xcursor Xinput) if(NOT X11_${x11ext}_FOUND) diff --git a/src/core/codecs/codecs.cmake b/src/core/codecs/codecs.cmake index d1d37dd0d..26746cc7f 100644 --- a/src/core/codecs/codecs.cmake +++ b/src/core/codecs/codecs.cmake @@ -1,39 +1,14 @@ set(CORE_HEADERS ${CORE_HEADERS} - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qisciicodec_p.h - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qlatincodec_p.h - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qsimplecodec_p.h ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodec_p.h ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodec.h - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtsciicodec_p.h - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qutfcodec_p.h + ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qicucodec_p.h ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodecplugin.h ) set(CORE_SOURCES ${CORE_SOURCES} - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qisciicodec.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qlatincodec.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qsimplecodec.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodec.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtsciicodec.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qutfcodec.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodecplugin.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qfontlaocodec.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qicucodec.cpp ) - -if(WITH_ICONV AND ICONV_FOUND) - set(CORE_HEADERS - ${CORE_HEADERS} - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qiconvcodec_p.h - ) - set(CORE_SOURCES - ${CORE_SOURCES} - ${CMAKE_CURRENT_SOURCE_DIR}/codecs/qiconvcodec.cpp - ) - set(EXTRA_CORE_LIBS - ${EXTRA_CORE_LIBS} - ${ICONV_LIBRARIES} - ) - include_directories(${ICONV_INCLUDES}) -endif() diff --git a/src/core/codecs/qfontlaocodec.cpp b/src/core/codecs/qfontlaocodec.cpp deleted file mode 100644 index b93f606e6..000000000 --- a/src/core/codecs/qfontlaocodec.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qfontlaocodec_p.h" -#include "qlist.h" - -#ifndef QT_NO_CODECS -#ifndef QT_NO_BIG_CODECS - -QT_BEGIN_NAMESPACE - -static unsigned char const unicode_to_mulelao[256] = - { - // U+0E80 - 0x00, 0xa1, 0xa2, 0x00, 0xa4, 0x00, 0x00, 0xa7, - 0xa8, 0x00, 0xaa, 0x00, 0x00, 0xad, 0x00, 0x00, - // U+0E90 - 0x00, 0x00, 0x00, 0x00, 0xb4, 0xb5, 0xb6, 0xb7, - 0x00, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - // U+0EA0 - 0x00, 0xc1, 0xc2, 0xc3, 0x00, 0xc5, 0x00, 0xc7, - 0x00, 0x00, 0xca, 0xcb, 0x00, 0xcd, 0xce, 0xcf, - // U+0EB0 - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0x00, 0xdb, 0xdc, 0xdd, 0x00, 0x00, - // U+0EC0 - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0x00, 0xe6, 0x00, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0x00, 0x00, - // U+0ED0 - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0x00, 0x00, 0xfc, 0xfd, 0x00, 0x00, - // U+0EE0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // U+0EF0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - }; - - -QFontLaoCodec::~QFontLaoCodec() -{ -} - -QByteArray QFontLaoCodec::name() const -{ - return "mulelao-1"; -} - -int QFontLaoCodec::mibEnum() const -{ - return -4242; -} - -QString QFontLaoCodec::convertToUnicode(const char *, int, ConverterState *) const -{ - return QString(); -} - -QByteArray QFontLaoCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *) const -{ - QByteArray rstring(len, Qt::Uninitialized); - uchar *rdata = (uchar *) rstring.data(); - const QChar *sdata = uc; - int i = 0; - for (; i < len; ++i, ++sdata, ++rdata) { - if (sdata->unicode() < 0x80) { - *rdata = (uchar) sdata->unicode(); - } else if (sdata->unicode() >= 0x0e80 && sdata->unicode() <= 0x0eff) { - uchar lao = unicode_to_mulelao[sdata->unicode() - 0x0e80]; - if (lao) - *rdata = lao; - else - *rdata = 0; - } else { - *rdata = 0; - } - } - return rstring; -} - -QT_END_NAMESPACE - -#endif // QT_NO_BIG_CODECS -#endif // QT_NO_CODECS diff --git a/src/core/codecs/qfontlaocodec_p.h b/src/core/codecs/qfontlaocodec_p.h deleted file mode 100644 index e9eeebe34..000000000 --- a/src/core/codecs/qfontlaocodec_p.h +++ /dev/null @@ -1,70 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QFONTLAOCODEC_P_H -#define QFONTLAOCODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists for the convenience -// of qfontencodings_x11.cpp and qfont_x11.cpp. This header file may -// change from version to version without notice, or even be removed. -// -// We mean it. -// - -#include "QtCore/qtextcodec.h" - -QT_BEGIN_NAMESPACE - -#ifndef QT_NO_CODECS - -class Q_CORE_EXPORT QFontLaoCodec : public QTextCodec -{ -public: - ~QFontLaoCodec(); - - QByteArray name() const; - int mibEnum() const; - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; -}; - -#endif // QT_NO_CODECS - -QT_END_NAMESPACE - -#endif // QFONTLAOCODEC_P_H diff --git a/src/core/codecs/qiconvcodec.cpp b/src/core/codecs/qiconvcodec.cpp deleted file mode 100644 index 904dc6612..000000000 --- a/src/core/codecs/qiconvcodec.cpp +++ /dev/null @@ -1,447 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qiconvcodec_p.h" -#include "qtextcodec_p.h" -#include "qlibrary.h" -#include "qdebug.h" - -#include -#include -#include - -// unistd.h is needed for the _XOPEN_UNIX macro -#include -#if defined(_XOPEN_UNIX) && !defined(Q_OS_OSF) -# include -#endif - -#if defined(Q_OS_HPUX) -# define NO_BOM -# define UTF16 "ucs2" -#elif defined(Q_OS_AIX) -# define NO_BOM -# define UTF16 "UCS-2" -#elif defined(Q_OS_FREEBSD) || (defined(Q_OS_LINUX) && !defined(__GLIBC__)) -# define NO_BOM -# if Q_BYTE_ORDER == Q_BIG_ENDIAN -# define UTF16 "UTF-16BE" -# else -# define UTF16 "UTF-16LE" -# endif -#else -# define UTF16 "UTF-16" -#endif - -QT_BEGIN_NAMESPACE - -extern bool qt_locale_initialized; - -QIconvCodec::QIconvCodec() - : utf16Codec(Q_NULLPTR) -{ - utf16Codec = QTextCodec::codecForMib(1015); - Q_ASSERT_X(utf16Codec != Q_NULLPTR, - "QIconvCodec::convertToUnicode", - "internal error, UTF-16 codec not found"); - if (!utf16Codec) { - fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n"); - utf16Codec = Q_NULLPTR; - } -} - -QIconvCodec::~QIconvCodec() -{ -} - -QIconvCodec::IconvState::IconvState(iconv_t x) - : buffer(array), bufferLen(sizeof array), cd(x) -{ -} - -QIconvCodec::IconvState::~IconvState() -{ - if (cd != reinterpret_cast(-1)) - iconv_close(cd); - if (buffer != array) - delete[] buffer; -} - -void QIconvCodec::IconvState::saveChars(const char *c, int count) -{ - if (count > bufferLen) { - if (buffer != array) - delete[] buffer; - buffer = new char[bufferLen = count]; - } - - memcpy(buffer, c, count); -} - -thread_local QIconvCodec::IconvState* toUnicodeState = Q_NULLPTR; - -QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const -{ - if (!utf16Codec) - return QString::fromLatin1(chars, len); - - int invalidCount = 0; - int remainingCount = 0; - char *remainingBuffer = 0; - IconvState *temporaryState = 0; - IconvState *&state = (qt_locale_initialized && toUnicodeState) ? toUnicodeState : temporaryState; - - if (!state) { - // first time, create the state - iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0); - if (cd == reinterpret_cast(-1)) { - static int reported = 0; - if (!reported++) { - fprintf(stderr, - "QIconvCodec::convertToUnicode: using Latin-1 for conversion, iconv_open failed\n"); - } - return QString::fromLatin1(chars, len); - } - - state = new IconvState(cd); - } - - size_t inBytesLeft = len; - // best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM - char *inBytes = const_cast(chars); - - QByteArray in; - if (remainingCount) { - // we have to prepend the remaining bytes from the previous conversion - inBytesLeft += remainingCount; - in.resize(inBytesLeft); - inBytes = in.data(); - - memcpy(in.data(), remainingBuffer, remainingCount); - memcpy(in.data() + remainingCount, chars, len); - - remainingCount = 0; - } - - size_t outBytesLeft = len * 2 + 2; - QByteArray ba(outBytesLeft, Qt::Uninitialized); - char *outBytes = ba.data(); - do { - size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft); - if (ret == (size_t) -1) { - if (errno == E2BIG) { - int offset = ba.size() - outBytesLeft; - ba.resize(ba.size() * 2); - outBytes = ba.data() + offset; - outBytesLeft = ba.size() - offset; - - continue; - } - - if (errno == EILSEQ) { - // conversion stopped because of an invalid character in the sequence - ++invalidCount; - } else if (errno == EINVAL && convState) { - // conversion stopped because the remaining inBytesLeft make up - // an incomplete multi-byte sequence; save them for later - state->saveChars(inBytes, inBytesLeft); - remainingCount = inBytesLeft; - break; - } - - if (errno == EILSEQ || errno == EINVAL) { - // skip the next character - ++inBytes; - --inBytesLeft; - continue; - } - - // some other error - // note, cannot use qWarning() since we are implementing the codecForLocale :) - perror("QIconvCodec::convertToUnicode: using Latin-1 for conversion, iconv failed"); - - if (!convState) { - // reset state - iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft); - } - - delete temporaryState; - return QString::fromLatin1(chars, len); - } - } while (inBytesLeft != 0); - - QString s; - - if (convState) { - s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft, &state->internalState); - - convState->invalidChars = invalidCount; - convState->remainingChars = remainingCount; - } else { - s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft); - - // reset state - iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft); - } - - delete temporaryState; - return s; -} - -thread_local QIconvCodec::IconvState* fromUnicodeState = Q_NULLPTR; - -static bool setByteOrder(iconv_t cd) -{ -#if !defined(NO_BOM) - // give iconv() a BOM - char buf[4]; - ushort bom[] = { QChar::ByteOrderMark }; - - char *outBytes = buf; - char *inBytes = reinterpret_cast(bom); - size_t outBytesLeft = sizeof buf; - size_t inBytesLeft = sizeof bom; - - if (iconv(cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) { - return false; - } -#endif // NO_BOM - - return true; -} - -QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const -{ - char *inBytes; - char *outBytes; - size_t inBytesLeft; - - IconvState *temporaryState = 0; - IconvState *&state = (qt_locale_initialized && fromUnicodeState) ? fromUnicodeState : temporaryState; - if (!state) { - iconv_t cd = QIconvCodec::createIconv_t(0, UTF16); - if (cd != reinterpret_cast(-1)) { - if (!setByteOrder(cd)) { - perror("QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv failed for BOM"); - - iconv_close(cd); - cd = reinterpret_cast(-1); - - return QString(uc, len).toLatin1(); - } - } - state = new IconvState(cd); - } - if (state->cd == reinterpret_cast(-1)) { - static int reported = 0; - if (!reported++) { - fprintf(stderr, - "QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv_open failed\n"); - } - delete temporaryState; - return QString(uc, len).toLatin1(); - } - - size_t outBytesLeft = len; - QByteArray ba(outBytesLeft, Qt::Uninitialized); - outBytes = ba.data(); - - // now feed iconv() the real data - inBytes = const_cast(reinterpret_cast(uc)); - inBytesLeft = len * sizeof(QChar); - - QByteArray in; - if (convState && convState->remainingChars) { - // we have one surrogate char to be prepended - in.resize(sizeof(QChar) + len); - inBytes = in.data(); - - QChar remaining = convState->state_data[0]; - memcpy(in.data(), &remaining, sizeof(QChar)); - memcpy(in.data() + sizeof(QChar), uc, inBytesLeft); - - inBytesLeft += sizeof(QChar); - convState->remainingChars = 0; - } - - int invalidCount = 0; - while (inBytesLeft != 0) { - if (iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) { - if (errno == EINVAL && convState) { - // buffer ends in a surrogate - Q_ASSERT(inBytesLeft == 2); - convState->remainingChars = 1; - convState->state_data[0] = uc[len - 1].unicode(); - break; - } - - switch (errno) { - case EILSEQ: - ++invalidCount; - // fall through - case EINVAL: - { - inBytes += sizeof(QChar); - inBytesLeft -= sizeof(QChar); - break; - } - case E2BIG: - { - int offset = ba.size() - outBytesLeft; - ba.resize(ba.size() * 2); - outBytes = ba.data() + offset; - outBytesLeft = ba.size() - offset; - break; - } - default: - { - // note, cannot use qWarning() since we are implementing the codecForLocale :) - perror("QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv failed"); - - // reset to initial state - iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft); - - delete temporaryState; - return QString(uc, len).toLatin1(); - } - } - } - } - - // reset to initial state - iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft); - setByteOrder(state->cd); - - ba.resize(ba.size() - outBytesLeft); - - if (convState) - convState->invalidChars = invalidCount; - - delete temporaryState; - return ba; -} - -QByteArray QIconvCodec::name() const -{ - return "System"; -} - -int QIconvCodec::mibEnum() const -{ - return 0; -} - -iconv_t QIconvCodec::createIconv_t(const char *to, const char *from) -{ - Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0)); - - // both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string - const char *codeset = ""; - iconv_t cd = iconv_open(to ? to : codeset, from ? from : codeset); - -#if defined(_XOPEN_UNIX) && !defined(Q_OS_OSF) - if (cd == (iconv_t) -1) { - codeset = nl_langinfo(CODESET); - if (codeset) - cd = iconv_open(to ? to : codeset, from ? from : codeset); - } -#endif - - if (cd == (iconv_t) -1) { - // Very poorly defined and followed standards causes lots of - // code to try to get all the cases... This logic is - // duplicated in QTextCodec, so if you change it here, change - // it there too. - - // Try to determine locale codeset from locale name assigned to - // LC_CTYPE category. - - // First part is getting that locale name. First try setlocale() which - // definitely knows it, but since we cannot fully trust it, get ready - // to fall back to environment variables. - char * ctype = qstrdup(setlocale(LC_CTYPE, 0)); - - // Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG - // environment variables. - char * lang = qstrdup(qgetenv("LC_ALL").constData()); - if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) { - if (lang) delete [] lang; - lang = qstrdup(qgetenv("LC_CTYPE").constData()); - } - if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) { - if (lang) delete [] lang; - lang = qstrdup(qgetenv("LANG").constData()); - } - - // Now try these in order: - // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15) - // 2. CODESET from lang if it contains a .CODESET part - // 3. ctype (maybe the locale is named "ISO-8859-1" or something) - // 4. locale (ditto) - // 5. check for "@euro" - - // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15) - codeset = ctype ? strchr(ctype, '.') : 0; - if (codeset && *codeset == '.') { - ++codeset; - cd = iconv_open(to ? to : codeset, from ? from : codeset); - } - - // 2. CODESET from lang if it contains a .CODESET part - codeset = lang ? strchr(lang, '.') : 0; - if (cd == (iconv_t) -1 && codeset && *codeset == '.') { - ++codeset; - cd = iconv_open(to ? to : codeset, from ? from : codeset); - } - - // 3. ctype (maybe the locale is named "ISO-8859-1" or something) - if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0) - cd = iconv_open(to ? to : ctype, from ? from : ctype); - - - // 4. locale (ditto) - if (cd == (iconv_t) -1 && lang && *lang != 0) - cd = iconv_open(to ? to : lang, from ? from : lang); - - // 5. "@euro" - if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro"))) - cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15"); - - delete [] ctype; - delete [] lang; - } - - return cd; -} - -QT_END_NAMESPACE diff --git a/src/core/codecs/qiconvcodec_p.h b/src/core/codecs/qiconvcodec_p.h deleted file mode 100644 index af67fc493..000000000 --- a/src/core/codecs/qiconvcodec_p.h +++ /dev/null @@ -1,93 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QICONVCODEC_P_H -#define QICONVCODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists for the convenience -// of the QLibrary class. This header file may change from -// version to version without notice, or even be removed. -// -// We mean it. -// - -#include "qtextcodec.h" - -#if !defined(QT_NO_ICONV) && !defined(QT_BOOTSTRAPPED) - -#include - -QT_BEGIN_NAMESPACE - -class QIconvCodec: public QTextCodec -{ -private: - mutable QTextCodec *utf16Codec; - -public: - QIconvCodec(); - ~QIconvCodec(); - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - - QByteArray name() const; - int mibEnum() const; - - static iconv_t createIconv_t(const char *to, const char *from); - - class IconvState - { - public: - IconvState(iconv_t x); - ~IconvState(); - ConverterState internalState; - char *buffer; - int bufferLen; - iconv_t cd; - - char array[8]; - - void saveChars(const char *c, int count); - }; -}; - -QT_END_NAMESPACE - -#endif // Q_OS_UNIX && !QT_NO_ICONV && !QT_BOOTSTRAPPED - -#endif // QICONVCODEC_P_H diff --git a/src/core/codecs/qicucodec.cpp b/src/core/codecs/qicucodec.cpp new file mode 100644 index 000000000..171180e75 --- /dev/null +++ b/src/core/codecs/qicucodec.cpp @@ -0,0 +1,557 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2016-2019 Ivailo Monev +** +** This file is part of the QtCore module of the Katie Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qicucodec_p.h" + +#include "qtextcodec_p.h" +#include "qdebug.h" + +#include + +QT_BEGIN_NAMESPACE + +static inline bool qTextCodecNameMatch(const char *n, const char *h) +{ + return ucnv_compareNames(n, h) == 0; +} + +/* The list below is generated from http://www.iana.org/assignments/character-sets/ + using the snippet of code below: + +#include +#include + +int main(int argc, char **argv) +{ + QCoreApplication app(argc, argv); + + QFile file("character-sets.txt"); + file.open(QFile::ReadOnly); + QByteArray name; + int mib = -1; + QByteArray nameList; + int pos = 0; + while (!file.atEnd()) { + QByteArray s = file.readLine().trimmed(); + if (s.isEmpty()) { + if (mib != -1) { + UErrorCode error = U_ZERO_ERROR; + const char *standard_name = ucnv_getStandardName(name, "MIME", &error); + if (U_FAILURE(error) || !standard_name) { + error = U_ZERO_ERROR; + standard_name = ucnv_getStandardName(name, "IANA", &error); + } + UConverter *conv = ucnv_open(standard_name, &error); + if (!U_FAILURE(error) && conv && standard_name) { + ucnv_close(conv); + printf(" { %d, %d },\n", mib, pos); + nameList += "\""; + nameList += standard_name; + nameList += "\\0\"\n"; + pos += strlen(standard_name) + 1; + } + } + name = QByteArray(); + mib = -1; + } + if (s.startsWith("Name: ")) { + name = s.mid(5).trimmed(); + if (name.indexOf(' ') > 0) + name = name.left(name.indexOf(' ')); + } + if (s.startsWith("MIBenum:")) + mib = s.mid(8).trimmed().toInt(); + if (s.startsWith("Alias:") && s.contains("MIME")) { + name = s.mid(6).trimmed(); + name = name.left(name.indexOf(' ')).trimmed(); + } + } + qDebug() << nameList; +} +*/ + +struct MibToName { + short mib; + short index; +}; + +static const MibToName mibToName[] = { + { 3, 0 }, + { 4, 9 }, + { 5, 20 }, + { 6, 31 }, + { 7, 42 }, + { 8, 53 }, + { 9, 64 }, + { 10, 75 }, + { 11, 86 }, + { 12, 97 }, + { 13, 108 }, + { 16, 120 }, + { 17, 134 }, + { 18, 144 }, + { 30, 151 }, + { 36, 160 }, + { 37, 167 }, + { 38, 179 }, + { 39, 186 }, + { 40, 198 }, + { 57, 212 }, + { 81, 223 }, + { 82, 234 }, + { 84, 245 }, + { 85, 256 }, + { 104, 267 }, + { 105, 279 }, + { 106, 295 }, + { 109, 301 }, + { 110, 313 }, + { 111, 325 }, + { 113, 337 }, + { 114, 341 }, + { 1000, 349 }, + { 1001, 356 }, + { 1011, 363 }, + { 1012, 368 }, + { 1013, 374 }, + { 1014, 383 }, + { 1015, 392 }, + { 1016, 399 }, + { 1017, 406 }, + { 1018, 413 }, + { 1019, 422 }, + { 1020, 431 }, + { 2004, 438 }, + { 2005, 448 }, + { 2009, 472 }, + { 2013, 479 }, + { 2016, 486 }, + { 2024, 495 }, + { 2025, 505 }, + { 2026, 512 }, + { 2027, 517 }, + { 2028, 527 }, + { 2030, 534 }, + { 2033, 541 }, + { 2034, 548 }, + { 2035, 555 }, + { 2037, 562 }, + { 2038, 569 }, + { 2039, 576 }, + { 2040, 583 }, + { 2041, 590 }, + { 2043, 597 }, + { 2011, 604 }, + { 2044, 611 }, + { 2045, 618 }, + { 2010, 624 }, + { 2046, 631 }, + { 2047, 638 }, + { 2048, 645 }, + { 2049, 652 }, + { 2050, 659 }, + { 2051, 666 }, + { 2052, 673 }, + { 2053, 680 }, + { 2054, 687 }, + { 2055, 694 }, + { 2056, 701 }, + { 2062, 708 }, + { 2063, 715 }, + { 2084, 723 }, + { 2085, 730 }, + { 2086, 741 }, + { 2087, 748 }, + { 2088, 755 }, + { 2089, 762 }, + { 2091, 771 }, + { 2092, 780 }, + { 2093, 789 }, + { 2094, 798 }, + { 2095, 807 }, + { 2096, 816 }, + { 2097, 825 }, + { 2098, 834 }, + { 2099, 843 }, + { 2100, 852 }, + { 2101, 861 }, + { 2102, 872 }, + { 2250, 880 }, + { 2251, 893 }, + { 2252, 906 }, + { 2253, 919 }, + { 2254, 932 }, + { 2255, 945 }, + { 2256, 958 }, + { 2257, 971 }, + { 2258, 984 }, + { 2259, 997 }, +}; +int mibToNameSize = sizeof(mibToName)/sizeof(MibToName); + +static const char mibToNameTable[] = + "US-ASCII\0" + "ISO-8859-1\0" + "ISO-8859-2\0" + "ISO-8859-3\0" + "ISO-8859-4\0" + "ISO-8859-5\0" + "ISO-8859-6\0" + "ISO-8859-7\0" + "ISO-8859-8\0" + "ISO-8859-9\0" + "ISO-8859-10\0" + "ISO-2022-JP-1\0" + "Shift_JIS\0" + "EUC-JP\0" + "US-ASCII\0" + "EUC-KR\0" + "ISO-2022-KR\0" + "EUC-KR\0" + "ISO-2022-JP\0" + "ISO-2022-JP-2\0" + "GB_2312-80\0" + "ISO-8859-6\0" + "ISO-8859-6\0" + "ISO-8859-8\0" + "ISO-8859-8\0" + "ISO-2022-CN\0" + "ISO-2022-CN-EXT\0" + "UTF-8\0" + "ISO-8859-13\0" + "ISO-8859-14\0" + "ISO-8859-15\0" + "GBK\0" + "GB18030\0" + "UTF-16\0" + "UTF-32\0" + "SCSU\0" + "UTF-7\0" + "UTF-16BE\0" + "UTF-16LE\0" + "UTF-16\0" + "CESU-8\0" + "UTF-32\0" + "UTF-32BE\0" + "UTF-32LE\0" + "BOCU-1\0" + "hp-roman8\0" + "Adobe-Standard-Encoding\0" + "IBM850\0" + "IBM862\0" + "IBM-Thai\0" + "Shift_JIS\0" + "GB2312\0" + "Big5\0" + "macintosh\0" + "IBM037\0" + "IBM273\0" + "IBM277\0" + "IBM278\0" + "IBM280\0" + "IBM284\0" + "IBM285\0" + "IBM290\0" + "IBM297\0" + "IBM420\0" + "IBM424\0" + "IBM437\0" + "IBM500\0" + "cp851\0" + "IBM852\0" + "IBM855\0" + "IBM857\0" + "IBM860\0" + "IBM861\0" + "IBM863\0" + "IBM864\0" + "IBM865\0" + "IBM868\0" + "IBM869\0" + "IBM870\0" + "IBM871\0" + "IBM918\0" + "IBM1026\0" + "KOI8-R\0" + "HZ-GB-2312\0" + "IBM866\0" + "IBM775\0" + "KOI8-U\0" + "IBM00858\0" + "IBM01140\0" + "IBM01141\0" + "IBM01142\0" + "IBM01143\0" + "IBM01144\0" + "IBM01145\0" + "IBM01146\0" + "IBM01147\0" + "IBM01148\0" + "IBM01149\0" + "Big5-HKSCS\0" + "IBM1047\0" + "windows-1250\0" + "windows-1251\0" + "windows-1252\0" + "windows-1253\0" + "windows-1254\0" + "windows-1255\0" + "windows-1256\0" + "windows-1257\0" + "windows-1258\0" + "TIS-620\0"; + +/// \threadsafe +QList QIcuCodec::availableCodecs() +{ + QList codecs; + int n = ucnv_countAvailable(); + for (int i = 0; i < n; ++i) { + const char *name = ucnv_getAvailableName(i); + + UErrorCode error = U_ZERO_ERROR; + const char *standardName = ucnv_getStandardName(name, "MIME", &error); + if (U_FAILURE(error) || !standardName) { + error = U_ZERO_ERROR; + standardName = ucnv_getStandardName(name, "IANA", &error); + } + if (U_FAILURE(error)) + continue; + + error = U_ZERO_ERROR; + int ac = ucnv_countAliases(standardName, &error); + if (U_FAILURE(error)) + continue; + for (int j = 0; j < ac; ++j) { + error = U_ZERO_ERROR; + const char *alias = ucnv_getAlias(standardName, j, &error); + if (U_FAILURE(error)) + continue; + codecs += alias; + } + } + + return codecs; +} + +/// \threadsafe +QList QIcuCodec::availableMibs() +{ + QList mibs; + mibs.reserve(mibToNameSize + 1); + for (int i = 0; i < mibToNameSize; ++i) + mibs += mibToName[i].mib; + + return mibs; +} + +QIcuCodec::QIcuCodec(const char *name) + : m_name(name) +{ +} + +QIcuCodec::~QIcuCodec() +{ +} + +UConverter *QIcuCodec::getConverter(QTextCodec::ConverterState *state) const +{ + UConverter *conv = Q_NULLPTR; + if (state) { + if (!state->d) { + // first time + UErrorCode error = U_ZERO_ERROR; + state->d = ucnv_open(m_name, &error); + if (Q_UNLIKELY(U_FAILURE(error))) + qWarning("getConverter(state) ucnv_open failed %s %s", m_name, u_errorName(error)); + + if (state->d) { + error = U_ZERO_ERROR; + ucnv_setSubstChars(static_cast(state->d), + state->flags & QTextCodec::ConvertInvalidToNull ? "\0" : "?", 1, &error); + if (Q_UNLIKELY(U_FAILURE(error))) + qWarning("getConverter(state) ucnv_setSubstChars failed %s %s", m_name, u_errorName(error)); + + conv = static_cast(state->d); + } + } + } + if (!conv) { + // stateless conversion + UErrorCode error = U_ZERO_ERROR; + conv = ucnv_open(m_name, &error); + if (Q_UNLIKELY(U_FAILURE(error))) + qWarning("getConverter(no state) ucnv_open failed %s %s", m_name, u_errorName(error)); + + if (conv) { + error = U_ZERO_ERROR; + ucnv_setSubstChars(conv, "?", 1, &error); + if (Q_UNLIKELY(U_FAILURE(error))) + qWarning("getConverter(no state) ucnv_setSubstChars failed %s %s", m_name, u_errorName(error)); + } + } + return conv; +} + +QString QIcuCodec::convertToUnicode(const char *chars, int length, QTextCodec::ConverterState *state) const +{ + UConverter *conv = getConverter(state); + + QString string(length + 2, Qt::Uninitialized); + + bool failed = false; + const char *end = chars + length; + int convertedChars = 0; + UErrorCode error = U_ZERO_ERROR; + while (1) { + UChar *uc = reinterpret_cast(string.data()); + const UChar *ucEnd = uc + string.length(); + uc += convertedChars; + ucnv_toUnicode(conv, + &uc, ucEnd, + &chars, end, + 0, false, &error); + if (Q_UNLIKELY(U_FAILURE(error) && error != U_BUFFER_OVERFLOW_ERROR)) { + qWarning("convertToUnicode failed: %s", u_errorName(error)); + failed = true; + break; + } + + convertedChars = uc - reinterpret_cast(string.constData()); + if (chars >= end) + break; + string.resize(string.length()*2); + } + string.resize(convertedChars); + + if (failed && state) { + error = U_ZERO_ERROR; + char errorbytes[10]; + int8_t invalidlen = 0; + ucnv_getInvalidChars(conv, + errorbytes, + &invalidlen, + &error); + state->invalidChars = invalidlen; + } + + if (!state) + ucnv_close(conv); + return string; +} + + +QByteArray QIcuCodec::convertFromUnicode(const QChar *unicode, int length, QTextCodec::ConverterState *state) const +{ + UConverter *conv = getConverter(state); + + int requiredLength = UCNV_GET_MAX_BYTES_FOR_STRING(length, ucnv_getMaxCharSize(conv)); + QByteArray string(requiredLength, Qt::Uninitialized); + + bool failed = false; + UErrorCode error = U_ZERO_ERROR; + const UChar *uc = reinterpret_cast(unicode); + const UChar *end = uc + length; + int convertedChars = 0; + while (1) { + char *ch = (char *)string.data(); + char *chEnd = ch + string.length(); + ch += convertedChars; + ucnv_fromUnicode(conv, + &ch, chEnd, + &uc, end, + 0, false, &error); + if (Q_UNLIKELY(U_FAILURE(error))) { + qWarning("convertFromUnicode failed: %s", u_errorName(error)); + failed = true; + break; + } + convertedChars = ch - string.data(); + if (uc >= end) + break; + string.resize(string.length()*2); + } + string.resize(convertedChars); + + if (failed && state) { + error = U_ZERO_ERROR; + char errorbytes[10]; + int8_t invalidlen = 0; + ucnv_getInvalidChars(conv, + errorbytes, + &invalidlen, + &error); + state->invalidChars = invalidlen; + } + + if (!state) + ucnv_close(conv); + + return string; +} + + +QByteArray QIcuCodec::name() const +{ + return m_name; +} + + +QList QIcuCodec::aliases() const +{ + UErrorCode error = U_ZERO_ERROR; + + int n = ucnv_countAliases(m_name, &error); + + QList aliases; + for (int i = 0; i < n; ++i) { + const char *a = ucnv_getAlias(m_name, i, &error); + // skip the canonical name + if (!a || !qstrcmp(a, m_name)) + continue; + aliases += a; + } + + return aliases; +} + + +int QIcuCodec::mibEnum() const +{ + for (int i = 0; i < mibToNameSize; ++i) { + if (qTextCodecNameMatch(m_name, (mibToNameTable + mibToName[i].index))) + return mibToName[i].mib; + } + + return 0; +} + +QT_END_NAMESPACE diff --git a/src/core/codecs/qicucodec_p.h b/src/core/codecs/qicucodec_p.h new file mode 100644 index 000000000..3de398f6d --- /dev/null +++ b/src/core/codecs/qicucodec_p.h @@ -0,0 +1,82 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2016-2019 Ivailo Monev +** +** This file is part of the QtCore module of the Katie Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QICUCODEC_P_H +#define QICUCODEC_P_H + +#include "QtCore/qtextcodec.h" + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include + +#include + +#ifndef QT_NO_TEXTCODEC + +QT_BEGIN_NAMESPACE + +class QIcuCodec : public QTextCodec +{ +public: + QIcuCodec(const char *name); + ~QIcuCodec(); + + static QList availableCodecs(); + static QList availableMibs(); + + QString convertToUnicode(const char *, int, ConverterState *) const override; + QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const override; + + QByteArray name() const override; + QList aliases() const override; + int mibEnum() const override; + +private: + UConverter *getConverter(QTextCodec::ConverterState *state) const; + + const char *m_name; +}; + +QT_END_NAMESPACE + +#endif // QT_NO_TEXTCODEC + +#endif diff --git a/src/core/codecs/qisciicodec.cpp b/src/core/codecs/qisciicodec.cpp deleted file mode 100644 index 3df68dbf0..000000000 --- a/src/core/codecs/qisciicodec.cpp +++ /dev/null @@ -1,277 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qisciicodec_p.h" -#include "qlist.h" - -#ifndef QT_NO_CODECS - -QT_BEGIN_NAMESPACE - -/*! - \class QIsciiCodec - \brief The QIsciiCodec class provides conversion to and from the ISCII encoding. - - \internal -*/ - - -struct Codecs { - const char name[10]; - ushort base; -}; - -static const Codecs codecs [] = { - { "Iscii-Dev", 0x900 }, - { "Iscii-Bng", 0x980 }, - { "Iscii-Pnj", 0xa00 }, - { "Iscii-Gjr", 0xa80 }, - { "Iscii-Ori", 0xb00 }, - { "Iscii-Tml", 0xb80 }, - { "Iscii-Tlg", 0xc00 }, - { "Iscii-Knd", 0xc80 }, - { "Iscii-Mlm", 0xd00 } -}; - -QIsciiCodec::~QIsciiCodec() -{ -} - -QByteArray QIsciiCodec::name() const -{ - return codecs[idx].name; -} - -int QIsciiCodec::mibEnum() const -{ - /* There is no MIBEnum for Iscii */ - return -3000-idx; -} - -static const uchar inv = 0xFF; - -/* iscii range from 0xa0 - 0xff */ -static const uchar iscii_to_uni_table[0x60] = { - 0x00, 0x01, 0x02, 0x03, - 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0a, 0x0b, 0x0e, - 0x0f, 0x20, 0x0d, 0x12, - - 0x13, 0x14, 0x11, 0x15, - 0x16, 0x17, 0x18, 0x19, - 0x1a, 0x1b, 0x1c, 0x1d, - 0x1e, 0x1f, 0x20, 0x21, - - 0x22, 0x23, 0x24, 0x25, - 0x26, 0x27, 0x28, 0x29, - 0x2a, 0x2b, 0x2c, 0x2d, - 0x2e, 0x2f, 0x5f, 0x30, - - 0x31, 0x32, 0x33, 0x34, - 0x35, 0x36, 0x37, 0x38, - 0x39, inv, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, - - 0x46, 0x47, 0x48, 0x45, - 0x4a, 0x4b, 0x4c, 0x49, - 0x4d, 0x3c, 0x64, 0x00, - 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x66, 0x67, 0x68, - 0x69, 0x6a, 0x6b, 0x6c, - 0x6d, 0x6e, 0x6f, 0x00, - 0x00, 0x00, 0x00, 0x00 -}; - -static const uchar uni_to_iscii_table[0x80] = { - 0x00, 0xa1, 0xa2, 0xa3, - 0x00, 0xa4, 0xa5, 0xa6, - 0xa7, 0xa8, 0xa9, 0xaa, - 0x00, 0xae, 0xab, 0xac, - - 0xad, 0xb2, 0xaf, 0xb0, - 0xb1, 0xb3, 0xb4, 0xb5, - 0xb6, 0xb7, 0xb8, 0xb9, - 0xba, 0xbb, 0xbc, 0xbd, - - 0xbe, 0xbf, 0xc0, 0xc1, - 0xc2, 0xc3, 0xc4, 0xc5, - 0xc6, 0xc7, 0xc8, 0xc9, - 0xca, 0xcb, 0xcc, 0xcd, - - 0xcf, 0xd0, 0xd1, 0xd2, - 0xd3, 0xd4, 0xd5, 0xd6, - 0xd7, 0xd8, 0x00, 0x00, - 0xe9, 0x00, 0xda, 0xdb, - - 0xdc, 0xdd, 0xde, 0xdf, - 0x00, 0xe3, 0xe0, 0xe1, - 0xe2, 0xe7, 0xe4, 0xe5, - 0xe6, 0xe8, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x01, 0x02, 0x03, 0x04, // decomposable into the uc codes listed here + nukta - 0x05, 0x06, 0x07, 0xce, - - 0x00, 0x00, 0x00, 0x00, - 0xea, 0x08, 0xf1, 0xf2, - 0xf3, 0xf4, 0xf5, 0xf6, - 0xf7, 0xf8, 0xf9, 0xfa, - - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00 -}; - -static const uchar uni_to_iscii_pairs[] = { - 0x00, 0x00, - 0x15, 0x3c, // 0x958 - 0x16, 0x3c, // 0x959 - 0x17, 0x3c, // 0x95a - 0x1c, 0x3c, // 0x95b - 0x21, 0x3c, // 0x95c - 0x22, 0x3c, // 0x95d - 0x2b, 0x3c, // 0x95e - 0x64, 0x64 // 0x965 -}; - - -QByteArray QIsciiCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const -{ - char replacement = '?'; - bool halant = false; - if (state) { - if (state->flags & ConvertInvalidToNull) - replacement = 0; - halant = state->state_data[0]; - } - int invalid = 0; - - QByteArray result(2 * len, Qt::Uninitialized); //worst case - - uchar *ch = reinterpret_cast(result.data()); - - const int base = codecs[idx].base; - - for (int i =0; i < len; ++i) { - const ushort codePoint = uc[i].unicode(); - - /* The low 7 bits of ISCII is plain ASCII. However, we go all the - * way up to 0xA0 such that we can roundtrip with convertToUnicode()'s - * behavior. */ - if(codePoint < 0xA0) { - *ch++ = static_cast(codePoint); - continue; - } - - const int pos = codePoint - base; - if (pos > 0 && pos < 0x80) { - uchar iscii = uni_to_iscii_table[pos]; - if (iscii > 0x80) { - *ch++ = iscii; - } else if (iscii) { - const uchar *pair = uni_to_iscii_pairs + 2*iscii; - *ch++ = *pair++; - *ch++ = *pair++; - } else { - *ch++ = replacement; - ++invalid; - } - } else { - if (uc[i].unicode() == 0x200c) { // ZWNJ - if (halant) - // Consonant Halant ZWNJ -> Consonant Halant Halant - *ch++ = 0xe8; - } else if (uc[i].unicode() == 0x200d) { // ZWJ - if (halant) - // Consonant Halant ZWJ -> Consonant Halant Nukta - *ch++ = 0xe9; - } else { - *ch++ = replacement; - ++invalid; - } - } - halant = (pos == 0x4d); - } - result.truncate(ch - (uchar *)result.data()); - - if (state) { - state->invalidChars += invalid; - state->state_data[0] = halant; - } - return result; -} - -QString QIsciiCodec::convertToUnicode(const char* chars, int len, ConverterState *state) const -{ - bool halant = false; - if (state) { - halant = state->state_data[0]; - } - - QString result(len, Qt::Uninitialized); - QChar *uc = result.data(); - - const int base = codecs[idx].base; - - for (int i = 0; i < len; ++i) { - ushort ch = (uchar) chars[i]; - if (ch < 0xa0) - *uc++ = ch; - else { - ushort c = iscii_to_uni_table[ch - 0xa0]; - if (halant && (c == inv || c == 0xe9)) { - // Consonant Halant inv -> Consonant Halant ZWJ - // Consonant Halant Nukta -> Consonant Halant ZWJ - *uc++ = QChar(0x200d); - } else if (halant && c == 0xe8) { - // Consonant Halant Halant -> Consonant Halant ZWNJ - *uc++ = QChar(0x200c); - } else { - *uc++ = QChar(c+base); - } - } - halant = ((uchar)chars[i] == 0xe8); - } - result.resize(uc - result.unicode()); - - if (state) { - state->state_data[0] = halant; - } - return result; -} - -QT_END_NAMESPACE - -#endif // QT_NO_CODECS diff --git a/src/core/codecs/qisciicodec_p.h b/src/core/codecs/qisciicodec_p.h deleted file mode 100644 index 36341509e..000000000 --- a/src/core/codecs/qisciicodec_p.h +++ /dev/null @@ -1,73 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QISCIICODEC_P_H -#define QISCIICODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "QtCore/qtextcodec.h" - -QT_BEGIN_NAMESPACE - -#ifndef QT_NO_CODECS - -class QIsciiCodec : public QTextCodec { -public: - explicit QIsciiCodec(int i) : idx(i) {} - ~QIsciiCodec(); - - QByteArray name() const; - int mibEnum() const; - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - -private: - int idx; -}; - -#endif // QT_NO_CODECS - -QT_END_NAMESPACE - -#endif // QISCIICODEC_P_H diff --git a/src/core/codecs/qlatincodec.cpp b/src/core/codecs/qlatincodec.cpp deleted file mode 100644 index 5e23dd717..000000000 --- a/src/core/codecs/qlatincodec.cpp +++ /dev/null @@ -1,236 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qlatincodec_p.h" -#include "qlist.h" - -#ifndef QT_NO_TEXTCODEC - -QT_BEGIN_NAMESPACE - -QLatin1Codec::~QLatin1Codec() -{ -} - -QString QLatin1Codec::convertToUnicode(const char *chars, int len, ConverterState *) const -{ - if (!chars) - return QString(); - - return QString::fromLatin1(chars, len); -} - - -QByteArray QLatin1Codec::convertFromUnicode(const QChar *ch, int len, ConverterState *state) const -{ - const char replacement = (state && state->flags & ConvertInvalidToNull) ? 0 : '?'; - QByteArray r(len, Qt::Uninitialized); - char *d = r.data(); - int invalid = 0; - for (int i = 0; i < len; ++i) { - if (ch[i] > 0xff) { - d[i] = replacement; - ++invalid; - } else { - d[i] = (char)ch[i].cell(); - } - } - if (state) { - state->invalidChars += invalid; - } - return r; -} - -QByteArray QLatin1Codec::name() const -{ - return "ISO-8859-1"; -} - -QList QLatin1Codec::aliases() const -{ - QList list; - list << "latin1" - << "CP819" - << "IBM819" - << "iso-ir-100" - << "csISOLatin1"; - return list; -} - - -int QLatin1Codec::mibEnum() const -{ - return 4; -} - - -QLatin15Codec::~QLatin15Codec() -{ -} - -QString QLatin15Codec::convertToUnicode(const char* chars, int len, ConverterState *) const -{ - if (!chars) - return QString(); - - QString str = QString::fromLatin1(chars, len); - QChar *uc = str.data(); - while(len--) { - switch(uc->unicode()) { - case 0xa4: - *uc = 0x20ac; - break; - case 0xa6: - *uc = 0x0160; - break; - case 0xa8: - *uc = 0x0161; - break; - case 0xb4: - *uc = 0x017d; - break; - case 0xb8: - *uc = 0x017e; - break; - case 0xbc: - *uc = 0x0152; - break; - case 0xbd: - *uc = 0x0153; - break; - case 0xbe: - *uc = 0x0178; - break; - default: - break; - } - uc++; - } - return str; -} - -QByteArray QLatin15Codec::convertFromUnicode(const QChar *in, int length, ConverterState *state) const -{ - const char replacement = (state && state->flags & ConvertInvalidToNull) ? 0 : '?'; - QByteArray r(length, Qt::Uninitialized); - char *d = r.data(); - int invalid = 0; - for (int i = 0; i < length; ++i) { - uchar c; - ushort uc = in[i].unicode(); - if (uc < 0x0100) { - if (uc > 0xa3) { - switch(uc) { - case 0xa4: - case 0xa6: - case 0xa8: - case 0xb4: - case 0xb8: - case 0xbc: - case 0xbd: - case 0xbe: - c = replacement; - ++invalid; - break; - default: - c = (unsigned char) uc; - break; - } - } else { - c = (unsigned char) uc; - } - } else { - if (uc == 0x20ac) - c = 0xa4; - else if ((uc & 0xff00) == 0x0100) { - switch(uc) { - case 0x0160: - c = 0xa6; - break; - case 0x0161: - c = 0xa8; - break; - case 0x017d: - c = 0xb4; - break; - case 0x017e: - c = 0xb8; - break; - case 0x0152: - c = 0xbc; - break; - case 0x0153: - c = 0xbd; - break; - case 0x0178: - c = 0xbe; - break; - default: - c = replacement; - ++invalid; - } - } else { - c = replacement; - ++invalid; - } - } - d[i] = (char)c; - } - if (state) { - state->remainingChars = 0; - state->invalidChars += invalid; - } - return r; -} - - -QByteArray QLatin15Codec::name() const -{ - return "ISO-8859-15"; -} - -QList QLatin15Codec::aliases() const -{ - QList list; - list << "latin9"; - return list; -} - -int QLatin15Codec::mibEnum() const -{ - return 111; -} - -QT_END_NAMESPACE - -#endif // QT_NO_TEXTCODEC diff --git a/src/core/codecs/qlatincodec_p.h b/src/core/codecs/qlatincodec_p.h deleted file mode 100644 index daa8c1d30..000000000 --- a/src/core/codecs/qlatincodec_p.h +++ /dev/null @@ -1,84 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QLATINCODEC_P_H -#define QLATINCODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "QtCore/qtextcodec.h" - -QT_BEGIN_NAMESPACE - -#ifndef QT_NO_TEXTCODEC - -class QLatin1Codec : public QTextCodec -{ -public: - ~QLatin1Codec(); - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - - QByteArray name() const; - QList aliases() const; - int mibEnum() const; -}; - -class QLatin15Codec: public QTextCodec -{ -public: - ~QLatin15Codec(); - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - - QByteArray name() const; - QList aliases() const; - int mibEnum() const; -}; - -#endif // QT_NO_TEXTCODEC - -QT_END_NAMESPACE - -#endif // QLATINCODEC_P_H diff --git a/src/core/codecs/qsimplecodec.cpp b/src/core/codecs/qsimplecodec.cpp deleted file mode 100644 index b5cc8a4d3..000000000 --- a/src/core/codecs/qsimplecodec.cpp +++ /dev/null @@ -1,719 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qsimplecodec_p.h" -#include "qlist.h" - -#ifndef QT_NO_TEXTCODEC - -QT_BEGIN_NAMESPACE - -#define LAST_MIB 2004 - -static const struct { - const char *mime; - const char *aliases[7]; - int mib; - quint16 values[128]; -} unicodevalues[QSimpleTextCodec::numSimpleCodecs] = { - // from RFC 1489, ftp://ftp.isi.edu/in-notes/rfc1489.txt - { "KOI8-R", { "csKOI8R", Q_NULLPTR }, 2084, - { 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, - 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590, - 0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219/**/, 0x221A, 0x2248, - 0x2264, 0x2265, 0x00A0, 0x2321, 0x00B0, 0x00B2, 0x00B7, 0x00F7, - 0x2550, 0x2551, 0x2552, 0x0451, 0x2553, 0x2554, 0x2555, 0x2556, - 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x255C, 0x255D, 0x255E, - 0x255F, 0x2560, 0x2561, 0x0401, 0x2562, 0x2563, 0x2564, 0x2565, - 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x256B, 0x256C, 0x00A9, - 0x044E, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, - 0x0445, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, - 0x043F, 0x044F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, - 0x044C, 0x044B, 0x0437, 0x0448, 0x044D, 0x0449, 0x0447, 0x044A, - 0x042E, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, - 0x0425, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, - 0x041F, 0x042F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, - 0x042C, 0x042B, 0x0417, 0x0428, 0x042D, 0x0429, 0x0427, 0x042A } }, - // /**/ - The BULLET OPERATOR is confused. Some people think - // it should be 0x2022 (BULLET). - - // from RFC 2319, ftp://ftp.isi.edu/in-notes/rfc2319.txt - { "KOI8-U", { "KOI8-RU", Q_NULLPTR }, 2088, - { 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, - 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590, - 0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219, 0x221A, 0x2248, - 0x2264, 0x2265, 0x00A0, 0x2321, 0x00B0, 0x00B2, 0x00B7, 0x00F7, - 0x2550, 0x2551, 0x2552, 0x0451, 0x0454, 0x2554, 0x0456, 0x0457, - 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x0491, 0x255D, 0x255E, - 0x255F, 0x2560, 0x2561, 0x0401, 0x0404, 0x2563, 0x0406, 0x0407, - 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x0490, 0x256C, 0x00A9, - 0x044E, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, - 0x0445, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, - 0x043F, 0x044F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, - 0x044C, 0x044B, 0x0437, 0x0448, 0x044D, 0x0449, 0x0447, 0x044A, - 0x042E, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, - 0x0425, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, - 0x041F, 0x042F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, - 0x042C, 0x042B, 0x0417, 0x0428, 0x042D, 0x0429, 0x0427, 0x042A } }, - - // next bits generated from tables on the Unicode 2.0 CD. we can - // use these tables since this is part of the transition to using - // unicode everywhere in qt. - - // $ for A in 8 9 A B C D E F ; do for B in 0 1 2 3 4 5 6 7 8 9 A B C D E F ; do echo 0x${A}${B} 0xFFFD ; done ; done > /tmp/digits ; for a in 8859-* ; do (awk '/^0x[89ABCDEF]/{ print $1, $2 }' < $a ; cat /tmp/digits) | sort | uniq -w4 | cut -c6- | paste '-d ' - - - - - - - - | sed -e 's/ /, /g' -e 's/$/,/' -e '$ s/,$/} },/' -e '1 s/^/{ /' > ~/tmp/$a ; done - - // then I inserted the files manually. - { "ISO-8859-2", {"latin2", "iso-ir-101", "csISOLatin2", Q_NULLPTR }, 5, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x0104, 0x02D8, 0x0141, 0x00A4, 0x013D, 0x015A, 0x00A7, - 0x00A8, 0x0160, 0x015E, 0x0164, 0x0179, 0x00AD, 0x017D, 0x017B, - 0x00B0, 0x0105, 0x02DB, 0x0142, 0x00B4, 0x013E, 0x015B, 0x02C7, - 0x00B8, 0x0161, 0x015F, 0x0165, 0x017A, 0x02DD, 0x017E, 0x017C, - 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, - 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, - 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, - 0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, - 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, - 0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F, - 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, - 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9} }, - { "ISO-8859-3", { "latin3", "iso-ir-109", "csISOLatin3", Q_NULLPTR }, 6, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x0126, 0x02D8, 0x00A3, 0x00A4, 0xFFFD, 0x0124, 0x00A7, - 0x00A8, 0x0130, 0x015E, 0x011E, 0x0134, 0x00AD, 0xFFFD, 0x017B, - 0x00B0, 0x0127, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x0125, 0x00B7, - 0x00B8, 0x0131, 0x015F, 0x011F, 0x0135, 0x00BD, 0xFFFD, 0x017C, - 0x00C0, 0x00C1, 0x00C2, 0xFFFD, 0x00C4, 0x010A, 0x0108, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0xFFFD, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x0120, 0x00D6, 0x00D7, - 0x011C, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x016C, 0x015C, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0xFFFD, 0x00E4, 0x010B, 0x0109, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0xFFFD, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x0121, 0x00F6, 0x00F7, - 0x011D, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x016D, 0x015D, 0x02D9} }, - { "ISO-8859-4", { "latin4", "iso-ir-110", "csISOLatin4", Q_NULLPTR }, 7, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x0104, 0x0138, 0x0156, 0x00A4, 0x0128, 0x013B, 0x00A7, - 0x00A8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00AD, 0x017D, 0x00AF, - 0x00B0, 0x0105, 0x02DB, 0x0157, 0x00B4, 0x0129, 0x013C, 0x02C7, - 0x00B8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014A, 0x017E, 0x014B, - 0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E, - 0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x012A, - 0x0110, 0x0145, 0x014C, 0x0136, 0x00D4, 0x00D5, 0x00D6, 0x00D7, - 0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x0168, 0x016A, 0x00DF, - 0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F, - 0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x012B, - 0x0111, 0x0146, 0x014D, 0x0137, 0x00F4, 0x00F5, 0x00F6, 0x00F7, - 0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x0169, 0x016B, 0x02D9} }, - { "ISO-8859-5", { "cyrillic", "iso-ir-144", "csISOLatinCyrillic", Q_NULLPTR }, 8, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, - 0x0408, 0x0409, 0x040A, 0x040B, 0x040C, 0x00AD, 0x040E, 0x040F, - 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, - 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, - 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, - 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, - 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, - 0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, - 0x0458, 0x0459, 0x045A, 0x045B, 0x045C, 0x00A7, 0x045E, 0x045F} }, - { "ISO-8859-6", { "ISO-8859-6-I", "ECMA-114", "ASMO-708", "arabic", "iso-ir-127", "csISOLatinArabic", Q_NULLPTR }, 82, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A4, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x060C, 0x00AD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0x061B, 0xFFFD, 0xFFFD, 0xFFFD, 0x061F, - 0xFFFD, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, - 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, - 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, - 0x0638, 0x0639, 0x063A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, - 0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F, - 0x0650, 0x0651, 0x0652, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD} }, - { "ISO-8859-7", { "ECMA-118", "greek", "iso-ir-126", "csISOLatinGreek", Q_NULLPTR }, 10, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x2018, 0x2019, 0x00A3, 0xFFFD, 0xFFFD, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0xFFFD, 0x2015, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x0385, 0x0386, 0x00B7, - 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, - 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, - 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, - 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, - 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, - 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, - 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, - 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, - 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD} }, - { "ISO-8859-8", { "ISO 8859-8-I", "iso-ir-138", "hebrew", "csISOLatinHebrew", Q_NULLPTR }, 85, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x203E, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2017, - 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, - 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, - 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, - 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD} }, - { "ISO-8859-9", { "iso-ir-148", "latin5", "csISOLatin5", Q_NULLPTR }, 12, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, - 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, - 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, - 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, 0x015F, 0x00FF} }, - { "ISO-8859-10", { "iso-ir-157", "latin6", "ISO-8859-10:1992", "csISOLatin6", Q_NULLPTR }, 13, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x0104, 0x0112, 0x0122, 0x012A, 0x0128, 0x0136, 0x00A7, - 0x013B, 0x0110, 0x0160, 0x0166, 0x017D, 0x00AD, 0x016A, 0x014A, - 0x00B0, 0x0105, 0x0113, 0x0123, 0x012B, 0x0129, 0x0137, 0x00B7, - 0x013C, 0x0111, 0x0161, 0x0167, 0x017E, 0x2015, 0x016B, 0x014B, - 0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E, - 0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x00CF, - 0x00D0, 0x0145, 0x014C, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x0168, - 0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, - 0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F, - 0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x00EF, - 0x00F0, 0x0146, 0x014D, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x0169, - 0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x0138} }, - { "ISO-8859-13", { Q_NULLPTR }, 109, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x201D, 0x00A2, 0x00A3, 0x00A4, 0x201E, 0x00A6, 0x00A7, - 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x201C, 0x00B5, 0x00B6, 0x00B7, - 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, - 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, - 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, - 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, - 0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, - 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, - 0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C, - 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, - 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019} }, - { "ISO-8859-14", { "iso-ir-199", "latin8", "iso-celtic", Q_NULLPTR }, 110, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x1E02, 0x1E03, 0x00A3, 0x010A, 0x010B, 0x1E0A, 0x00A7, - 0x1E80, 0x00A9, 0x1E82, 0x1E0B, 0x1EF2, 0x00AD, 0x00AE, 0x0178, - 0x1E1E, 0x1E1F, 0x0120, 0x0121, 0x1E40, 0x1E41, 0x00B6, 0x1E56, - 0x1E81, 0x1E57, 0x1E83, 0x1E60, 0x1EF3, 0x1E84, 0x1E85, 0x1E61, - 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0x0174, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x1E6A, - 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x0176, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0x0175, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x1E6B, - 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x0177, 0x00FF} }, - { "ISO-8859-16", { "iso-ir-226", "latin10", Q_NULLPTR }, 112, - { 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - 0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7, - 0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B, - 0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7, - 0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C, - 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A, - 0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B, - 0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF} }, - - // next bits generated again from tables on the Unicode 3.0 CD. - - // $ for a in CP* ; do (awk '/^0x[89ABCDEF]/{ print $1, $2 }' < $a) | sort | sed -e 's/#UNDEF.*$/0xFFFD/' | cut -c6- | paste '-d ' - - - - - - - - | sed -e 's/ /, /g' -e 's/$/,/' -e '$ s/,$/} },/' -e '1 s/^/{ /' > ~/tmp/$a ; done - { "IBM850", { "CP850", "csPC850Multilingual", Q_NULLPTR }, 2009, - { 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, - 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, - 0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, - 0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x0192, - 0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, - 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, - 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1, 0x00C2, 0x00C0, - 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, - 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3, - 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, - 0x00F0, 0x00D0, 0x00CA, 0x00CB, 0x00C8, 0x0131, 0x00CD, 0x00CE, - 0x00CF, 0x2518, 0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, - 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x00FE, - 0x00DE, 0x00DA, 0x00DB, 0x00D9, 0x00FD, 0x00DD, 0x00AF, 0x00B4, - 0x00AD, 0x00B1, 0x2017, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, - 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2, 0x25A0, 0x00A0} }, - { "IBM874", { "CP874", Q_NULLPTR }, -874, //### what is the mib? - { 0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0x00A0, 0x0E01, 0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07, - 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, - 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, 0x0E16, 0x0E17, - 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F, - 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27, - 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F, - 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, - 0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, - 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, - 0x0E48, 0x0E49, 0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, - 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, - 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD} }, - { "IBM866", { "CP866", "csIBM866", Q_NULLPTR }, 2086, - { 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, - 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, - 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, - 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, - 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, - 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, - 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, - 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F, - 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, - 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, - 0x256A, 0x2518, 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, - 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E, - 0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4, 0x25A0, 0x00A0} }, - - { "windows-1250", { "CP1250", Q_NULLPTR }, 2250, - { 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, - 0xFFFD, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0xFFFD, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, - 0x00A0, 0x02C7, 0x02D8, 0x0141, 0x00A4, 0x0104, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, - 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, - 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, - 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, - 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, - 0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, - 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, - 0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F, - 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, - 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9} }, - { "windows-1251", { "CP1251", Q_NULLPTR }, 2251, - { 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, - 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, - 0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, - 0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7, - 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, - 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7, - 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, - 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, - 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, - 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, - 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, - 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F} }, - { "windows-1252", { "CP1252", Q_NULLPTR }, 2252, - { 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, - 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0x017D, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0x017E, 0x0178, - 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, - 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, - 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, - 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF} }, - { "windows-1253", {"CP1253", Q_NULLPTR }, 2253, - { 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, - 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0x00A0, 0x0385, 0x0386, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5, 0x00B6, 0x00B7, - 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, - 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, - 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, - 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, - 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, - 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, - 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, - 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, - 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD} }, - { "windows-1254", { "CP1254", Q_NULLPTR }, 2254, - { 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, - 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, - 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, - 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, - 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, - 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, 0x015F, 0x00FF} }, - { "windows-1255", { "CP1255", Q_NULLPTR }, 2255, - { 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, - 0x02C6, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0x02DC, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x20AA, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, - 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, - 0x05B8, 0x05B9, 0xFFFD, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, - 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, - 0x05F4, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, - 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, - 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, - 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD} }, - { "windows-1256", { "CP1256", Q_NULLPTR }, 2256, - { 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, - 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, - 0x06AF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, - 0x00A0, 0x060C, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, - 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, - 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, - 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, - 0x0637, 0x0638, 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, - 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0649, 0x064A, 0x00EE, 0x00EF, - 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, - 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, 0x200F, 0x06D2} }, - { "windows-1257", { "CP1257", Q_NULLPTR }, 2257, - { 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, - 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0x00A8, 0x02C7, 0x00B8, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0x00AF, 0x02DB, 0xFFFD, - 0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0xFFFD, 0x00A6, 0x00A7, - 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, - 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, - 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, - 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, - 0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, - 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, - 0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C, - 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, - 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x02D9} }, - { "windows-1258", { "CP1258", Q_NULLPTR }, 2258, - { 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, - 0x02C6, 0x2030, 0xFFFD, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0x02DC, 0x2122, 0xFFFD, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, - 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, - 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, - 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, - 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, - 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0301, 0x00ED, 0x00EE, 0x00EF, - 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, - 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, 0x20AB, 0x00FF} }, - - { "Apple Roman", { "macintosh", "MacRoman", Q_NULLPTR }, -168, - { 0x00C4, 0x00C5, 0x00C7, 0x00C9, 0x00D1, 0x00D6, 0x00DC, 0x00E1, - 0x00E0, 0x00E2, 0x00E4, 0x00E3, 0x00E5, 0x00E7, 0x00E9, 0x00E8, - 0x00EA, 0x00EB, 0x00ED, 0x00EC, 0x00EE, 0x00EF, 0x00F1, 0x00F3, - 0x00F2, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x00F9, 0x00FB, 0x00FC, - 0x2020, 0x00B0, 0x00A2, 0x00A3, 0x00A7, 0x2022, 0x00B6, 0x00DF, - 0x00AE, 0x00A9, 0x2122, 0x00B4, 0x00A8, 0x2260, 0x00C6, 0x00D8, - 0x221E, 0x00B1, 0x2264, 0x2265, 0x00A5, 0x00B5, 0x2202, 0x2211, - 0x220F, 0x03C0, 0x222B, 0x00AA, 0x00BA, 0x03A9, 0x00E6, 0x00F8, - 0x00BF, 0x00A1, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB, - 0x00BB, 0x2026, 0x00A0, 0x00C0, 0x00C3, 0x00D5, 0x0152, 0x0153, - 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA, - 0x00FF, 0x0178, 0x2044, 0x20AC, 0x2039, 0x203A, 0xFB01, 0xFB02, - 0x2021, 0x00B7, 0x201A, 0x201E, 0x2030, 0x00C2, 0x00CA, 0x00C1, - 0x00CB, 0x00C8, 0x00CD, 0x00CE, 0x00CF, 0x00CC, 0x00D3, 0x00D4, - 0xF8FF, 0x00D2, 0x00DA, 0x00DB, 0x00D9, 0x0131, 0x02C6, 0x02DC, - 0x00AF, 0x02D8, 0x02D9, 0x02DA, 0x00B8, 0x02DD, 0x02DB, 0x02C7} }, - - // This one is based on the charmap file - // /usr/share/i18n/charmaps/SAMI-WS2.gz, which is manually adapted - // to this format by Boerre Gaup - { "WINSAMI2", { "WS2", Q_NULLPTR }, -165, - { 0x20AC, 0xFFFD, 0x010C, 0x0192, 0x010D, 0x01B7, 0x0292, 0x01EE, - 0x01EF, 0x0110, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0x0111, 0x01E6, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, - 0x00A0, 0x01E7, 0x01E4, 0x00A3, 0x00A4, 0x01E5, 0x00A6, 0x00A7, - 0x00A8, 0x00A9, 0x021E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x021F, - 0x00B0, 0x00B1, 0x01E8, 0x01E9, 0x00B4, 0x00B5, 0x00B6, 0x00B7, - 0x014A, 0x014B, 0x0166, 0x00BB, 0x0167, 0x00BD, 0x017D, 0x017E, - 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, - 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, - 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, - 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, - 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, - 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF} }, - - // this one is generated from the charmap file located in /usr/share/i18n/charmaps - // on most Linux distributions. The thai character set tis620 is byte by byte equivalent - // to iso8859-11, so we name it 8859-11 here, but recognise the name tis620 too. - - // $ for A in 8 9 A B C D E F ; do for B in 0 1 2 3 4 5 6 7 8 9 A B C D E F ; do echo x${A}${B} 0xFFFD ; done ; done > /tmp/digits ; (cut -c25- < TIS-620 ; cat /tmp/digits) | awk '/^x[89ABCDEF]/{ print $1, $2 }' | sed -e 's///' | sort | uniq -w4 | cut -c5- | paste '-d ' - - - - - - - - | sed -e 's/ /, /g' -e 's/$/,/' -e '$ s/,$/} },/' -e '1 s/^/{ /' > ~/tmp/tis-620 - { "TIS-620", { "ISO 8859-11", Q_NULLPTR }, 2259, // Thai character set mib enum taken from tis620 (which is byte by byte equivalent) - { 0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0x0E01, 0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07, - 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, - 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, 0x0E16, 0x0E17, - 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F, - 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27, - 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F, - 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, - 0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, - 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, - 0x0E48, 0x0E49, 0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, - 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, - 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD } }, - - /* - Name: hp-roman8 [HP-PCL5,RFC1345,KXS2] - MIBenum: 2004 - Source: LaserJet IIP Printer User's Manual, - HP part no 33471-90901, Hewlet-Packard, June 1989. - Alias: roman8 - Alias: r8 - Alias: csHPRoman8 - */ - { "roman8", { "hp-roman8", "csHPRoman8", Q_NULLPTR }, 2004, - { 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, - 0x00A0, 0x00C0, 0x00C2, 0x00C8, 0x00CA, 0x00CB, 0x00CE, 0x00CF, - 0x00B4, 0x02CB, 0x02C6, 0x00A8, 0x02DC, 0x00D9, 0x00DB, 0x20A4, - 0x00AF, 0x00DD, 0x00FD, 0x00B0, 0x00C7, 0x00E7, 0x00D1, 0x00F1, - 0x00A1, 0x00BF, 0x00A4, 0x00A3, 0x00A5, 0x00A7, 0x0192, 0x00A2, - 0x00E2, 0x00EA, 0x00F4, 0x00FB, 0x00E1, 0x00E9, 0x00F3, 0x00FA, - 0x00E0, 0x00E8, 0x00F2, 0x00F9, 0x00E4, 0x00EB, 0x00F6, 0x00FC, - 0x00C5, 0x00EE, 0x00D8, 0x00C6, 0x00E5, 0x00ED, 0x00F8, 0x00E6, - 0x00C4, 0x00EC, 0x00D6, 0x00DC, 0x00C9, 0x00EF, 0x00DF, 0x00D4, - 0x00C1, 0x00C3, 0x00E3, 0x00D0, 0x00F0, 0x00CD, 0x00CC, 0x00D3, - 0x00D2, 0x00D5, 0x00F5, 0x0160, 0x0161, 0x00DA, 0x0178, 0x00FF, - 0x00DE, 0x00FE, 0x00B7, 0x00B5, 0x00B6, 0x00BE, 0x2014, 0x00BC, - 0x00BD, 0x00AA, 0x00BA, 0x00AB, 0x25A0, 0x00BB, 0x00B1, 0xFFFD } } - - // if you add more chacater sets at the end, change LAST_MIB above -}; - -QSimpleTextCodec::QSimpleTextCodec(int i) : forwardIndex(i), reverseMap(Q_NULLPTR) -{ -} - - -QSimpleTextCodec::~QSimpleTextCodec() -{ - delete reverseMap; -} - -static QByteArray *buildReverseMap(int forwardIndex) -{ - QByteArray *map = new QByteArray(); - int m = 0; - int i = 0; - while(i < 128) { - if (unicodevalues[forwardIndex].values[i] > m && - unicodevalues[forwardIndex].values[i] < 0xfffd) - m = unicodevalues[forwardIndex].values[i]; - i++; - } - m++; - map->resize(m); - for(i = 0; i < 128 && i < m; i++) - (*map)[i] = (char)i; - for(;i < m; i++) - (*map)[i] = 0; - for(i=128; i<256; i++) { - int u = unicodevalues[forwardIndex].values[i-128]; - if (u < m) - (*map)[u] = (char)(unsigned char)(i); - } - return map; -} - -QString QSimpleTextCodec::convertToUnicode(const char* chars, int len, ConverterState *) const -{ - if (len <= 0 || !chars) - return QString(); - - const unsigned char * c = (const unsigned char *)chars; - - QString r(len, Qt::Uninitialized); - QChar* uc = r.data(); - - for (int i = 0; i < len; i++) { - if (c[i] > 127) - uc[i] = unicodevalues[forwardIndex].values[c[i]-128]; - else - uc[i] = QLatin1Char(c[i]); - } - return r; -} - -QByteArray QSimpleTextCodec::convertFromUnicode(const QChar *in, int length, ConverterState *state) const -{ - const char replacement = (state && state->flags & ConvertInvalidToNull) ? 0 : '?'; - int invalid = 0; - - if (!reverseMap){ - QByteArray *tmp = buildReverseMap(this->forwardIndex); - if (!reverseMap.testAndSetOrdered(0, tmp)) - delete tmp; - } - - QByteArray r(length, Qt::Uninitialized); - int i = length; - int u; - const QChar* ucp = in; - unsigned char* rp = (unsigned char *)r.data(); - const unsigned char* rmp = (const unsigned char *)reverseMap->constData(); - int rmsize = (int) reverseMap->size(); - while(i--) - { - u = ucp->unicode(); - if (u < 128) { - *rp = (char)u; - } else { - *rp = ((u < rmsize) ? (*(rmp+u)) : 0); - if (*rp == 0) { - *rp = replacement; - ++invalid; - } - } - rp++; - ucp++; - } - - if (state) { - state->invalidChars += invalid; - } - return r; -} - -QByteArray QSimpleTextCodec::name() const -{ - return unicodevalues[forwardIndex].mime; -} - -QList QSimpleTextCodec::aliases() const -{ - QList list; - const char * const*a = unicodevalues[forwardIndex].aliases; - while (*a) { - list << *a; - ++a; - } - return list; -} - -int QSimpleTextCodec::mibEnum() const -{ - return unicodevalues[forwardIndex].mib; -} - -QT_END_NAMESPACE - -#endif // QT_NO_TEXTCODEC diff --git a/src/core/codecs/qsimplecodec_p.h b/src/core/codecs/qsimplecodec_p.h deleted file mode 100644 index ffaa5dc6b..000000000 --- a/src/core/codecs/qsimplecodec_p.h +++ /dev/null @@ -1,79 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QSIMPLECODEC_P_H -#define QSIMPLECODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "QtCore/qtextcodec.h" - -QT_BEGIN_NAMESPACE - -#ifndef QT_NO_TEXTCODEC - -template class QAtomicPointer; - -class QSimpleTextCodec: public QTextCodec -{ -public: - enum { numSimpleCodecs = 30 }; - explicit QSimpleTextCodec(int); - ~QSimpleTextCodec(); - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - - QByteArray name() const; - QList aliases() const; - int mibEnum() const; - -private: - int forwardIndex; - mutable QAtomicPointer reverseMap; -}; - -#endif // QT_NO_TEXTCODEC - -QT_END_NAMESPACE - -#endif // QSIMPLECODEC_P_H diff --git a/src/core/codecs/qtextcodec.cpp b/src/core/codecs/qtextcodec.cpp index 148806e8e..5e2bd28a1 100644 --- a/src/core/codecs/qtextcodec.cpp +++ b/src/core/codecs/qtextcodec.cpp @@ -45,10 +45,7 @@ #include "qlocale_tools_p.h" #include "qmutex.h" #include "qhash.h" -#include "qiconvcodec_p.h" -#include "qutfcodec_p.h" -#include "qsimplecodec_p.h" -#include "qlatincodec_p.h" +#include "qicucodec_p.h" #ifndef QT_NO_LIBRARY # include "qcoreapplication.h" @@ -56,14 +53,6 @@ # include "qfactoryloader_p.h" #endif -#ifndef QT_NO_CODECS -# include "qtsciicodec_p.h" -# include "qisciicodec_p.h" -# if defined(Q_WS_X11) && !defined(QT_BOOTSTRAPPED) -# include "qfontlaocodec_p.h" -# endif -#endif // QT_NO_CODECS - #include #include #include @@ -133,7 +122,8 @@ static QTextCodec *createForName(const QByteArray &name) #else Q_UNUSED(name); #endif - return 0; + + return new QIcuCodec(name.constData()); } static QTextCodec *createForMib(int mib) @@ -146,16 +136,16 @@ static QTextCodec *createForMib(int mib) #else Q_UNUSED(mib); #endif - return 0; + return Q_NULLPTR; } -static QList *all = 0; +static QList *all = Q_NULLPTR; #ifdef Q_DEBUG_TEXTCODEC static bool destroying_is_ok = false; #endif -static QTextCodec *localeMapper = 0; -QTextCodec *QTextCodec::cftr = 0; +static QTextCodec *localeMapper = Q_NULLPTR; +QTextCodec *QTextCodec::cftr = Q_NULLPTR; class QTextCodecCleanup @@ -179,13 +169,13 @@ QTextCodecCleanup::~QTextCodecCleanup() #endif QList *myAll = all; - all = 0; // Otherwise the d'tor destroys the iterator + all = Q_NULLPTR; // Otherwise the d'tor destroys the iterator for (QList::const_iterator it = myAll->constBegin() ; it != myAll->constEnd(); ++it) { delete *it; } delete myAll; - localeMapper = 0; + localeMapper = Q_NULLPTR; #ifdef Q_DEBUG_TEXTCODEC destroying_is_ok = false; @@ -194,107 +184,6 @@ QTextCodecCleanup::~QTextCodecCleanup() Q_GLOBAL_STATIC(QTextCodecCleanup, createQTextCodecCleanup) -/* locale names mostly copied from XFree86 */ -static const char * const iso8859_2locales[] = { - "croatian", "cs", "cs_CS", "cs_CZ","cz", "cz_CZ", "czech", "hr", - "hr_HR", "hu", "hu_HU", "hungarian", "pl", "pl_PL", "polish", "ro", - "ro_RO", "rumanian", "serbocroatian", "sh", "sh_SP", "sh_YU", "sk", - "sk_SK", "sl", "sl_CS", "sl_SI", "slovak", "slovene", "sr_SP", 0 }; - -static const char * const iso8859_3locales[] = { - "eo", 0 }; - -static const char * const iso8859_4locales[] = { - "ee", "ee_EE", 0 }; - -static const char * const iso8859_5locales[] = { - "mk", "mk_MK", "sp", "sp_YU", 0 }; - -static const char * const cp_1251locales[] = { - "be", "be_BY", "bg", "bg_BG", "bulgarian", 0 }; - -static const char * const pt_154locales[] = { - "ba_RU", "ky", "ky_KG", "kk", "kk_KZ", 0 }; - -static const char * const iso8859_6locales[] = { - "ar_AA", "ar_SA", "arabic", 0 }; - -static const char * const iso8859_7locales[] = { - "el", "el_GR", "greek", 0 }; - -static const char * const iso8859_8locales[] = { - "hebrew", "he", "he_IL", "iw", "iw_IL", 0 }; - -static const char * const iso8859_9locales[] = { - "tr", "tr_TR", "turkish", 0 }; - -static const char * const iso8859_13locales[] = { - "lt", "lt_LT", "lv", "lv_LV", 0 }; - -static const char * const iso8859_15locales[] = { - "et", "et_EE", - // Euro countries - "br_FR", "ca_ES", "de", "de_AT", "de_BE", "de_DE", "de_LU", "en_IE", - "es", "es_ES", "eu_ES", "fi", "fi_FI", "finnish", "fr", "fr_FR", - "fr_BE", "fr_LU", "french", "ga_IE", "gl_ES", "it", "it_IT", "oc_FR", - "nl", "nl_BE", "nl_NL", "pt", "pt_PT", "sv_FI", "wa_BE", - 0 }; - -static const char * const koi8_ulocales[] = { - "uk", "uk_UA", "ru_UA", "ukrainian", 0 }; - -static const char * const tis_620locales[] = { - "th", "th_TH", "thai", 0 }; - -// static const char * const tcvnlocales[] = { -// "vi", "vi_VN", 0 }; - -static bool try_locale_list(const char * const locale[], const QByteArray &lang) -{ - int i; - for(i=0; locale[i] && lang != locale[i]; i++) - ; - return locale[i] != 0; -} - -// For the probably_koi8_locales we have to look. the standard says -// these are 8859-5, but almost all Russian users use KOI8-R and -// incorrectly set $LANG to ru_RU. We'll check tolower() to see what -// it thinks ru_RU means. - -// If you read the history, it seems that many Russians blame ISO and -// Perestroika for the confusion. -// -// The real bug is that some programs break if the user specifies -// ru_RU.KOI8-R. - -static const char * const probably_koi8_rlocales[] = { - "ru", "ru_SU", "ru_RU", "russian", 0 }; - -static QTextCodec * ru_RU_hack(const char * i) { - QTextCodec * ru_RU_codec = 0; - - const QByteArray origlocale(setlocale(LC_CTYPE, i)); - // unicode koi8r latin5 name - // 0x044E 0xC0 0xEE CYRILLIC SMALL LETTER YU - // 0x042E 0xE0 0xCE CYRILLIC CAPITAL LETTER YU - int latin5 = tolower(0xCE); - int koi8r = tolower(0xE0); - if (koi8r == 0xC0 && latin5 != 0xEE) { - ru_RU_codec = QTextCodec::codecForName("KOI8-R"); - } else if (koi8r != 0xC0 && latin5 == 0xEE) { - ru_RU_codec = QTextCodec::codecForName("ISO 8859-5"); - } else { - // something else again... let's assume... *throws dice* - ru_RU_codec = QTextCodec::codecForName("KOI8-R"); - qWarning("QTextCodec: Using KOI8-R, probe failed (%02x %02x %s)", - koi8r, latin5, i); - } - setlocale(LC_CTYPE, origlocale.constData()); - - return ru_RU_codec; -} - static QTextCodec *checkForCodec(const QByteArray &name) { QTextCodec *c = QTextCodec::codecForName(name); if (!c) { @@ -311,24 +200,9 @@ static QTextCodec *checkForCodec(const QByteArray &name) { */ static void setupLocaleMapper() { - -#ifndef QT_NO_ICONV - localeMapper = QTextCodec::codecForName("System"); -#endif - -#if defined (_XOPEN_UNIX) && !defined(Q_OS_OSF) - if (!localeMapper) { - char *charset = nl_langinfo (CODESET); - if (charset) - localeMapper = QTextCodec::codecForName(charset); - } -#endif - if (!localeMapper) { // Very poorly defined and followed standards causes lots of - // code to try to get all the cases... This logic is - // duplicated in QIconvCodec, so if you change it here, change - // it there too. + // code to try to get all the cases... // Try to determine locale codeset from locale name assigned to // LC_CTYPE category. @@ -354,8 +228,6 @@ static void setupLocaleMapper() // 3. ctype (maybe the locale is named "ISO-8859-1" or something) // 4. locale (ditto) // 5. check for "@euro" - // 6. guess locale from ctype unless ctype is "C" - // 7. guess locale from lang // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15) int indexOfDot = ctype.indexOf('.'); @@ -379,52 +251,22 @@ static void setupLocaleMapper() // 5. "@euro" if ((!localeMapper && ctype.contains("@euro")) || lang.contains("@euro")) - localeMapper = checkForCodec("ISO 8859-15"); - - // 6. guess locale from ctype unless ctype is "C" - // 7. guess locale from lang - const QByteArray &try_by_name = (!ctype.isEmpty() && ctype != "C") ? lang : ctype; - - // Now do the guessing. - if (!lang.isEmpty() && !localeMapper && !try_by_name.isEmpty()) { - if (try_locale_list(iso8859_15locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-15"); - else if (try_locale_list(iso8859_2locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-2"); - else if (try_locale_list(iso8859_3locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-3"); - else if (try_locale_list(iso8859_4locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-4"); - else if (try_locale_list(iso8859_5locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-5"); - else if (try_locale_list(iso8859_6locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-6"); - else if (try_locale_list(iso8859_7locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-7"); - else if (try_locale_list(iso8859_8locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-8-I"); - else if (try_locale_list(iso8859_9locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-9"); - else if (try_locale_list(iso8859_13locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-13"); - else if (try_locale_list(tis_620locales, lang)) - localeMapper = QTextCodec::codecForName("ISO 8859-11"); - else if (try_locale_list(koi8_ulocales, lang)) - localeMapper = QTextCodec::codecForName("KOI8-U"); - else if (try_locale_list(cp_1251locales, lang)) - localeMapper = QTextCodec::codecForName("CP 1251"); - else if (try_locale_list(pt_154locales, lang)) - localeMapper = QTextCodec::codecForName("PT 154"); - else if (try_locale_list(probably_koi8_rlocales, lang)) - localeMapper = ru_RU_hack(lang.constData()); - } + localeMapper = checkForCodec("ISO-8859-15"); } +#if defined (_XOPEN_UNIX) && !defined(Q_OS_OSF) + if (!localeMapper) { + char *charset = nl_langinfo (CODESET); + if (charset) + localeMapper = QTextCodec::codecForName(charset); + } +#endif + // If everything failed, we default to 8859-1 // We could perhaps default to 8859-15. if (!localeMapper) - localeMapper = QTextCodec::codecForName("ISO 8859-1"); + localeMapper = QTextCodec::codecForName("ISO-8859-1"); } #ifndef QT_NO_THREAD @@ -446,34 +288,12 @@ static void setup() // create the cleanup object to cleanup all codecs on exit (void) createQTextCodecCleanup(); -#ifndef QT_NO_CODECS - (void)new QTsciiCodec; - for (int i = 0; i < 9; ++i) - (void)new QIsciiCodec(i); - - for (int i = 0; i < QSimpleTextCodec::numSimpleCodecs; ++i) - (void)new QSimpleTextCodec(i); - - -# if defined(Q_WS_X11) && !defined(QT_BOOTSTRAPPED) - // no font codecs when bootstrapping - (void)new QFontLaoCodec; -# endif // Q_WS_X11 -#endif // QT_NO_CODECS - - (void)new QUtf16Codec; - (void)new QUtf16BECodec; - (void)new QUtf16LECodec; - (void)new QUtf32Codec; - (void)new QUtf32BECodec; - (void)new QUtf32LECodec; - (void)new QLatin15Codec; - (void)new QLatin1Codec; - (void)new QUtf8Codec; - -#if !defined(QT_NO_ICONV) && !defined(QT_BOOTSTRAPPED) - // QIconvCodec depends on the UTF-16 codec, so it needs to be created last - (void) new QIconvCodec(); +#if !defined(QT_NO_TEXTCODEC) + // codecs have to be explicitly created for use in QString + (void) new QIcuCodec("US-ASCII"); + (void) new QIcuCodec("UTF-8"); + (void) new QIcuCodec("UTF-16"); + (void) new QIcuCodec("UTF-32"); #endif if (!localeMapper) @@ -487,8 +307,6 @@ static void setup() \value ConvertInvalidToNull If this flag is set, each invalid input character is output as a null character. \value IgnoreHeader Ignore any Unicode byte-order mark and don't generate any. - - \omitvalue FreeFunction */ /*! @@ -502,8 +320,8 @@ static void setup() */ QTextCodec::ConverterState::~ConverterState() { - if (flags & FreeFunction) - (QTextCodecUnalignedPointer::decode(state_data))(this); + if (d) + ucnv_close(static_cast(d)); } /*! @@ -524,31 +342,108 @@ QTextCodec::ConverterState::~ConverterState() The supported encodings are: - - \list - \o Apple Roman - \o IBM 850 - \o IBM 866 - \o IBM 874 - \o ISO 8859-1 to 10 - \o ISO 8859-13 to 16 - \o Iscii-Bng, Dev, Gjr, Knd, Mlm, Ori, Pnj, Tlg, and Tml + \o Adobe-Standard-Encoding + \o Big5 + \o Big5-HKSCS + \o BOCU-1 + \o CESU-8 + \o cp851 + \o EUC-JP + \o EUC-KR + \o GB18030 + \o GB2312 + \o GB_2312-80 + \o GBK + \o hp-roman8 + \o HZ-GB-2312 + \o IBM00858 + \o IBM01140 + \o IBM01141 + \o IBM01142 + \o IBM01143 + \o IBM01144 + \o IBM01145 + \o IBM01146 + \o IBM01147 + \o IBM01148 + \o IBM01149 + \o IBM037 + \o IBM1026 + \o IBM1047 + \o IBM273 + \o IBM277 + \o IBM278 + \o IBM280 + \o IBM284 + \o IBM285 + \o IBM290 + \o IBM297 + \o IBM420 + \o IBM424 + \o IBM437 + \o IBM500 + \o IBM775 + \o IBM850 + \o IBM852 + \o IBM855 + \o IBM857 + \o IBM860 + \o IBM861 + \o IBM862 + \o IBM863 + \o IBM864 + \o IBM865 + \o IBM866 + \o IBM868 + \o IBM869 + \o IBM870 + \o IBM871 + \o IBM918 + \o IBM-Thai + \o ISO-2022-CN + \o ISO-2022-CN-EXT + \o ISO-2022-JP + \o ISO-2022-JP-1 + \o ISO-2022-JP-2 + \o ISO-2022-KR + \o ISO-8859-1 + \o ISO-8859-10 + \o ISO-8859-13 + \o ISO-8859-14 + \o ISO-8859-15 + \o ISO-8859-2 + \o ISO-8859-3 + \o ISO-8859-4 + \o ISO-8859-5 + \o ISO-8859-6 + \o ISO-8859-7 + \o ISO-8859-8 + \o ISO-8859-9 \o KOI8-R \o KOI8-U - \o MuleLao-1 - \o ROMAN8 + \o macintosh + \o SCSU + \o Shift_JIS \o TIS-620 - \o \l{TSCII Text Codec}{TSCII} - \o UTF-8 + \o US-ASCII \o UTF-16 \o UTF-16BE \o UTF-16LE \o UTF-32 \o UTF-32BE \o UTF-32LE - \o Windows-1250 to 1258 - \o WINSAMI2 + \o UTF-7 + \o UTF-8 + \o windows-1250 + \o windows-1251 + \o windows-1252 + \o windows-1253 + \o windows-1254 + \o windows-1255 + \o windows-1256 + \o windows-1257 + \o windows-1258 \endlist QTextCodecs can be used as follows to convert some locally encoded @@ -686,7 +581,12 @@ QTextCodec::~QTextCodec() QTextCodec *QTextCodec::codecForName(const QByteArray &name) { if (name.isEmpty()) - return 0; + return Q_NULLPTR; + + if (name == "System") + return QTextCodec::codecForLocale(); + else if (name == "Latin1") + return QTextCodec::codecForName("US-ASCII"); #ifndef QT_NO_THREAD QMutexLocker locker(textCodecsMutex()); @@ -778,6 +678,7 @@ QList QTextCodec::availableCodecs() setup(); QList codecs; + codecs << "System" << "Latin1"; for (int i = 0; i < all->size(); ++i) { codecs += all->at(i)->name(); codecs += all->at(i)->aliases(); @@ -860,11 +761,6 @@ void QTextCodec::setCodecForLocale(QTextCodec *c) /*! Returns a pointer to the codec most suitable for this locale. - - On Windows, the codec will be based on a system locale. On Unix - systems, starting with Qt 4.2, the codec will be using the \e - iconv library. Note that in both cases the codec's name will be - "System". */ QTextCodec* QTextCodec::codecForLocale() @@ -875,7 +771,7 @@ QTextCodec* QTextCodec::codecForLocale() #ifndef QT_NO_THREAD QMutexLocker locker(textCodecsMutex()); #endif - setup(); + setupLocaleMapper(); return localeMapper; } @@ -1181,19 +1077,7 @@ QString QTextDecoder::toUnicode(const char *chars, int len) void QTextDecoder::toUnicode(QString *target, const char *chars, int len) { Q_ASSERT(target); - switch (c->mibEnum()) { - case 106: // utf8 - static_cast(c)->convertToUnicode(target, chars, len, &state); - break; - case 4: { // latin1 - target->resize(len); - ushort *data = (ushort*)target->data(); - for (int i = len; i >=0; --i) - data[i] = (uchar) chars[i]; - } break; - default: - *target = c->toUnicode(chars, len, &state); - } + *target = c->toUnicode(chars, len, &state); } diff --git a/src/core/codecs/qtextcodec.h b/src/core/codecs/qtextcodec.h index dd758e033..aef203658 100644 --- a/src/core/codecs/qtextcodec.h +++ b/src/core/codecs/qtextcodec.h @@ -86,19 +86,20 @@ public: enum ConversionFlag { DefaultConversion, ConvertInvalidToNull = 0x80000000, - IgnoreHeader = 0x1, - FreeFunction = 0x2 + IgnoreHeader = 0x1 }; Q_DECLARE_FLAGS(ConversionFlags, ConversionFlag) struct Q_CORE_EXPORT ConverterState { ConverterState(ConversionFlags f = DefaultConversion) - : flags(f), remainingChars(0), invalidChars(0) { state_data[0] = state_data[1] = state_data[2] = 0; } + : flags(f), remainingChars(0), invalidChars(0), d(Q_NULLPTR) + { + } ~ConverterState(); ConversionFlags flags; int remainingChars; int invalidChars; - uint state_data[3]; + void *d; private: Q_DISABLE_COPY(ConverterState) }; @@ -122,8 +123,6 @@ protected: QTextCodec(); virtual ~QTextCodec(); -public: - private: friend class QTextCodecCleanup; static QTextCodec *cftr; diff --git a/src/core/codecs/qtextcodec_p.h b/src/core/codecs/qtextcodec_p.h index 87ca58348..1c5180b2e 100644 --- a/src/core/codecs/qtextcodec_p.h +++ b/src/core/codecs/qtextcodec_p.h @@ -50,26 +50,7 @@ QT_BEGIN_NAMESPACE -#ifndef QT_NO_TEXTCODEC - -typedef void (*QTextCodecStateFreeFunction)(QTextCodec::ConverterState*); - -struct QTextCodecUnalignedPointer -{ - static inline QTextCodecStateFreeFunction decode(const uint *src) - { - quintptr data; - memcpy(&data, src, sizeof(data)); - return reinterpret_cast(data); - } - static inline void encode(uint *dst, QTextCodecStateFreeFunction fn) - { - quintptr data = reinterpret_cast(fn); - memcpy(dst, &data, sizeof(data)); - } -}; - -#else +#ifdef QT_NO_TEXTCODEC class QTextCodec { @@ -77,19 +58,18 @@ public: enum ConversionFlag { DefaultConversion, ConvertInvalidToNull = 0x80000000, - IgnoreHeader = 0x1, - FreeFunction = 0x2 + IgnoreHeader = 0x1 }; Q_DECLARE_FLAGS(ConversionFlags, ConversionFlag) struct ConverterState { ConverterState(ConversionFlags f = DefaultConversion) - : flags(f), remainingChars(0), invalidChars(0), { state_data[0] = state_data[1] = state_data[2] = 0; } + : flags(f), remainingChars(0), invalidChars(0), d(Q_NULLPTR) { } ~ConverterState() { } ConversionFlags flags; int remainingChars; int invalidChars; - uint state_data[3]; + void *d; private: Q_DISABLE_COPY(ConverterState) }; diff --git a/src/core/codecs/qtsciicodec.cpp b/src/core/codecs/qtsciicodec.cpp deleted file mode 100644 index 40fec7aaa..000000000 --- a/src/core/codecs/qtsciicodec.cpp +++ /dev/null @@ -1,490 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -// Most of the code here was originally written by Hans Petter Bieker, -// and is included in Qt with the author's permission, and the grateful -// thanks of the Qt team. - -#include "qtsciicodec_p.h" -#include "qlist.h" - -#ifndef QT_NO_CODECS - -QT_BEGIN_NAMESPACE - -static unsigned char qt_UnicodeToTSCII(ushort u1, ushort u2, ushort u3); -static unsigned int qt_TSCIIToUnicode(unsigned int code, uint *s); - -#define IsTSCIIChar(c) (((c) >= 0x80) && ((c) <= 0xfd)) - -/*! \class QTsciiCodec - \reentrant - \internal -*/ - -/*! - Destroys the text codec object. -*/ -QTsciiCodec::~QTsciiCodec() -{ -} - -/*! - Converts the first \a len characters in \a uc from Unicode to this - encoding, and returns the result in a byte array. The \a state contains - some conversion flags, and is used by the codec to maintain state - information. -*/ -QByteArray QTsciiCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const -{ - char replacement = '?'; - if (state) { - if (state->flags & ConvertInvalidToNull) - replacement = 0; - } - int invalid = 0; - - QByteArray rstr(len, Qt::Uninitialized); - uchar* cursor = (uchar*)rstr.data(); - for (int i = 0; i < len; i++) { - QChar ch = uc[i]; - uchar j; - if (ch.row() == 0x00 && ch.cell() < 0x80) { - // ASCII - j = ch.cell(); - } else if ((j = qt_UnicodeToTSCII(uc[i].unicode(), - uc[i + 1].unicode(), - uc[i + 2].unicode()))) { - // We have to check the combined chars first! - i += 2; - } else if ((j = qt_UnicodeToTSCII(uc[i].unicode(), - uc[i + 1].unicode(), 0))) { - i++; - } else if ((j = qt_UnicodeToTSCII(uc[i].unicode(), 0, 0))) { - } else { - // Error - j = replacement; - ++invalid; - } - *cursor++ = j; - } - rstr.resize(cursor - (const uchar*)rstr.constData()); - - if (state) { - state->invalidChars += invalid; - } - return rstr; -} - -/*! - Converts the first \a len characters in \a chars from this encoding - to Unicode, and returns the result in a QString. The \a state contains - some conversion flags, and is used by the codec to maintain state - information. -*/ -QString QTsciiCodec::convertToUnicode(const char* chars, int len, ConverterState *state) const -{ - QChar replacement = QChar::ReplacementCharacter; - if (state) { - if (state->flags & ConvertInvalidToNull) - replacement = QChar::Null; - } - int invalid = 0; - - QString result; - for (int i = 0; i < len; i++) { - uchar ch = chars[i]; - if (ch < 0x80) { - // ASCII - result += QLatin1Char(ch); - } else if (IsTSCIIChar(ch)) { - // TSCII - uint s[3]; - uint u = qt_TSCIIToUnicode(ch, s); - uint *p = s; - while (u--) { - uint c = *p++; - if (c) - result += QChar(c); - else { - result += replacement; - ++invalid; - } - } - } else { - // Invalid - result += replacement; - ++invalid; - } - } - - if (state) { - state->invalidChars += invalid; - } - return result; -} - -/*! - Returns the official name for the encoding that is handled by the codec. - - \sa QTextCodec::name() -*/ -QByteArray QTsciiCodec::name() const -{ - return "TSCII"; -} - -/*! - Returns the MIB enum for the encoding. - - \sa QTextCodec::mibEnum() -*/ -int QTsciiCodec::mibEnum() const -{ - return 2107; -} - -static const int UnToTsLast = 124; // 125 items -- so the last will be 124 -static const ushort UnToTs [][4] = { - // *Sorted* list of TSCII maping for unicode chars - //FIRST SECOND THIRD TSCII - {0x00A0, 0x0000, 0x0000, 0xA0}, - {0x00A9, 0x0000, 0x0000, 0xA9}, - {0x0B83, 0x0000, 0x0000, 0xB7}, - {0x0B85, 0x0000, 0x0000, 0xAB}, - {0x0B86, 0x0000, 0x0000, 0xAC}, - {0x0B87, 0x0000, 0x0000, 0xAD}, - {0x0B88, 0x0000, 0x0000, 0xAE}, - {0x0B89, 0x0000, 0x0000, 0xAF}, - {0x0B8A, 0x0000, 0x0000, 0xB0}, - {0x0B8E, 0x0000, 0x0000, 0xB1}, - {0x0B8F, 0x0000, 0x0000, 0xB2}, - {0x0B90, 0x0000, 0x0000, 0xB3}, - {0x0B92, 0x0000, 0x0000, 0xB4}, - {0x0B93, 0x0000, 0x0000, 0xB5}, - {0x0B94, 0x0000, 0x0000, 0xB6}, - {0x0B95, 0x0000, 0x0000, 0xB8}, - {0x0B95, 0x0B82, 0x0000, 0xEC}, - {0x0B95, 0x0BC1, 0x0000, 0xCC}, - {0x0B95, 0x0BC2, 0x0000, 0xDC}, - {0x0B99, 0x0000, 0x0000, 0xB9}, - {0x0B99, 0x0B82, 0x0000, 0xED}, - {0x0B99, 0x0BC1, 0x0000, 0x99}, - {0x0B99, 0x0BC2, 0x0000, 0x9B}, - {0x0B9A, 0x0000, 0x0000, 0xBA}, - {0x0B9A, 0x0B82, 0x0000, 0xEE}, - {0x0B9A, 0x0BC1, 0x0000, 0xCD}, - {0x0B9A, 0x0BC2, 0x0000, 0xDD}, - {0x0B9C, 0x0000, 0x0000, 0x83}, - {0x0B9C, 0x0B82, 0x0000, 0x88}, - {0x0B9E, 0x0000, 0x0000, 0xBB}, - {0x0B9E, 0x0B82, 0x0000, 0xEF}, - {0x0B9E, 0x0BC1, 0x0000, 0x9A}, - {0x0B9E, 0x0BC2, 0x0000, 0x9C}, - {0x0B9F, 0x0000, 0x0000, 0xBC}, - {0x0B9F, 0x0B82, 0x0000, 0xF0}, - {0x0B9F, 0x0BBF, 0x0000, 0xCA}, - {0x0B9F, 0x0BC0, 0x0000, 0xCB}, - {0x0B9F, 0x0BC1, 0x0000, 0xCE}, - {0x0B9F, 0x0BC2, 0x0000, 0xDE}, - {0x0BA1, 0x0B82, 0x0000, 0xF2}, - {0x0BA3, 0x0000, 0x0000, 0xBD}, - {0x0BA3, 0x0B82, 0x0000, 0xF1}, - {0x0BA3, 0x0BC1, 0x0000, 0xCF}, - {0x0BA3, 0x0BC2, 0x0000, 0xDF}, - {0x0BA4, 0x0000, 0x0000, 0xBE}, - {0x0BA4, 0x0BC1, 0x0000, 0xD0}, - {0x0BA4, 0x0BC2, 0x0000, 0xE0}, - {0x0BA8, 0x0000, 0x0000, 0xBF}, - {0x0BA8, 0x0B82, 0x0000, 0xF3}, - {0x0BA8, 0x0BC1, 0x0000, 0xD1}, - {0x0BA8, 0x0BC2, 0x0000, 0xE1}, - {0x0BA9, 0x0000, 0x0000, 0xC9}, - {0x0BA9, 0x0B82, 0x0000, 0xFD}, - {0x0BA9, 0x0BC1, 0x0000, 0xDB}, - {0x0BA9, 0x0BC2, 0x0000, 0xEB}, - {0x0BAA, 0x0000, 0x0000, 0xC0}, - {0x0BAA, 0x0B82, 0x0000, 0xF4}, - {0x0BAA, 0x0BC1, 0x0000, 0xD2}, - {0x0BAA, 0x0BC2, 0x0000, 0xE2}, - {0x0BAE, 0x0000, 0x0000, 0xC1}, - {0x0BAE, 0x0B82, 0x0000, 0xF5}, - {0x0BAE, 0x0BC1, 0x0000, 0xD3}, - {0x0BAE, 0x0BC2, 0x0000, 0xE3}, - {0x0BAF, 0x0000, 0x0000, 0xC2}, - {0x0BAF, 0x0B82, 0x0000, 0xF6}, - {0x0BAF, 0x0BC1, 0x0000, 0xD4}, - {0x0BAF, 0x0BC2, 0x0000, 0xE4}, - {0x0BB0, 0x0000, 0x0000, 0xC3}, - {0x0BB0, 0x0B82, 0x0000, 0xF7}, - {0x0BB0, 0x0BC1, 0x0000, 0xD5}, - {0x0BB0, 0x0BC2, 0x0000, 0xE5}, - {0x0BB1, 0x0000, 0x0000, 0xC8}, - {0x0BB1, 0x0B82, 0x0000, 0xFC}, - {0x0BB1, 0x0BC1, 0x0000, 0xDA}, - {0x0BB1, 0x0BC2, 0x0000, 0xEA}, - {0x0BB2, 0x0000, 0x0000, 0xC4}, - {0x0BB2, 0x0B82, 0x0000, 0xF8}, - {0x0BB2, 0x0BC1, 0x0000, 0xD6}, - {0x0BB2, 0x0BC2, 0x0000, 0xE6}, - {0x0BB3, 0x0000, 0x0000, 0xC7}, - {0x0BB3, 0x0B82, 0x0000, 0xFB}, - {0x0BB3, 0x0BC1, 0x0000, 0xD9}, - {0x0BB3, 0x0BC2, 0x0000, 0xE9}, - {0x0BB4, 0x0000, 0x0000, 0xC6}, - {0x0BB4, 0x0B82, 0x0000, 0xFA}, - {0x0BB4, 0x0BC1, 0x0000, 0xD8}, - {0x0BB4, 0x0BC2, 0x0000, 0xE8}, - {0x0BB5, 0x0000, 0x0000, 0xC5}, - {0x0BB5, 0x0B82, 0x0000, 0xF9}, - {0x0BB5, 0x0BC1, 0x0000, 0xD7}, - {0x0BB5, 0x0BC2, 0x0000, 0xE7}, - {0x0BB7, 0x0000, 0x0000, 0x84}, - {0x0BB7, 0x0B82, 0x0000, 0x89}, - {0x0BB8, 0x0000, 0x0000, 0x85}, - {0x0BB8, 0x0B82, 0x0000, 0x8A}, - {0x0BB9, 0x0000, 0x0000, 0x86}, - {0x0BB9, 0x0B82, 0x0000, 0x8B}, - {0x0BBE, 0x0000, 0x0000, 0xA1}, - {0x0BBF, 0x0000, 0x0000, 0xA2}, - {0x0BC0, 0x0000, 0x0000, 0xA3}, - {0x0BC1, 0x0000, 0x0000, 0xA4}, - {0x0BC2, 0x0000, 0x0000, 0xA5}, - {0x0BC6, 0x0000, 0x0000, 0xA6}, - {0x0BC7, 0x0000, 0x0000, 0xA7}, - {0x0BC8, 0x0000, 0x0000, 0xA8}, - {0x0BCC, 0x0000, 0x0000, 0xAA}, - {0x0BE6, 0x0000, 0x0000, 0x80}, - {0x0BE7, 0x0000, 0x0000, 0x81}, - {0x0BE7, 0x0BB7, 0x0000, 0x87}, - {0x0BE7, 0x0BB7, 0x0B82, 0x8C}, - {0x0BE8, 0x0000, 0x0000, 0x8D}, - {0x0BE9, 0x0000, 0x0000, 0x8E}, - {0x0BEA, 0x0000, 0x0000, 0x8F}, - {0x0BEB, 0x0000, 0x0000, 0x90}, - {0x0BEC, 0x0000, 0x0000, 0x95}, - {0x0BED, 0x0000, 0x0000, 0x96}, - {0x0BEE, 0x0000, 0x0000, 0x97}, - {0x0BEF, 0x0000, 0x0000, 0x98}, - {0x0BF0, 0x0000, 0x0000, 0x9D}, - {0x0BF1, 0x0000, 0x0000, 0x9E}, - {0x0BF2, 0x0000, 0x0000, 0x9F}, - {0x2018, 0x0000, 0x0000, 0x91}, - {0x2019, 0x0000, 0x0000, 0x92}, - {0x201C, 0x0000, 0x0000, 0x93}, - {0x201C, 0x0000, 0x0000, 0x94} -}; - -static const ushort TsToUn [][3] = { - // Starting at 0x80 - {0x0BE6, 0x0000, 0x0000}, - {0x0BE7, 0x0000, 0x0000}, - {0x0000, 0x0000, 0x0000}, // unknown - {0x0B9C, 0x0000, 0x0000}, - {0x0BB7, 0x0000, 0x0000}, - {0x0BB8, 0x0000, 0x0000}, - {0x0BB9, 0x0000, 0x0000}, - {0x0BE7, 0x0BB7, 0x0000}, - {0x0B9C, 0x0B82, 0x0000}, - {0x0BB7, 0x0B82, 0x0000}, - {0x0BB8, 0x0B82, 0x0000}, - {0x0BB9, 0x0B82, 0x0000}, - {0x0BE7, 0x0BB7, 0x0B82}, - {0x0BE8, 0x0000, 0x0000}, - {0x0BE9, 0x0000, 0x0000}, - {0x0BEA, 0x0000, 0x0000}, - {0x0BEB, 0x0000, 0x0000}, - {0x2018, 0x0000, 0x0000}, - {0x2019, 0x0000, 0x0000}, - {0x201C, 0x0000, 0x0000}, - {0x201C, 0x0000, 0x0000}, // two of the same?? - {0x0BEC, 0x0000, 0x0000}, - {0x0BED, 0x0000, 0x0000}, - {0x0BEE, 0x0000, 0x0000}, - {0x0BEF, 0x0000, 0x0000}, - {0x0B99, 0x0BC1, 0x0000}, - {0x0B9E, 0x0BC1, 0x0000}, - {0x0B99, 0x0BC2, 0x0000}, - {0x0B9E, 0x0BC2, 0x0000}, - {0x0BF0, 0x0000, 0x0000}, - {0x0BF1, 0x0000, 0x0000}, - {0x0BF2, 0x0000, 0x0000}, - {0x00A0, 0x0000, 0x0000}, - {0x0BBE, 0x0000, 0x0000}, - {0x0BBF, 0x0000, 0x0000}, - {0x0BC0, 0x0000, 0x0000}, - {0x0BC1, 0x0000, 0x0000}, - {0x0BC2, 0x0000, 0x0000}, - {0x0BC6, 0x0000, 0x0000}, - {0x0BC7, 0x0000, 0x0000}, - {0x0BC8, 0x0000, 0x0000}, - {0x00A9, 0x0000, 0x0000}, - {0x0BCC, 0x0000, 0x0000}, - {0x0B85, 0x0000, 0x0000}, - {0x0B86, 0x0000, 0x0000}, - {0x0B87, 0x0000, 0x0000}, - {0x0B88, 0x0000, 0x0000}, - {0x0B89, 0x0000, 0x0000}, - {0x0B8A, 0x0000, 0x0000}, - {0x0B8E, 0x0000, 0x0000}, - {0x0B8F, 0x0000, 0x0000}, - {0x0B90, 0x0000, 0x0000}, - {0x0B92, 0x0000, 0x0000}, - {0x0B93, 0x0000, 0x0000}, - {0x0B94, 0x0000, 0x0000}, - {0x0B83, 0x0000, 0x0000}, - {0x0B95, 0x0000, 0x0000}, - {0x0B99, 0x0000, 0x0000}, - {0x0B9A, 0x0000, 0x0000}, - {0x0B9E, 0x0000, 0x0000}, - {0x0B9F, 0x0000, 0x0000}, - {0x0BA3, 0x0000, 0x0000}, - {0x0BA4, 0x0000, 0x0000}, - {0x0BA8, 0x0000, 0x0000}, - {0x0BAA, 0x0000, 0x0000}, - {0x0BAE, 0x0000, 0x0000}, - {0x0BAF, 0x0000, 0x0000}, - {0x0BB0, 0x0000, 0x0000}, - {0x0BB2, 0x0000, 0x0000}, - {0x0BB5, 0x0000, 0x0000}, - {0x0BB4, 0x0000, 0x0000}, - {0x0BB3, 0x0000, 0x0000}, - {0x0BB1, 0x0000, 0x0000}, - {0x0BA9, 0x0000, 0x0000}, - {0x0B9F, 0x0BBF, 0x0000}, - {0x0B9F, 0x0BC0, 0x0000}, - {0x0B95, 0x0BC1, 0x0000}, - {0x0B9A, 0x0BC1, 0x0000}, - {0x0B9F, 0x0BC1, 0x0000}, - {0x0BA3, 0x0BC1, 0x0000}, - {0x0BA4, 0x0BC1, 0x0000}, - {0x0BA8, 0x0BC1, 0x0000}, - {0x0BAA, 0x0BC1, 0x0000}, - {0x0BAE, 0x0BC1, 0x0000}, - {0x0BAF, 0x0BC1, 0x0000}, - {0x0BB0, 0x0BC1, 0x0000}, - {0x0BB2, 0x0BC1, 0x0000}, - {0x0BB5, 0x0BC1, 0x0000}, - {0x0BB4, 0x0BC1, 0x0000}, - {0x0BB3, 0x0BC1, 0x0000}, - {0x0BB1, 0x0BC1, 0x0000}, - {0x0BA9, 0x0BC1, 0x0000}, - {0x0B95, 0x0BC2, 0x0000}, - {0x0B9A, 0x0BC2, 0x0000}, - {0x0B9F, 0x0BC2, 0x0000}, - {0x0BA3, 0x0BC2, 0x0000}, - {0x0BA4, 0x0BC2, 0x0000}, - {0x0BA8, 0x0BC2, 0x0000}, - {0x0BAA, 0x0BC2, 0x0000}, - {0x0BAE, 0x0BC2, 0x0000}, - {0x0BAF, 0x0BC2, 0x0000}, - {0x0BB0, 0x0BC2, 0x0000}, - {0x0BB2, 0x0BC2, 0x0000}, - {0x0BB5, 0x0BC2, 0x0000}, - {0x0BB4, 0x0BC2, 0x0000}, - {0x0BB3, 0x0BC2, 0x0000}, - {0x0BB1, 0x0BC2, 0x0000}, - {0x0BA9, 0x0BC2, 0x0000}, - {0x0B95, 0x0B82, 0x0000}, - {0x0B99, 0x0B82, 0x0000}, - {0x0B9A, 0x0B82, 0x0000}, - {0x0B9E, 0x0B82, 0x0000}, - {0x0B9F, 0x0B82, 0x0000}, - {0x0BA3, 0x0B82, 0x0000}, - {0x0BA1, 0x0B82, 0x0000}, - {0x0BA8, 0x0B82, 0x0000}, - {0x0BAA, 0x0B82, 0x0000}, - {0x0BAE, 0x0B82, 0x0000}, - {0x0BAF, 0x0B82, 0x0000}, - {0x0BB0, 0x0B82, 0x0000}, - {0x0BB2, 0x0B82, 0x0000}, - {0x0BB5, 0x0B82, 0x0000}, - {0x0BB4, 0x0B82, 0x0000}, - {0x0BB3, 0x0B82, 0x0000}, - {0x0BB1, 0x0B82, 0x0000}, - {0x0BA9, 0x0B82, 0x0000} -}; - -static int cmp(const ushort *s1, const ushort *s2, size_t len) -{ - int diff = 0; - - while (len-- && (diff = *s1++ - *s2++) == 0) - ; - - return diff; -} - -static unsigned char qt_UnicodeToTSCII(ushort u1, ushort u2, ushort u3) -{ - ushort s[3]; - s[0] = u1; - s[1] = u2; - s[2] = u3; - - int a = 0; // start pos - int b = UnToTsLast; // end pos - - // do a binary search for the composed unicode in the list - while (a <= b) { - int w = (a + b) / 2; - int j = cmp(UnToTs[w], s, 3); - - if (j == 0) - // found it - return UnToTs[w][3]; - - if (j < 0) - a = w + 1; - else - b = w - 1; - } - - return 0; -} - -static unsigned int qt_TSCIIToUnicode(uint code, uint *s) -{ - int len = 0; - for (int i = 0; i < 3; i++) { - uint u = TsToUn[code & 0x7f][i]; - s[i] = u; - if (s[i]) len = i + 1; - } - - return len; -} - -QT_END_NAMESPACE - -#endif // QT_NO_CODECS diff --git a/src/core/codecs/qtsciicodec_p.h b/src/core/codecs/qtsciicodec_p.h deleted file mode 100644 index c901592d4..000000000 --- a/src/core/codecs/qtsciicodec_p.h +++ /dev/null @@ -1,98 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -// Most of the code here was originally written by Hans Petter Bieker, -// and is included in Qt with the author's permission, and the grateful -// thanks of the Qt team. - -/* - * Copyright (C) 2000 Hans Petter Bieker. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef QTSCIICODEC_P_H -#define QTSCIICODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists for the convenience -// of other Qt classes. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "QtCore/qtextcodec.h" - -QT_BEGIN_NAMESPACE - -#ifndef QT_NO_CODECS - -class QTsciiCodec : public QTextCodec { -public: - ~QTsciiCodec(); - - QByteArray name() const; - int mibEnum() const; - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; -}; - -#endif // QT_NO_CODECS - -QT_END_NAMESPACE - -#endif // QTSCIICODEC_P_H diff --git a/src/core/codecs/qutfcodec.cpp b/src/core/codecs/qutfcodec.cpp deleted file mode 100644 index 9edaab3f9..000000000 --- a/src/core/codecs/qutfcodec.cpp +++ /dev/null @@ -1,658 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qutfcodec_p.h" -#include "qlist.h" -#include "qendian.h" -#include "qchar.h" - -QT_BEGIN_NAMESPACE - -enum { Endian = 0, Data = 1 }; - -static inline bool isUnicodeNonCharacter(uint ucs4) -{ - // Unicode has a couple of "non-characters" that one can use internally, - // but are not allowed to be used for text interchange. - // - // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF, - // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and - // U+FDEF (inclusive) - - return (ucs4 & 0xfffe) == 0xfffe || (ucs4 - 0xfdd0U) < 16; -} - -QByteArray QUtf8::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state) -{ - uchar replacement = '?'; - int rlen = 3*len; - int surrogate_high = -1; - if (state) { - if (state->flags & QTextCodec::ConvertInvalidToNull) - replacement = 0; - if (!(state->flags & QTextCodec::IgnoreHeader)) - rlen += 3; - if (state->remainingChars) - surrogate_high = state->state_data[0]; - } - - QByteArray rstr(rlen, Qt::Uninitialized); - uchar* cursor = (uchar*)rstr.data(); - const QChar *ch = uc; - int invalid = 0; - if (state && !(state->flags & QTextCodec::IgnoreHeader)) { - *cursor++ = 0xef; - *cursor++ = 0xbb; - *cursor++ = 0xbf; - } - - const QChar *end = ch + len; - while (ch < end) { - uint u = ch->unicode(); - if (surrogate_high >= 0) { - if (ch->isLowSurrogate()) { - u = QChar::surrogateToUcs4(surrogate_high, u); - surrogate_high = -1; - } else { - // high surrogate without low - *cursor = replacement; - ++ch; - ++invalid; - surrogate_high = -1; - continue; - } - } else if (ch->isLowSurrogate()) { - // low surrogate without high - *cursor = replacement; - ++ch; - ++invalid; - continue; - } else if (ch->isHighSurrogate()) { - surrogate_high = u; - ++ch; - continue; - } - - if (u < 0x80) { - *cursor++ = (uchar)u; - } else { - if (u < 0x0800) { - *cursor++ = 0xc0 | ((uchar) (u >> 6)); - } else { - // is it one of the Unicode non-characters? - if (isUnicodeNonCharacter(u)) { - *cursor++ = replacement; - ++ch; - ++invalid; - continue; - } - - if (u > 0xffff) { - *cursor++ = 0xf0 | ((uchar) (u >> 18)); - *cursor++ = 0x80 | (((uchar) (u >> 12)) & 0x3f); - } else { - *cursor++ = 0xe0 | (((uchar) (u >> 12)) & 0x3f); - } - *cursor++ = 0x80 | (((uchar) (u >> 6)) & 0x3f); - } - *cursor++ = 0x80 | ((uchar) (u&0x3f)); - } - ++ch; - } - - rstr.resize(cursor - (const uchar*)rstr.constData()); - if (state) { - state->invalidChars += invalid; - state->flags |= QTextCodec::IgnoreHeader; - state->remainingChars = 0; - if (surrogate_high >= 0) { - state->remainingChars = 1; - state->state_data[0] = surrogate_high; - } - } - return rstr; -} - -QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state) -{ - bool headerdone = false; - ushort replacement = QChar::ReplacementCharacter; - int need = 0; - int error = -1; - uint uc = 0; - uint min_uc = 0; - if (state) { - if (state->flags & QTextCodec::IgnoreHeader) - headerdone = true; - if (state->flags & QTextCodec::ConvertInvalidToNull) - replacement = QChar::Null; - need = state->remainingChars; - if (need) { - uc = state->state_data[0]; - min_uc = state->state_data[1]; - } - } - if (!headerdone && len > 3 - && (uchar)chars[0] == 0xef && (uchar)chars[1] == 0xbb && (uchar)chars[2] == 0xbf) { - // starts with a byte order mark - chars += 3; - len -= 3; - headerdone = true; - } - - QString result(need + len + 1, Qt::Uninitialized); // worst case - ushort *qch = (ushort *)result.unicode(); - uchar ch; - int invalid = 0; - - for (int i = 0; i < len; ++i) { - ch = chars[i]; - if (need) { - if ((ch&0xc0) == 0x80) { - uc = (uc << 6) | (ch & 0x3f); - --need; - if (!need) { - // utf-8 bom composes into 0xfeff code point - bool nonCharacter; - if (!headerdone && uc == 0xfeff) { - // don't do anything, just skip the BOM - } else if (!(nonCharacter = isUnicodeNonCharacter(uc)) && uc > 0xffff && uc < 0x110000) { - // surrogate pair - Q_ASSERT((qch - (ushort*)result.unicode()) + 2 < result.length()); - *qch++ = QChar::highSurrogate(uc); - *qch++ = QChar::lowSurrogate(uc); - } else if ((uc < min_uc) || (uc >= 0xd800 && uc <= 0xdfff) || nonCharacter || uc >= 0x110000) { - // error: overlong sequence, UTF16 surrogate or non-character - *qch++ = replacement; - ++invalid; - } else { - *qch++ = uc; - } - headerdone = true; - } - } else { - // error - i = error; - *qch++ = replacement; - ++invalid; - need = 0; - headerdone = true; - } - } else { - if (ch < 128) { - *qch++ = ushort(ch); - headerdone = true; - } else if ((ch & 0xe0) == 0xc0) { - uc = ch & 0x1f; - need = 1; - error = i; - min_uc = 0x80; - headerdone = true; - } else if ((ch & 0xf0) == 0xe0) { - uc = ch & 0x0f; - need = 2; - error = i; - min_uc = 0x800; - } else if ((ch&0xf8) == 0xf0) { - uc = ch & 0x07; - need = 3; - error = i; - min_uc = 0x10000; - headerdone = true; - } else { - // error - *qch++ = replacement; - ++invalid; - headerdone = true; - } - } - } - if (!state && need > 0) { - // unterminated UTF sequence - for (int i = error; i < len; ++i) { - *qch++ = replacement; - ++invalid; - } - } - result.truncate(qch - (ushort *)result.unicode()); - if (state) { - state->invalidChars += invalid; - state->remainingChars = need; - if (headerdone) - state->flags |= QTextCodec::IgnoreHeader; - state->state_data[0] = need ? uc : 0; - state->state_data[1] = need ? min_uc : 0; - } - return result; -} - -QByteArray QUtf16::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state, DataEndianness e) -{ - DataEndianness endian = e; - int length = 2*len; - if (!state || (!(state->flags & QTextCodec::IgnoreHeader))) { - length += 2; - } - if (e == DetectEndianness) { - endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; - } - - QByteArray result(length, Qt::Uninitialized); - char *data = result.data(); - if (!state || !(state->flags & QTextCodec::IgnoreHeader)) { - QChar bom(QChar::ByteOrderMark); - if (endian == BigEndianness) { - data[0] = bom.row(); - data[1] = bom.cell(); - } else { - data[0] = bom.cell(); - data[1] = bom.row(); - } - data += 2; - } - if (endian == BigEndianness) { - for (int i = 0; i < len; ++i) { - *(data++) = uc[i].row(); - *(data++) = uc[i].cell(); - } - } else { - for (int i = 0; i < len; ++i) { - *(data++) = uc[i].cell(); - *(data++) = uc[i].row(); - } - } - - if (state) { - state->remainingChars = 0; - state->flags |= QTextCodec::IgnoreHeader; - } - return result; -} - -QString QUtf16::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state, DataEndianness e) -{ - DataEndianness endian = e; - bool half = false; - uchar buf = 0; - bool headerdone = false; - if (state) { - headerdone = state->flags & QTextCodec::IgnoreHeader; - if (endian == DetectEndianness) - endian = (DataEndianness)state->state_data[Endian]; - if (state->remainingChars) { - half = true; - buf = state->state_data[Data]; - } - } - if (headerdone && endian == DetectEndianness) - endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; - - QString result(len, Qt::Uninitialized); // worst case - QChar *qch = (QChar *)result.unicode(); - while (len--) { - if (half) { - QChar ch; - if (endian == LittleEndianness) { - ch.setRow(*chars++); - ch.setCell(buf); - } else { - ch.setRow(buf); - ch.setCell(*chars++); - } - if (!headerdone) { - headerdone = true; - if (endian == DetectEndianness) { - if (ch == QChar::ByteOrderSwapped) { - endian = LittleEndianness; - } else if (ch == QChar::ByteOrderMark) { - endian = BigEndianness; - } else { - if (QSysInfo::ByteOrder == QSysInfo::BigEndian) { - endian = BigEndianness; - } else { - endian = LittleEndianness; - ch = QChar((ch.unicode() >> 8) | ((ch.unicode() & 0xff) << 8)); - } - *qch++ = ch; - } - } else if (ch != QChar::ByteOrderMark) { - *qch++ = ch; - } - } else { - *qch++ = ch; - } - half = false; - } else { - buf = *chars++; - half = true; - } - } - result.truncate(qch - result.unicode()); - - if (state) { - if (headerdone) - state->flags |= QTextCodec::IgnoreHeader; - state->state_data[Endian] = endian; - if (half) { - state->remainingChars = 1; - state->state_data[Data] = buf; - } else { - state->remainingChars = 0; - state->state_data[Data] = 0; - } - } - return result; -} - -QByteArray QUtf32::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state, DataEndianness e) -{ - DataEndianness endian = e; - int length = 4*len; - if (!state || (!(state->flags & QTextCodec::IgnoreHeader))) { - length += 4; - } - if (e == DetectEndianness) { - endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; - } - - QByteArray d(length, Qt::Uninitialized); - char *data = d.data(); - if (!state || !(state->flags & QTextCodec::IgnoreHeader)) { - if (endian == BigEndianness) { - data[0] = 0; - data[1] = 0; - data[2] = (char)0xfe; - data[3] = (char)0xff; - } else { - data[0] = (char)0xff; - data[1] = (char)0xfe; - data[2] = 0; - data[3] = 0; - } - data += 4; - } - if (endian == BigEndianness) { - for (int i = 0; i < len; ++i) { - uint cp = uc[i].unicode(); - if (uc[i].isHighSurrogate() && i < len - 1) - cp = QChar::surrogateToUcs4(cp, uc[++i].unicode()); - *(data++) = cp >> 24; - *(data++) = (cp >> 16) & 0xff; - *(data++) = (cp >> 8) & 0xff; - *(data++) = cp & 0xff; - } - } else { - for (int i = 0; i < len; ++i) { - uint cp = uc[i].unicode(); - if (uc[i].isHighSurrogate() && i < len - 1) - cp = QChar::surrogateToUcs4(cp, uc[++i].unicode()); - *(data++) = cp & 0xff; - *(data++) = (cp >> 8) & 0xff; - *(data++) = (cp >> 16) & 0xff; - *(data++) = cp >> 24; - } - } - - if (state) { - state->remainingChars = 0; - state->flags |= QTextCodec::IgnoreHeader; - } - return d; -} - -QString QUtf32::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state, DataEndianness e) -{ - DataEndianness endian = e; - uchar tuple[4]; - int num = 0; - bool headerdone = false; - if (state) { - headerdone = state->flags & QTextCodec::IgnoreHeader; - if (endian == DetectEndianness) { - endian = (DataEndianness)state->state_data[Endian]; - } - num = state->remainingChars; - memcpy(tuple, &state->state_data[Data], 4); - } - if (headerdone && endian == DetectEndianness) - endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; - - QString result((num + len) >> 2 << 1, Qt::Uninitialized); // worst case - QChar *qch = (QChar *)result.unicode(); - - const char *end = chars + len; - while (chars < end) { - tuple[num++] = *chars++; - if (num == 4) { - if (!headerdone) { - if (endian == DetectEndianness) { - if (endian == DetectEndianness) { - if (tuple[0] == 0xff && tuple[1] == 0xfe && tuple[2] == 0 && tuple[3] == 0 && endian != BigEndianness) { - endian = LittleEndianness; - num = 0; - continue; - } else if (tuple[0] == 0 && tuple[1] == 0 && tuple[2] == 0xfe && tuple[3] == 0xff && endian != LittleEndianness) { - endian = BigEndianness; - num = 0; - continue; - } else if (QSysInfo::ByteOrder == QSysInfo::BigEndian) { - endian = BigEndianness; - } else { - endian = LittleEndianness; - } - } - } else if (((endian == BigEndianness) ? qFromBigEndian(tuple) : qFromLittleEndian(tuple)) == QChar::ByteOrderMark) { - num = 0; - continue; - } - } - uint code = (endian == BigEndianness) ? qFromBigEndian(tuple) : qFromLittleEndian(tuple); - if (code >= 0x10000) { - *qch++ = QChar::highSurrogate(code); - *qch++ = QChar::lowSurrogate(code); - } else { - *qch++ = code; - } - num = 0; - } - } - result.truncate(qch - result.unicode()); - - if (state) { - if (headerdone) - state->flags |= QTextCodec::IgnoreHeader; - state->state_data[Endian] = endian; - state->remainingChars = num; - memcpy(&state->state_data[Data], tuple, 4); - } - return result; -} - - -#ifndef QT_NO_TEXTCODEC - -QUtf8Codec::~QUtf8Codec() -{ -} - -QByteArray QUtf8Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const -{ - return QUtf8::convertFromUnicode(uc, len, state); -} - -void QUtf8Codec::convertToUnicode(QString *target, const char *chars, int len, ConverterState *state) const -{ - *target += QUtf8::convertToUnicode(chars, len, state); -} - -QString QUtf8Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const -{ - return QUtf8::convertToUnicode(chars, len, state); -} - -QByteArray QUtf8Codec::name() const -{ - return "UTF-8"; -} - -int QUtf8Codec::mibEnum() const -{ - return 106; -} - -QUtf16Codec::~QUtf16Codec() -{ -} - -QByteArray QUtf16Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const -{ - return QUtf16::convertFromUnicode(uc, len, state, e); -} - -QString QUtf16Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const -{ - return QUtf16::convertToUnicode(chars, len, state, e); -} - -int QUtf16Codec::mibEnum() const -{ - return 1015; -} - -QByteArray QUtf16Codec::name() const -{ - return "UTF-16"; -} - -QList QUtf16Codec::aliases() const -{ - return QList(); -} - -int QUtf16BECodec::mibEnum() const -{ - return 1013; -} - -QByteArray QUtf16BECodec::name() const -{ - return "UTF-16BE"; -} - -QList QUtf16BECodec::aliases() const -{ - QList list; - return list; -} - -int QUtf16LECodec::mibEnum() const -{ - return 1014; -} - -QByteArray QUtf16LECodec::name() const -{ - return "UTF-16LE"; -} - -QList QUtf16LECodec::aliases() const -{ - QList list; - return list; -} - -QUtf32Codec::~QUtf32Codec() -{ -} - -QByteArray QUtf32Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const -{ - return QUtf32::convertFromUnicode(uc, len, state, e); -} - -QString QUtf32Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const -{ - return QUtf32::convertToUnicode(chars, len, state, e); -} - -int QUtf32Codec::mibEnum() const -{ - return 1017; -} - -QByteArray QUtf32Codec::name() const -{ - return "UTF-32"; -} - -QList QUtf32Codec::aliases() const -{ - QList list; - return list; -} - -int QUtf32BECodec::mibEnum() const -{ - return 1018; -} - -QByteArray QUtf32BECodec::name() const -{ - return "UTF-32BE"; -} - -QList QUtf32BECodec::aliases() const -{ - QList list; - return list; -} - -int QUtf32LECodec::mibEnum() const -{ - return 1019; -} - -QByteArray QUtf32LECodec::name() const -{ - return "UTF-32LE"; -} - -QList QUtf32LECodec::aliases() const -{ - QList list; - return list; -} - -#endif //QT_NO_TEXTCODEC - -QT_END_NAMESPACE diff --git a/src/core/codecs/qutfcodec_p.h b/src/core/codecs/qutfcodec_p.h deleted file mode 100644 index 89a1dad05..000000000 --- a/src/core/codecs/qutfcodec_p.h +++ /dev/null @@ -1,162 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2015 The Qt Company Ltd. -** Copyright (C) 2016-2019 Ivailo Monev -** -** This file is part of the QtCore module of the Katie Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 2.1 or version 3 as published by the Free -** Software Foundation and appearing in the file LICENSE.LGPLv21 and -** LICENSE.LGPLv3 included in the packaging of this file. Please review the -** following information to ensure the GNU Lesser General Public License -** requirements will be met: https://www.gnu.org/licenses/lgpl.html and -** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -** -** As a special exception, The Qt Company gives you certain additional -** rights. These rights are described in The Qt Company LGPL Exception -** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3.0 as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL included in the -** packaging of this file. Please review the following information to -** ensure the GNU General Public License version 3.0 requirements will be -** met: http://www.gnu.org/copyleft/gpl.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QUTFCODEC_P_H -#define QUTFCODEC_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Katie API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "QtCore/qtextcodec.h" -#include "qtextcodec_p.h" - -QT_BEGIN_NAMESPACE - -enum DataEndianness -{ - DetectEndianness, - BigEndianness, - LittleEndianness -}; - -struct QUtf8 -{ - static QString convertToUnicode(const char *, int, QTextCodec::ConverterState *); - static QByteArray convertFromUnicode(const QChar *, int, QTextCodec::ConverterState *); -}; - -struct QUtf16 -{ - static QString convertToUnicode(const char *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness); - static QByteArray convertFromUnicode(const QChar *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness); -}; - -struct QUtf32 -{ - static QString convertToUnicode(const char *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness); - static QByteArray convertFromUnicode(const QChar *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness); -}; - -#ifndef QT_NO_TEXTCODEC - -class QUtf8Codec : public QTextCodec { -public: - ~QUtf8Codec(); - - QByteArray name() const; - int mibEnum() const; - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - void convertToUnicode(QString *target, const char *, int, ConverterState *) const; -}; - -class QUtf16Codec : public QTextCodec { -protected: -public: - QUtf16Codec() { e = DetectEndianness; } - ~QUtf16Codec(); - - QByteArray name() const; - QList aliases() const; - int mibEnum() const; - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - -protected: - DataEndianness e; -}; - -class QUtf16BECodec : public QUtf16Codec { -public: - QUtf16BECodec() : QUtf16Codec() { e = BigEndianness; } - QByteArray name() const; - QList aliases() const; - int mibEnum() const; -}; - -class QUtf16LECodec : public QUtf16Codec { -public: - QUtf16LECodec() : QUtf16Codec() { e = LittleEndianness; } - QByteArray name() const; - QList aliases() const; - int mibEnum() const; -}; - -class QUtf32Codec : public QTextCodec { -public: - QUtf32Codec() { e = DetectEndianness; } - ~QUtf32Codec(); - - QByteArray name() const; - QList aliases() const; - int mibEnum() const; - - QString convertToUnicode(const char *, int, ConverterState *) const; - QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const; - -protected: - DataEndianness e; -}; - -class QUtf32BECodec : public QUtf32Codec { -public: - QUtf32BECodec() : QUtf32Codec() { e = BigEndianness; } - QByteArray name() const; - QList aliases() const; - int mibEnum() const; -}; - -class QUtf32LECodec : public QUtf32Codec { -public: - QUtf32LECodec() : QUtf32Codec() { e = LittleEndianness; } - QByteArray name() const; - QList aliases() const; - int mibEnum() const; -}; - - -#endif // QT_NO_TEXTCODEC - -QT_END_NAMESPACE - -#endif // QUTFCODEC_P_H diff --git a/src/core/global/qconfig.h.cmake b/src/core/global/qconfig.h.cmake index 71c3d64e3..85be51167 100644 --- a/src/core/global/qconfig.h.cmake +++ b/src/core/global/qconfig.h.cmake @@ -89,6 +89,7 @@ #define QT_NO_RAWFONT #define QT_NO_IMAGE_TEXT #define QT_NO_GLIB +#define QT_NO_ICONV /* Qt build specs */ #ifndef QT_EDITION @@ -179,7 +180,6 @@ #cmakedefine QT_NO_GROUPBOX #cmakedefine QT_NO_HTTP #cmakedefine QT_NO_ICON -#cmakedefine QT_NO_ICONV #cmakedefine QT_NO_IDENTITYPROXYMODEL #cmakedefine QT_NO_IMAGEFORMATPLUGIN #cmakedefine QT_NO_IMAGEFORMAT_BMP diff --git a/src/core/io/qtextstream.cpp b/src/core/io/qtextstream.cpp index cffbf4cd0..d0c42b809 100644 --- a/src/core/io/qtextstream.cpp +++ b/src/core/io/qtextstream.cpp @@ -457,9 +457,6 @@ static void copyConverterStateHelper(QTextCodec::ConverterState *dest, // private. This function copies the structure manually. dest->flags = src->flags; dest->invalidChars = src->invalidChars; - dest->state_data[0] = src->state_data[0]; - dest->state_data[1] = src->state_data[1]; - dest->state_data[2] = src->state_data[2]; } #endif diff --git a/src/core/tools/qstring.cpp b/src/core/tools/qstring.cpp index ef5357a90..729ecc48d 100644 --- a/src/core/tools/qstring.cpp +++ b/src/core/tools/qstring.cpp @@ -36,7 +36,7 @@ #ifndef QT_NO_TEXTCODEC #include "qtextcodec.h" #endif -#include "qutfcodec_p.h" +#include "qicucodec_p.h" #include "qdatastream.h" #include "qlist.h" #include "qlocale.h" @@ -3455,7 +3455,8 @@ QByteArray QString::toUtf8() const if (isNull()) return QByteArray(); - return QUtf8::convertFromUnicode(constData(), length(), Q_NULLPTR); + QTextCodec *c = QTextCodec::codecForName("UTF-8"); + return c->fromUnicode(constData(), length(), Q_NULLPTR); } /*! @@ -3618,7 +3619,8 @@ QString QString::fromUtf8(const char *str, int size) if (size < 0) size = qstrlen(str); - return QUtf8::convertToUnicode(str, size, Q_NULLPTR); + QTextCodec *c = QTextCodec::codecForName("UTF-8"); + return c->toUnicode(str, size, Q_NULLPTR); } /*! @@ -3647,7 +3649,8 @@ QString QString::fromUtf16(const ushort *unicode, int size) while (unicode[size] != 0) ++size; } - return QUtf16::convertToUnicode((const char *)unicode, size*2, Q_NULLPTR); + QTextCodec *c = QTextCodec::codecForName("UTF-16"); + return c->toUnicode((const char *)unicode, size, Q_NULLPTR); } @@ -3671,7 +3674,8 @@ QString QString::fromUcs4(const uint *unicode, int size) while (unicode[size] != 0) ++size; } - return QUtf32::convertToUnicode((const char *)unicode, size*4, Q_NULLPTR); + QTextCodec *c = QTextCodec::codecForName("UTF-32"); + return c->toUnicode((const char *)unicode, size, Q_NULLPTR); } /*! @@ -8535,7 +8539,8 @@ QByteArray QStringRef::toUtf8() const if (isNull()) return QByteArray(); - return QUtf8::convertFromUnicode(constData(), length(), Q_NULLPTR); + QTextCodec *c = QTextCodec::codecForName("UTF-8"); + return c->fromUnicode(constData(), length(), Q_NULLPTR); } /*! diff --git a/src/gui/kernel/qapplication_x11.cpp b/src/gui/kernel/qapplication_x11.cpp index e83be8085..a955e0cea 100644 --- a/src/gui/kernel/qapplication_x11.cpp +++ b/src/gui/kernel/qapplication_x11.cpp @@ -886,10 +886,10 @@ static void qt_set_input_encoding() qt_input_mapper = QTextCodec::codecForName((char *)data); // make sure we have an input codec if(!qt_input_mapper) - qt_input_mapper = QTextCodec::codecForName("ISO 8859-1"); + qt_input_mapper = QTextCodec::codecForName("ISO-8859-1"); } if (qt_input_mapper && qt_input_mapper->mibEnum() == 11) // 8859-8 - qt_input_mapper = QTextCodec::codecForName("ISO 8859-8-I"); + qt_input_mapper = QTextCodec::codecForName("ISO-8859-8-I"); if(data) XFree((char *)data); } diff --git a/src/tools/moc/CMakeLists.txt b/src/tools/moc/CMakeLists.txt index 7f41d68bf..ffc325619 100644 --- a/src/tools/moc/CMakeLists.txt +++ b/src/tools/moc/CMakeLists.txt @@ -42,12 +42,8 @@ set(BOOTSTRAP_DEFINITIONS ) set(BOOTSTRAP_SOURCES - ${CMAKE_SOURCE_DIR}/src/core/codecs/qisciicodec.cpp - ${CMAKE_SOURCE_DIR}/src/core/codecs/qlatincodec.cpp - ${CMAKE_SOURCE_DIR}/src/core/codecs/qsimplecodec.cpp ${CMAKE_SOURCE_DIR}/src/core/codecs/qtextcodec.cpp - ${CMAKE_SOURCE_DIR}/src/core/codecs/qtsciicodec.cpp - ${CMAKE_SOURCE_DIR}/src/core/codecs/qutfcodec.cpp + ${CMAKE_SOURCE_DIR}/src/core/codecs/qicucodec.cpp ${CMAKE_SOURCE_DIR}/src/core/global/qglobal.cpp ${CMAKE_SOURCE_DIR}/src/core/global/qnumeric.cpp ${CMAKE_SOURCE_DIR}/src/core/io/qabstractfileengine.cpp diff --git a/tests/benchmarks/core/tools/qstring/main.cpp b/tests/benchmarks/core/tools/qstring/main.cpp index 779e6628d..2090eb730 100644 --- a/tests/benchmarks/core/tools/qstring/main.cpp +++ b/tests/benchmarks/core/tools/qstring/main.cpp @@ -656,10 +656,6 @@ int fromUtf8_qt47(ushort *dst, const char *chars, int len) if (state->flags & QTextCodec::ConvertInvalidToNull) replacement = QChar::Null; need = state->remainingChars; - if (need) { - uc = state->state_data[0]; - min_uc = state->state_data[1]; - } } if (!headerdone && len > 3 && (uchar)chars[0] == 0xef && (uchar)chars[1] == 0xbb && (uchar)chars[2] == 0xbf) { @@ -750,8 +746,6 @@ int fromUtf8_qt47(ushort *dst, const char *chars, int len) state->remainingChars = need; if (headerdone) state->flags |= QTextCodec::IgnoreHeader; - state->state_data[0] = need ? uc : 0; - state->state_data[1] = need ? min_uc : 0; } //return result; return qch - dst;