Katana_mirror/katie
1
0
Fork 0
mirror of https://bitbucket.org/smil3y/katie.git synced 2025-02-24 02:42:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

backport ICU text codec

Browse source 
Signed-off-by: Ivailo Monev <xakepa10@laimg.moc>
This commit is contained in:
Ivailo Monev 2019-11-22 23:48:18 +00:00
parent 047d662eda
commit 82f1d03b02
27 changed files with 794 additions and 3942 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
CMakeLists.txt
View file

@ -127,9 +127,6 @@ add_feature_info(tiff WITH_TIFF "build TIFF support")
option(WITH_MNG "Build MNG support" ON)
add_feature_info(mng WITH_MNG "build MNG support")
option(WITH_ICONV "Build Iconv support" ON)
add_feature_info(iconv WITH_ICONV "build Iconv support")
option(WITH_DBUS "Build DBus support" ON)
add_feature_info(dbus WITH_DBUS "build DBus support")
@ -349,14 +346,6 @@ set_package_properties(MNG PROPERTIES
TYPE RECOMMENDED
)
find_package(Iconv)
set_package_properties(Iconv PROPERTIES
PURPOSE "Iconv support"
DESCRIPTION "iconv() implementation, for use on systems which don't have one or lack Unicode support"
URL "http://www.gnu.org/software/libiconv/"
TYPE RECOMMENDED
)
find_package(Cups)
set_package_properties(Cups PROPERTIES
PURPOSE "Required for printing support"
@ -497,11 +486,6 @@ if(NOT WITH_ACCESSIBILITY)
katie_definition(-DQT_NO_ACCESSIBILITY)
endif()
# optional packages with interface definition only for convenience
if(NOT WITH_ICONV OR NOT ICONV_FOUND)
katie_definition(-DQT_NO_ICONV)
endif()
# keep in sync with the gui component CMake file, only definitions setup is needed here
foreach(x11ext Xshape Xinerama Xrandr Xrender Xfixes Xcursor Xinput)
if(NOT X11_${x11ext}_FOUND)

29
src/core/codecs/codecs.cmake
View file

@ -1,39 +1,14 @@
set(CORE_HEADERS
${CORE_HEADERS}
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qisciicodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qlatincodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qsimplecodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodec.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtsciicodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qutfcodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qicucodec_p.h
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodecplugin.h
)
set(CORE_SOURCES
${CORE_SOURCES}
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qisciicodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qlatincodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qsimplecodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtsciicodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qutfcodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qtextcodecplugin.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qfontlaocodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qicucodec.cpp
)
if(WITH_ICONV AND ICONV_FOUND)
set(CORE_HEADERS
${CORE_HEADERS}
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qiconvcodec_p.h
)
set(CORE_SOURCES
${CORE_SOURCES}
${CMAKE_CURRENT_SOURCE_DIR}/codecs/qiconvcodec.cpp
)
set(EXTRA_CORE_LIBS
${EXTRA_CORE_LIBS}
${ICONV_LIBRARIES}
)
include_directories(${ICONV_INCLUDES})
endif()

115
src/core/codecs/qfontlaocodec.cpp
View file

@ -1,115 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qfontlaocodec_p.h"
#include "qlist.h"
#ifndef QT_NO_CODECS
#ifndef QT_NO_BIG_CODECS
QT_BEGIN_NAMESPACE
static unsigned char const unicode_to_mulelao[256] =
{
// U+0E80
0x00, 0xa1, 0xa2, 0x00, 0xa4, 0x00, 0x00, 0xa7,
0xa8, 0x00, 0xaa, 0x00, 0x00, 0xad, 0x00, 0x00,
// U+0E90
0x00, 0x00, 0x00, 0x00, 0xb4, 0xb5, 0xb6, 0xb7,
0x00, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
// U+0EA0
0x00, 0xc1, 0xc2, 0xc3, 0x00, 0xc5, 0x00, 0xc7,
0x00, 0x00, 0xca, 0xcb, 0x00, 0xcd, 0xce, 0xcf,
// U+0EB0
0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
0xd8, 0xd9, 0x00, 0xdb, 0xdc, 0xdd, 0x00, 0x00,
// U+0EC0
0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0x00, 0xe6, 0x00,
0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0x00, 0x00,
// U+0ED0
0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
0xf8, 0xf9, 0x00, 0x00, 0xfc, 0xfd, 0x00, 0x00,
// U+0EE0
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
// U+0EF0
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
QFontLaoCodec::~QFontLaoCodec()
{
}
QByteArray QFontLaoCodec::name() const
{
return "mulelao-1";
}
int QFontLaoCodec::mibEnum() const
{
return -4242;
}
QString QFontLaoCodec::convertToUnicode(const char *, int, ConverterState *) const
{
return QString();
}
QByteArray QFontLaoCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *) const
{
QByteArray rstring(len, Qt::Uninitialized);
uchar *rdata = (uchar *) rstring.data();
const QChar *sdata = uc;
int i = 0;
for (; i < len; ++i, ++sdata, ++rdata) {
if (sdata->unicode() < 0x80) {
*rdata = (uchar) sdata->unicode();
} else if (sdata->unicode() >= 0x0e80 && sdata->unicode() <= 0x0eff) {
uchar lao = unicode_to_mulelao[sdata->unicode() - 0x0e80];
if (lao)
*rdata = lao;
else
*rdata = 0;
} else {
*rdata = 0;
}
}
return rstring;
}
QT_END_NAMESPACE
#endif // QT_NO_BIG_CODECS
#endif // QT_NO_CODECS

70
src/core/codecs/qfontlaocodec_p.h
View file

@ -1,70 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QFONTLAOCODEC_P_H
#define QFONTLAOCODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists for the convenience
// of qfontencodings_x11.cpp and qfont_x11.cpp. This header file may
// change from version to version without notice, or even be removed.
//
// We mean it.
//
#include "QtCore/qtextcodec.h"
QT_BEGIN_NAMESPACE
#ifndef QT_NO_CODECS
class Q_CORE_EXPORT QFontLaoCodec : public QTextCodec
{
public:
~QFontLaoCodec();
QByteArray name() const;
int mibEnum() const;
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
};
#endif // QT_NO_CODECS
QT_END_NAMESPACE
#endif // QFONTLAOCODEC_P_H

447
src/core/codecs/qiconvcodec.cpp
View file

@ -1,447 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qiconvcodec_p.h"
#include "qtextcodec_p.h"
#include "qlibrary.h"
#include "qdebug.h"
#include <errno.h>
#include <locale.h>
#include <stdio.h>
// unistd.h is needed for the _XOPEN_UNIX macro
#include <unistd.h>
#if defined(_XOPEN_UNIX) && !defined(Q_OS_OSF)
# include <langinfo.h>
#endif
#if defined(Q_OS_HPUX)
# define NO_BOM
# define UTF16 "ucs2"
#elif defined(Q_OS_AIX)
# define NO_BOM
# define UTF16 "UCS-2"
#elif defined(Q_OS_FREEBSD) || (defined(Q_OS_LINUX) && !defined(__GLIBC__))
# define NO_BOM
# if Q_BYTE_ORDER == Q_BIG_ENDIAN
# define UTF16 "UTF-16BE"
# else
# define UTF16 "UTF-16LE"
# endif
#else
# define UTF16 "UTF-16"
#endif
QT_BEGIN_NAMESPACE
extern bool qt_locale_initialized;
QIconvCodec::QIconvCodec()
: utf16Codec(Q_NULLPTR)
{
utf16Codec = QTextCodec::codecForMib(1015);
Q_ASSERT_X(utf16Codec != Q_NULLPTR,
"QIconvCodec::convertToUnicode",
"internal error, UTF-16 codec not found");
if (!utf16Codec) {
fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n");
utf16Codec = Q_NULLPTR;
}
}
QIconvCodec::~QIconvCodec()
{
}
QIconvCodec::IconvState::IconvState(iconv_t x)
: buffer(array), bufferLen(sizeof array), cd(x)
{
}
QIconvCodec::IconvState::~IconvState()
{
if (cd != reinterpret_cast<iconv_t>(-1))
iconv_close(cd);
if (buffer != array)
delete[] buffer;
}
void QIconvCodec::IconvState::saveChars(const char *c, int count)
{
if (count > bufferLen) {
if (buffer != array)
delete[] buffer;
buffer = new char[bufferLen = count];
}
memcpy(buffer, c, count);
}
thread_local QIconvCodec::IconvState* toUnicodeState = Q_NULLPTR;
QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const
{
if (!utf16Codec)
return QString::fromLatin1(chars, len);
int invalidCount = 0;
int remainingCount = 0;
char *remainingBuffer = 0;
IconvState *temporaryState = 0;
IconvState *&state = (qt_locale_initialized && toUnicodeState) ? toUnicodeState : temporaryState;
if (!state) {
// first time, create the state
iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0);
if (cd == reinterpret_cast<iconv_t>(-1)) {
static int reported = 0;
if (!reported++) {
fprintf(stderr,
"QIconvCodec::convertToUnicode: using Latin-1 for conversion, iconv_open failed\n");
}
return QString::fromLatin1(chars, len);
}
state = new IconvState(cd);
}
size_t inBytesLeft = len;
// best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM
char *inBytes = const_cast<char *>(chars);
QByteArray in;
if (remainingCount) {
// we have to prepend the remaining bytes from the previous conversion
inBytesLeft += remainingCount;
in.resize(inBytesLeft);
inBytes = in.data();
memcpy(in.data(), remainingBuffer, remainingCount);
memcpy(in.data() + remainingCount, chars, len);
remainingCount = 0;
}
size_t outBytesLeft = len * 2 + 2;
QByteArray ba(outBytesLeft, Qt::Uninitialized);
char *outBytes = ba.data();
do {
size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft);
if (ret == (size_t) -1) {
if (errno == E2BIG) {
int offset = ba.size() - outBytesLeft;
ba.resize(ba.size() * 2);
outBytes = ba.data() + offset;
outBytesLeft = ba.size() - offset;
continue;
}
if (errno == EILSEQ) {
// conversion stopped because of an invalid character in the sequence
++invalidCount;
} else if (errno == EINVAL && convState) {
// conversion stopped because the remaining inBytesLeft make up
// an incomplete multi-byte sequence; save them for later
state->saveChars(inBytes, inBytesLeft);
remainingCount = inBytesLeft;
break;
}
if (errno == EILSEQ || errno == EINVAL) {
// skip the next character
++inBytes;
--inBytesLeft;
continue;
}
// some other error
// note, cannot use qWarning() since we are implementing the codecForLocale :)
perror("QIconvCodec::convertToUnicode: using Latin-1 for conversion, iconv failed");
if (!convState) {
// reset state
iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
}
delete temporaryState;
return QString::fromLatin1(chars, len);
}
} while (inBytesLeft != 0);
QString s;
if (convState) {
s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft, &state->internalState);
convState->invalidChars = invalidCount;
convState->remainingChars = remainingCount;
} else {
s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft);
// reset state
iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
}
delete temporaryState;
return s;
}
thread_local QIconvCodec::IconvState* fromUnicodeState = Q_NULLPTR;
static bool setByteOrder(iconv_t cd)
{
#if !defined(NO_BOM)
// give iconv() a BOM
char buf[4];
ushort bom[] = { QChar::ByteOrderMark };
char *outBytes = buf;
char *inBytes = reinterpret_cast<char *>(bom);
size_t outBytesLeft = sizeof buf;
size_t inBytesLeft = sizeof bom;
if (iconv(cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
return false;
}
#endif // NO_BOM
return true;
}
QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const
{
char *inBytes;
char *outBytes;
size_t inBytesLeft;
IconvState *temporaryState = 0;
IconvState *&state = (qt_locale_initialized && fromUnicodeState) ? fromUnicodeState : temporaryState;
if (!state) {
iconv_t cd = QIconvCodec::createIconv_t(0, UTF16);
if (cd != reinterpret_cast<iconv_t>(-1)) {
if (!setByteOrder(cd)) {
perror("QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv failed for BOM");
iconv_close(cd);
cd = reinterpret_cast<iconv_t>(-1);
return QString(uc, len).toLatin1();
}
}
state = new IconvState(cd);
}
if (state->cd == reinterpret_cast<iconv_t>(-1)) {
static int reported = 0;
if (!reported++) {
fprintf(stderr,
"QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv_open failed\n");
}
delete temporaryState;
return QString(uc, len).toLatin1();
}
size_t outBytesLeft = len;
QByteArray ba(outBytesLeft, Qt::Uninitialized);
outBytes = ba.data();
// now feed iconv() the real data
inBytes = const_cast<char *>(reinterpret_cast<const char *>(uc));
inBytesLeft = len * sizeof(QChar);
QByteArray in;
if (convState && convState->remainingChars) {
// we have one surrogate char to be prepended
in.resize(sizeof(QChar) + len);
inBytes = in.data();
QChar remaining = convState->state_data[0];
memcpy(in.data(), &remaining, sizeof(QChar));
memcpy(in.data() + sizeof(QChar), uc, inBytesLeft);
inBytesLeft += sizeof(QChar);
convState->remainingChars = 0;
}
int invalidCount = 0;
while (inBytesLeft != 0) {
if (iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
if (errno == EINVAL && convState) {
// buffer ends in a surrogate
Q_ASSERT(inBytesLeft == 2);
convState->remainingChars = 1;
convState->state_data[0] = uc[len - 1].unicode();
break;
}
switch (errno) {
case EILSEQ:
++invalidCount;
// fall through
case EINVAL:
{
inBytes += sizeof(QChar);
inBytesLeft -= sizeof(QChar);
break;
}
case E2BIG:
{
int offset = ba.size() - outBytesLeft;
ba.resize(ba.size() * 2);
outBytes = ba.data() + offset;
outBytesLeft = ba.size() - offset;
break;
}
default:
{
// note, cannot use qWarning() since we are implementing the codecForLocale :)
perror("QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv failed");
// reset to initial state
iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
delete temporaryState;
return QString(uc, len).toLatin1();
}
}
}
}
// reset to initial state
iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
setByteOrder(state->cd);
ba.resize(ba.size() - outBytesLeft);
if (convState)
convState->invalidChars = invalidCount;
delete temporaryState;
return ba;
}
QByteArray QIconvCodec::name() const
{
return "System";
}
int QIconvCodec::mibEnum() const
{
return 0;
}
iconv_t QIconvCodec::createIconv_t(const char *to, const char *from)
{
Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0));
// both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string
const char *codeset = "";
iconv_t cd = iconv_open(to ? to : codeset, from ? from : codeset);
#if defined(_XOPEN_UNIX) && !defined(Q_OS_OSF)
if (cd == (iconv_t) -1) {
codeset = nl_langinfo(CODESET);
if (codeset)
cd = iconv_open(to ? to : codeset, from ? from : codeset);
}
#endif
if (cd == (iconv_t) -1) {
// Very poorly defined and followed standards causes lots of
// code to try to get all the cases... This logic is
// duplicated in QTextCodec, so if you change it here, change
// it there too.
// Try to determine locale codeset from locale name assigned to
// LC_CTYPE category.
// First part is getting that locale name. First try setlocale() which
// definitely knows it, but since we cannot fully trust it, get ready
// to fall back to environment variables.
char * ctype = qstrdup(setlocale(LC_CTYPE, 0));
// Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG
// environment variables.
char * lang = qstrdup(qgetenv("LC_ALL").constData());
if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
if (lang) delete [] lang;
lang = qstrdup(qgetenv("LC_CTYPE").constData());
}
if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
if (lang) delete [] lang;
lang = qstrdup(qgetenv("LANG").constData());
}
// Now try these in order:
// 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
// 2. CODESET from lang if it contains a .CODESET part
// 3. ctype (maybe the locale is named "ISO-8859-1" or something)
// 4. locale (ditto)
// 5. check for "@euro"
// 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
codeset = ctype ? strchr(ctype, '.') : 0;
if (codeset && *codeset == '.') {
++codeset;
cd = iconv_open(to ? to : codeset, from ? from : codeset);
}
// 2. CODESET from lang if it contains a .CODESET part
codeset = lang ? strchr(lang, '.') : 0;
if (cd == (iconv_t) -1 && codeset && *codeset == '.') {
++codeset;
cd = iconv_open(to ? to : codeset, from ? from : codeset);
}
// 3. ctype (maybe the locale is named "ISO-8859-1" or something)
if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0)
cd = iconv_open(to ? to : ctype, from ? from : ctype);
// 4. locale (ditto)
if (cd == (iconv_t) -1 && lang && *lang != 0)
cd = iconv_open(to ? to : lang, from ? from : lang);
// 5. "@euro"
if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro")))
cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15");
delete [] ctype;
delete [] lang;
}
return cd;
}
QT_END_NAMESPACE

93
src/core/codecs/qiconvcodec_p.h
View file

@ -1,93 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QICONVCODEC_P_H
#define QICONVCODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists for the convenience
// of the QLibrary class. This header file may change from
// version to version without notice, or even be removed.
//
// We mean it.
//
#include "qtextcodec.h"
#if !defined(QT_NO_ICONV) && !defined(QT_BOOTSTRAPPED)
#include <iconv.h>
QT_BEGIN_NAMESPACE
class QIconvCodec: public QTextCodec
{
private:
mutable QTextCodec *utf16Codec;
public:
QIconvCodec();
~QIconvCodec();
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
QByteArray name() const;
int mibEnum() const;
static iconv_t createIconv_t(const char *to, const char *from);
class IconvState
{
public:
IconvState(iconv_t x);
~IconvState();
ConverterState internalState;
char *buffer;
int bufferLen;
iconv_t cd;
char array[8];
void saveChars(const char *c, int count);
};
};
QT_END_NAMESPACE
#endif // Q_OS_UNIX && !QT_NO_ICONV && !QT_BOOTSTRAPPED
#endif // QICONVCODEC_P_H

557
src/core/codecs/qicucodec.cpp Normal file
View file

@ -0,0 +1,557 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qicucodec_p.h"
#include "qtextcodec_p.h"
#include "qdebug.h"
#include <unicode/ucnv.h>
QT_BEGIN_NAMESPACE
static inline bool qTextCodecNameMatch(const char *n, const char *h)
{
return ucnv_compareNames(n, h) == 0;
}
/* The list below is generated from http://www.iana.org/assignments/character-sets/
using the snippet of code below:
#include <QtCore>
#include <unicode/ucnv.h>
int main(int argc, char **argv)
{
QCoreApplication app(argc, argv);
QFile file("character-sets.txt");
file.open(QFile::ReadOnly);
QByteArray name;
int mib = -1;
QByteArray nameList;
int pos = 0;
while (!file.atEnd()) {
QByteArray s = file.readLine().trimmed();
if (s.isEmpty()) {
if (mib != -1) {
UErrorCode error = U_ZERO_ERROR;
const char *standard_name = ucnv_getStandardName(name, "MIME", &error);
if (U_FAILURE(error) || !standard_name) {
error = U_ZERO_ERROR;
standard_name = ucnv_getStandardName(name, "IANA", &error);
}
UConverter *conv = ucnv_open(standard_name, &error);
if (!U_FAILURE(error) && conv && standard_name) {
ucnv_close(conv);
printf(" { %d, %d },\n", mib, pos);
nameList += "\"";
nameList += standard_name;
nameList += "\\0\"\n";
pos += strlen(standard_name) + 1;
}
}
name = QByteArray();
mib = -1;
}
if (s.startsWith("Name: ")) {
name = s.mid(5).trimmed();
if (name.indexOf(' ') > 0)
name = name.left(name.indexOf(' '));
}
if (s.startsWith("MIBenum:"))
mib = s.mid(8).trimmed().toInt();
if (s.startsWith("Alias:") && s.contains("MIME")) {
name = s.mid(6).trimmed();
name = name.left(name.indexOf(' ')).trimmed();
}
}
qDebug() << nameList;
}
*/
struct MibToName {
short mib;
short index;
};
static const MibToName mibToName[] = {
{ 3, 0 },
{ 4, 9 },
{ 5, 20 },
{ 6, 31 },
{ 7, 42 },
{ 8, 53 },
{ 9, 64 },
{ 10, 75 },
{ 11, 86 },
{ 12, 97 },
{ 13, 108 },
{ 16, 120 },
{ 17, 134 },
{ 18, 144 },
{ 30, 151 },
{ 36, 160 },
{ 37, 167 },
{ 38, 179 },
{ 39, 186 },
{ 40, 198 },
{ 57, 212 },
{ 81, 223 },
{ 82, 234 },
{ 84, 245 },
{ 85, 256 },
{ 104, 267 },
{ 105, 279 },
{ 106, 295 },
{ 109, 301 },
{ 110, 313 },
{ 111, 325 },
{ 113, 337 },
{ 114, 341 },
{ 1000, 349 },
{ 1001, 356 },
{ 1011, 363 },
{ 1012, 368 },
{ 1013, 374 },
{ 1014, 383 },
{ 1015, 392 },
{ 1016, 399 },
{ 1017, 406 },
{ 1018, 413 },
{ 1019, 422 },
{ 1020, 431 },
{ 2004, 438 },
{ 2005, 448 },
{ 2009, 472 },
{ 2013, 479 },
{ 2016, 486 },
{ 2024, 495 },
{ 2025, 505 },
{ 2026, 512 },
{ 2027, 517 },
{ 2028, 527 },
{ 2030, 534 },
{ 2033, 541 },
{ 2034, 548 },
{ 2035, 555 },
{ 2037, 562 },
{ 2038, 569 },
{ 2039, 576 },
{ 2040, 583 },
{ 2041, 590 },
{ 2043, 597 },
{ 2011, 604 },
{ 2044, 611 },
{ 2045, 618 },
{ 2010, 624 },
{ 2046, 631 },
{ 2047, 638 },
{ 2048, 645 },
{ 2049, 652 },
{ 2050, 659 },
{ 2051, 666 },
{ 2052, 673 },
{ 2053, 680 },
{ 2054, 687 },
{ 2055, 694 },
{ 2056, 701 },
{ 2062, 708 },
{ 2063, 715 },
{ 2084, 723 },
{ 2085, 730 },
{ 2086, 741 },
{ 2087, 748 },
{ 2088, 755 },
{ 2089, 762 },
{ 2091, 771 },
{ 2092, 780 },
{ 2093, 789 },
{ 2094, 798 },
{ 2095, 807 },
{ 2096, 816 },
{ 2097, 825 },
{ 2098, 834 },
{ 2099, 843 },
{ 2100, 852 },
{ 2101, 861 },
{ 2102, 872 },
{ 2250, 880 },
{ 2251, 893 },
{ 2252, 906 },
{ 2253, 919 },
{ 2254, 932 },
{ 2255, 945 },
{ 2256, 958 },
{ 2257, 971 },
{ 2258, 984 },
{ 2259, 997 },
};
int mibToNameSize = sizeof(mibToName)/sizeof(MibToName);
static const char mibToNameTable[] =
"US-ASCII\0"
"ISO-8859-1\0"
"ISO-8859-2\0"
"ISO-8859-3\0"
"ISO-8859-4\0"
"ISO-8859-5\0"
"ISO-8859-6\0"
"ISO-8859-7\0"
"ISO-8859-8\0"
"ISO-8859-9\0"
"ISO-8859-10\0"
"ISO-2022-JP-1\0"
"Shift_JIS\0"
"EUC-JP\0"
"US-ASCII\0"
"EUC-KR\0"
"ISO-2022-KR\0"
"EUC-KR\0"
"ISO-2022-JP\0"
"ISO-2022-JP-2\0"
"GB_2312-80\0"
"ISO-8859-6\0"
"ISO-8859-6\0"
"ISO-8859-8\0"
"ISO-8859-8\0"
"ISO-2022-CN\0"
"ISO-2022-CN-EXT\0"
"UTF-8\0"
"ISO-8859-13\0"
"ISO-8859-14\0"
"ISO-8859-15\0"
"GBK\0"
"GB18030\0"
"UTF-16\0"
"UTF-32\0"
"SCSU\0"
"UTF-7\0"
"UTF-16BE\0"
"UTF-16LE\0"
"UTF-16\0"
"CESU-8\0"
"UTF-32\0"
"UTF-32BE\0"
"UTF-32LE\0"
"BOCU-1\0"
"hp-roman8\0"
"Adobe-Standard-Encoding\0"
"IBM850\0"
"IBM862\0"
"IBM-Thai\0"
"Shift_JIS\0"
"GB2312\0"
"Big5\0"
"macintosh\0"
"IBM037\0"
"IBM273\0"
"IBM277\0"
"IBM278\0"
"IBM280\0"
"IBM284\0"
"IBM285\0"
"IBM290\0"
"IBM297\0"
"IBM420\0"
"IBM424\0"
"IBM437\0"
"IBM500\0"
"cp851\0"
"IBM852\0"
"IBM855\0"
"IBM857\0"
"IBM860\0"
"IBM861\0"
"IBM863\0"
"IBM864\0"
"IBM865\0"
"IBM868\0"
"IBM869\0"
"IBM870\0"
"IBM871\0"
"IBM918\0"
"IBM1026\0"
"KOI8-R\0"
"HZ-GB-2312\0"
"IBM866\0"
"IBM775\0"
"KOI8-U\0"
"IBM00858\0"
"IBM01140\0"
"IBM01141\0"
"IBM01142\0"
"IBM01143\0"
"IBM01144\0"
"IBM01145\0"
"IBM01146\0"
"IBM01147\0"
"IBM01148\0"
"IBM01149\0"
"Big5-HKSCS\0"
"IBM1047\0"
"windows-1250\0"
"windows-1251\0"
"windows-1252\0"
"windows-1253\0"
"windows-1254\0"
"windows-1255\0"
"windows-1256\0"
"windows-1257\0"
"windows-1258\0"
"TIS-620\0";
/// \threadsafe
QList<QByteArray> QIcuCodec::availableCodecs()
{
QList<QByteArray> codecs;
int n = ucnv_countAvailable();
for (int i = 0; i < n; ++i) {
const char *name = ucnv_getAvailableName(i);
UErrorCode error = U_ZERO_ERROR;
const char *standardName = ucnv_getStandardName(name, "MIME", &error);
if (U_FAILURE(error) || !standardName) {
error = U_ZERO_ERROR;
standardName = ucnv_getStandardName(name, "IANA", &error);
}
if (U_FAILURE(error))
continue;
error = U_ZERO_ERROR;
int ac = ucnv_countAliases(standardName, &error);
if (U_FAILURE(error))
continue;
for (int j = 0; j < ac; ++j) {
error = U_ZERO_ERROR;
const char *alias = ucnv_getAlias(standardName, j, &error);
if (U_FAILURE(error))
continue;
codecs += alias;
}
}
return codecs;
}
/// \threadsafe
QList<int> QIcuCodec::availableMibs()
{
QList<int> mibs;
mibs.reserve(mibToNameSize + 1);
for (int i = 0; i < mibToNameSize; ++i)
mibs += mibToName[i].mib;
return mibs;
}
QIcuCodec::QIcuCodec(const char *name)
: m_name(name)
{
}
QIcuCodec::~QIcuCodec()
{
}
UConverter *QIcuCodec::getConverter(QTextCodec::ConverterState *state) const
{
UConverter *conv = Q_NULLPTR;
if (state) {
if (!state->d) {
// first time
UErrorCode error = U_ZERO_ERROR;
state->d = ucnv_open(m_name, &error);
if (Q_UNLIKELY(U_FAILURE(error)))
qWarning("getConverter(state) ucnv_open failed %s %s", m_name, u_errorName(error));
if (state->d) {
error = U_ZERO_ERROR;
ucnv_setSubstChars(static_cast<UConverter *>(state->d),
state->flags & QTextCodec::ConvertInvalidToNull ? "\0" : "?", 1, &error);
if (Q_UNLIKELY(U_FAILURE(error)))
qWarning("getConverter(state) ucnv_setSubstChars failed %s %s", m_name, u_errorName(error));
conv = static_cast<UConverter *>(state->d);
}
}
}
if (!conv) {
// stateless conversion
UErrorCode error = U_ZERO_ERROR;
conv = ucnv_open(m_name, &error);
if (Q_UNLIKELY(U_FAILURE(error)))
qWarning("getConverter(no state) ucnv_open failed %s %s", m_name, u_errorName(error));
if (conv) {
error = U_ZERO_ERROR;
ucnv_setSubstChars(conv, "?", 1, &error);
if (Q_UNLIKELY(U_FAILURE(error)))
qWarning("getConverter(no state) ucnv_setSubstChars failed %s %s", m_name, u_errorName(error));
}
}
return conv;
}
QString QIcuCodec::convertToUnicode(const char *chars, int length, QTextCodec::ConverterState *state) const
{
UConverter *conv = getConverter(state);
QString string(length + 2, Qt::Uninitialized);
bool failed = false;
const char *end = chars + length;
int convertedChars = 0;
UErrorCode error = U_ZERO_ERROR;
while (1) {
UChar *uc = reinterpret_cast<UChar *>(string.data());
const UChar *ucEnd = uc + string.length();
uc += convertedChars;
ucnv_toUnicode(conv,
&uc, ucEnd,
&chars, end,
0, false, &error);
if (Q_UNLIKELY(U_FAILURE(error) && error != U_BUFFER_OVERFLOW_ERROR)) {
qWarning("convertToUnicode failed: %s", u_errorName(error));
failed = true;
break;
}
convertedChars = uc - reinterpret_cast<const UChar *>(string.constData());
if (chars >= end)
break;
string.resize(string.length()*2);
}
string.resize(convertedChars);
if (failed && state) {
error = U_ZERO_ERROR;
char errorbytes[10];
int8_t invalidlen = 0;
ucnv_getInvalidChars(conv,
errorbytes,
&invalidlen,
&error);
state->invalidChars = invalidlen;
}
if (!state)
ucnv_close(conv);
return string;
}
QByteArray QIcuCodec::convertFromUnicode(const QChar *unicode, int length, QTextCodec::ConverterState *state) const
{
UConverter *conv = getConverter(state);
int requiredLength = UCNV_GET_MAX_BYTES_FOR_STRING(length, ucnv_getMaxCharSize(conv));
QByteArray string(requiredLength, Qt::Uninitialized);
bool failed = false;
UErrorCode error = U_ZERO_ERROR;
const UChar *uc = reinterpret_cast<const UChar *>(unicode);
const UChar *end = uc + length;
int convertedChars = 0;
while (1) {
char *ch = (char *)string.data();
char *chEnd = ch + string.length();
ch += convertedChars;
ucnv_fromUnicode(conv,
&ch, chEnd,
&uc, end,
0, false, &error);
if (Q_UNLIKELY(U_FAILURE(error))) {
qWarning("convertFromUnicode failed: %s", u_errorName(error));
failed = true;
break;
}
convertedChars = ch - string.data();
if (uc >= end)
break;
string.resize(string.length()*2);
}
string.resize(convertedChars);
if (failed && state) {
error = U_ZERO_ERROR;
char errorbytes[10];
int8_t invalidlen = 0;
ucnv_getInvalidChars(conv,
errorbytes,
&invalidlen,
&error);
state->invalidChars = invalidlen;
}
if (!state)
ucnv_close(conv);
return string;
}
QByteArray QIcuCodec::name() const
{
return m_name;
}
QList<QByteArray> QIcuCodec::aliases() const
{
UErrorCode error = U_ZERO_ERROR;
int n = ucnv_countAliases(m_name, &error);
QList<QByteArray> aliases;
for (int i = 0; i < n; ++i) {
const char *a = ucnv_getAlias(m_name, i, &error);
// skip the canonical name
if (!a || !qstrcmp(a, m_name))
continue;
aliases += a;
}
return aliases;
}
int QIcuCodec::mibEnum() const
{
for (int i = 0; i < mibToNameSize; ++i) {
if (qTextCodecNameMatch(m_name, (mibToNameTable + mibToName[i].index)))
return mibToName[i].mib;
}
return 0;
}
QT_END_NAMESPACE

82
src/core/codecs/qicucodec_p.h Normal file
View file

@ -0,0 +1,82 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QICUCODEC_P_H
#define QICUCODEC_P_H
#include "QtCore/qtextcodec.h"
//
// W A R N I N G
// -------------
//
// This file is not part of the Qt API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include <qglobal.h>
#include <unicode/ucnv.h>
#ifndef QT_NO_TEXTCODEC
QT_BEGIN_NAMESPACE
class QIcuCodec : public QTextCodec
{
public:
QIcuCodec(const char *name);
~QIcuCodec();
static QList<QByteArray> availableCodecs();
static QList<int> availableMibs();
QString convertToUnicode(const char *, int, ConverterState *) const override;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const override;
QByteArray name() const override;
QList<QByteArray> aliases() const override;
int mibEnum() const override;
private:
UConverter *getConverter(QTextCodec::ConverterState *state) const;
const char *m_name;
};
QT_END_NAMESPACE
#endif // QT_NO_TEXTCODEC
#endif

277
src/core/codecs/qisciicodec.cpp
View file

@ -1,277 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qisciicodec_p.h"
#include "qlist.h"
#ifndef QT_NO_CODECS
QT_BEGIN_NAMESPACE
/*!
\class QIsciiCodec
\brief The QIsciiCodec class provides conversion to and from the ISCII encoding.
\internal
*/
struct Codecs {
const char name[10];
ushort base;
};
static const Codecs codecs [] = {
{ "Iscii-Dev", 0x900 },
{ "Iscii-Bng", 0x980 },
{ "Iscii-Pnj", 0xa00 },
{ "Iscii-Gjr", 0xa80 },
{ "Iscii-Ori", 0xb00 },
{ "Iscii-Tml", 0xb80 },
{ "Iscii-Tlg", 0xc00 },
{ "Iscii-Knd", 0xc80 },
{ "Iscii-Mlm", 0xd00 }
};
QIsciiCodec::~QIsciiCodec()
{
}
QByteArray QIsciiCodec::name() const
{
return codecs[idx].name;
}
int QIsciiCodec::mibEnum() const
{
/* There is no MIBEnum for Iscii */
return -3000-idx;
}
static const uchar inv = 0xFF;
/* iscii range from 0xa0 - 0xff */
static const uchar iscii_to_uni_table[0x60] = {
0x00, 0x01, 0x02, 0x03,
0x05, 0x06, 0x07, 0x08,
0x09, 0x0a, 0x0b, 0x0e,
0x0f, 0x20, 0x0d, 0x12,
0x13, 0x14, 0x11, 0x15,
0x16, 0x17, 0x18, 0x19,
0x1a, 0x1b, 0x1c, 0x1d,
0x1e, 0x1f, 0x20, 0x21,
0x22, 0x23, 0x24, 0x25,
0x26, 0x27, 0x28, 0x29,
0x2a, 0x2b, 0x2c, 0x2d,
0x2e, 0x2f, 0x5f, 0x30,
0x31, 0x32, 0x33, 0x34,
0x35, 0x36, 0x37, 0x38,
0x39, inv, 0x3e, 0x3f,
0x40, 0x41, 0x42, 0x43,
0x46, 0x47, 0x48, 0x45,
0x4a, 0x4b, 0x4c, 0x49,
0x4d, 0x3c, 0x64, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x66, 0x67, 0x68,
0x69, 0x6a, 0x6b, 0x6c,
0x6d, 0x6e, 0x6f, 0x00,
0x00, 0x00, 0x00, 0x00
};
static const uchar uni_to_iscii_table[0x80] = {
0x00, 0xa1, 0xa2, 0xa3,
0x00, 0xa4, 0xa5, 0xa6,
0xa7, 0xa8, 0xa9, 0xaa,
0x00, 0xae, 0xab, 0xac,
0xad, 0xb2, 0xaf, 0xb0,
0xb1, 0xb3, 0xb4, 0xb5,
0xb6, 0xb7, 0xb8, 0xb9,
0xba, 0xbb, 0xbc, 0xbd,
0xbe, 0xbf, 0xc0, 0xc1,
0xc2, 0xc3, 0xc4, 0xc5,
0xc6, 0xc7, 0xc8, 0xc9,
0xca, 0xcb, 0xcc, 0xcd,
0xcf, 0xd0, 0xd1, 0xd2,
0xd3, 0xd4, 0xd5, 0xd6,
0xd7, 0xd8, 0x00, 0x00,
0xe9, 0x00, 0xda, 0xdb,
0xdc, 0xdd, 0xde, 0xdf,
0x00, 0xe3, 0xe0, 0xe1,
0xe2, 0xe7, 0xe4, 0xe5,
0xe6, 0xe8, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x01, 0x02, 0x03, 0x04, // decomposable into the uc codes listed here + nukta
0x05, 0x06, 0x07, 0xce,
0x00, 0x00, 0x00, 0x00,
0xea, 0x08, 0xf1, 0xf2,
0xf3, 0xf4, 0xf5, 0xf6,
0xf7, 0xf8, 0xf9, 0xfa,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00
};
static const uchar uni_to_iscii_pairs[] = {
0x00, 0x00,
0x15, 0x3c, // 0x958
0x16, 0x3c, // 0x959
0x17, 0x3c, // 0x95a
0x1c, 0x3c, // 0x95b
0x21, 0x3c, // 0x95c
0x22, 0x3c, // 0x95d
0x2b, 0x3c, // 0x95e
0x64, 0x64 // 0x965
};
QByteArray QIsciiCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const
{
char replacement = '?';
bool halant = false;
if (state) {
if (state->flags & ConvertInvalidToNull)
replacement = 0;
halant = state->state_data[0];
}
int invalid = 0;
QByteArray result(2 * len, Qt::Uninitialized); //worst case
uchar *ch = reinterpret_cast<uchar *>(result.data());
const int base = codecs[idx].base;
for (int i =0; i < len; ++i) {
const ushort codePoint = uc[i].unicode();
/* The low 7 bits of ISCII is plain ASCII. However, we go all the
* way up to 0xA0 such that we can roundtrip with convertToUnicode()'s
* behavior. */
if(codePoint < 0xA0) {
*ch++ = static_cast<uchar>(codePoint);
continue;
}
const int pos = codePoint - base;
if (pos > 0 && pos < 0x80) {
uchar iscii = uni_to_iscii_table[pos];
if (iscii > 0x80) {
*ch++ = iscii;
} else if (iscii) {
const uchar *pair = uni_to_iscii_pairs + 2*iscii;
*ch++ = *pair++;
*ch++ = *pair++;
} else {
*ch++ = replacement;
++invalid;
}
} else {
if (uc[i].unicode() == 0x200c) { // ZWNJ
if (halant)
// Consonant Halant ZWNJ -> Consonant Halant Halant
*ch++ = 0xe8;
} else if (uc[i].unicode() == 0x200d) { // ZWJ
if (halant)
// Consonant Halant ZWJ -> Consonant Halant Nukta
*ch++ = 0xe9;
} else {
*ch++ = replacement;
++invalid;
}
}
halant = (pos == 0x4d);
}
result.truncate(ch - (uchar *)result.data());
if (state) {
state->invalidChars += invalid;
state->state_data[0] = halant;
}
return result;
}
QString QIsciiCodec::convertToUnicode(const char* chars, int len, ConverterState *state) const
{
bool halant = false;
if (state) {
halant = state->state_data[0];
}
QString result(len, Qt::Uninitialized);
QChar *uc = result.data();
const int base = codecs[idx].base;
for (int i = 0; i < len; ++i) {
ushort ch = (uchar) chars[i];
if (ch < 0xa0)
*uc++ = ch;
else {
ushort c = iscii_to_uni_table[ch - 0xa0];
if (halant && (c == inv || c == 0xe9)) {
// Consonant Halant inv -> Consonant Halant ZWJ
// Consonant Halant Nukta -> Consonant Halant ZWJ
*uc++ = QChar(0x200d);
} else if (halant && c == 0xe8) {
// Consonant Halant Halant -> Consonant Halant ZWNJ
*uc++ = QChar(0x200c);
} else {
*uc++ = QChar(c+base);
}
}
halant = ((uchar)chars[i] == 0xe8);
}
result.resize(uc - result.unicode());
if (state) {
state->state_data[0] = halant;
}
return result;
}
QT_END_NAMESPACE
#endif // QT_NO_CODECS

73
src/core/codecs/qisciicodec_p.h
View file

@ -1,73 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QISCIICODEC_P_H
#define QISCIICODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include "QtCore/qtextcodec.h"
QT_BEGIN_NAMESPACE
#ifndef QT_NO_CODECS
class QIsciiCodec : public QTextCodec {
public:
explicit QIsciiCodec(int i) : idx(i) {}
~QIsciiCodec();
QByteArray name() const;
int mibEnum() const;
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
private:
int idx;
};
#endif // QT_NO_CODECS
QT_END_NAMESPACE
#endif // QISCIICODEC_P_H

236
src/core/codecs/qlatincodec.cpp
View file

@ -1,236 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qlatincodec_p.h"
#include "qlist.h"
#ifndef QT_NO_TEXTCODEC
QT_BEGIN_NAMESPACE
QLatin1Codec::~QLatin1Codec()
{
}
QString QLatin1Codec::convertToUnicode(const char *chars, int len, ConverterState *) const
{
if (!chars)
return QString();
return QString::fromLatin1(chars, len);
}
QByteArray QLatin1Codec::convertFromUnicode(const QChar *ch, int len, ConverterState *state) const
{
const char replacement = (state && state->flags & ConvertInvalidToNull) ? 0 : '?';
QByteArray r(len, Qt::Uninitialized);
char *d = r.data();
int invalid = 0;
for (int i = 0; i < len; ++i) {
if (ch[i] > 0xff) {
d[i] = replacement;
++invalid;
} else {
d[i] = (char)ch[i].cell();
}
}
if (state) {
state->invalidChars += invalid;
}
return r;
}
QByteArray QLatin1Codec::name() const
{
return "ISO-8859-1";
}
QList<QByteArray> QLatin1Codec::aliases() const
{
QList<QByteArray> list;
list << "latin1"
<< "CP819"
<< "IBM819"
<< "iso-ir-100"
<< "csISOLatin1";
return list;
}
int QLatin1Codec::mibEnum() const
{
return 4;
}
QLatin15Codec::~QLatin15Codec()
{
}
QString QLatin15Codec::convertToUnicode(const char* chars, int len, ConverterState *) const
{
if (!chars)
return QString();
QString str = QString::fromLatin1(chars, len);
QChar *uc = str.data();
while(len--) {
switch(uc->unicode()) {
case 0xa4:
*uc = 0x20ac;
break;
case 0xa6:
*uc = 0x0160;
break;
case 0xa8:
*uc = 0x0161;
break;
case 0xb4:
*uc = 0x017d;
break;
case 0xb8:
*uc = 0x017e;
break;
case 0xbc:
*uc = 0x0152;
break;
case 0xbd:
*uc = 0x0153;
break;
case 0xbe:
*uc = 0x0178;
break;
default:
break;
}
uc++;
}
return str;
}
QByteArray QLatin15Codec::convertFromUnicode(const QChar *in, int length, ConverterState *state) const
{
const char replacement = (state && state->flags & ConvertInvalidToNull) ? 0 : '?';
QByteArray r(length, Qt::Uninitialized);
char *d = r.data();
int invalid = 0;
for (int i = 0; i < length; ++i) {
uchar c;
ushort uc = in[i].unicode();
if (uc < 0x0100) {
if (uc > 0xa3) {
switch(uc) {
case 0xa4:
case 0xa6:
case 0xa8:
case 0xb4:
case 0xb8:
case 0xbc:
case 0xbd:
case 0xbe:
c = replacement;
++invalid;
break;
default:
c = (unsigned char) uc;
break;
}
} else {
c = (unsigned char) uc;
}
} else {
if (uc == 0x20ac)
c = 0xa4;
else if ((uc & 0xff00) == 0x0100) {
switch(uc) {
case 0x0160:
c = 0xa6;
break;
case 0x0161:
c = 0xa8;
break;
case 0x017d:
c = 0xb4;
break;
case 0x017e:
c = 0xb8;
break;
case 0x0152:
c = 0xbc;
break;
case 0x0153:
c = 0xbd;
break;
case 0x0178:
c = 0xbe;
break;
default:
c = replacement;
++invalid;
}
} else {
c = replacement;
++invalid;
}
}
d[i] = (char)c;
}
if (state) {
state->remainingChars = 0;
state->invalidChars += invalid;
}
return r;
}
QByteArray QLatin15Codec::name() const
{
return "ISO-8859-15";
}
QList<QByteArray> QLatin15Codec::aliases() const
{
QList<QByteArray> list;
list << "latin9";
return list;
}
int QLatin15Codec::mibEnum() const
{
return 111;
}
QT_END_NAMESPACE
#endif // QT_NO_TEXTCODEC

84
src/core/codecs/qlatincodec_p.h
View file

@ -1,84 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QLATINCODEC_P_H
#define QLATINCODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include "QtCore/qtextcodec.h"
QT_BEGIN_NAMESPACE
#ifndef QT_NO_TEXTCODEC
class QLatin1Codec : public QTextCodec
{
public:
~QLatin1Codec();
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
};
class QLatin15Codec: public QTextCodec
{
public:
~QLatin15Codec();
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
};
#endif // QT_NO_TEXTCODEC
QT_END_NAMESPACE
#endif // QLATINCODEC_P_H

719
src/core/codecs/qsimplecodec.cpp
View file

@ -1,719 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qsimplecodec_p.h"
#include "qlist.h"
#ifndef QT_NO_TEXTCODEC
QT_BEGIN_NAMESPACE
#define LAST_MIB 2004
static const struct {
const char *mime;
const char *aliases[7];
int mib;
quint16 values[128];
} unicodevalues[QSimpleTextCodec::numSimpleCodecs] = {
// from RFC 1489, ftp://ftp.isi.edu/in-notes/rfc1489.txt
{ "KOI8-R", { "csKOI8R", Q_NULLPTR }, 2084,
{ 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524,
0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590,
0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219/**/, 0x221A, 0x2248,
0x2264, 0x2265, 0x00A0, 0x2321, 0x00B0, 0x00B2, 0x00B7, 0x00F7,
0x2550, 0x2551, 0x2552, 0x0451, 0x2553, 0x2554, 0x2555, 0x2556,
0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x255C, 0x255D, 0x255E,
0x255F, 0x2560, 0x2561, 0x0401, 0x2562, 0x2563, 0x2564, 0x2565,
0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x256B, 0x256C, 0x00A9,
0x044E, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433,
0x0445, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E,
0x043F, 0x044F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432,
0x044C, 0x044B, 0x0437, 0x0448, 0x044D, 0x0449, 0x0447, 0x044A,
0x042E, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413,
0x0425, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E,
0x041F, 0x042F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412,
0x042C, 0x042B, 0x0417, 0x0428, 0x042D, 0x0429, 0x0427, 0x042A } },
// /**/ - The BULLET OPERATOR is confused. Some people think
// it should be 0x2022 (BULLET).
// from RFC 2319, ftp://ftp.isi.edu/in-notes/rfc2319.txt
{ "KOI8-U", { "KOI8-RU", Q_NULLPTR }, 2088,
{ 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524,
0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590,
0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219, 0x221A, 0x2248,
0x2264, 0x2265, 0x00A0, 0x2321, 0x00B0, 0x00B2, 0x00B7, 0x00F7,
0x2550, 0x2551, 0x2552, 0x0451, 0x0454, 0x2554, 0x0456, 0x0457,
0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x0491, 0x255D, 0x255E,
0x255F, 0x2560, 0x2561, 0x0401, 0x0404, 0x2563, 0x0406, 0x0407,
0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x0490, 0x256C, 0x00A9,
0x044E, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433,
0x0445, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E,
0x043F, 0x044F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432,
0x044C, 0x044B, 0x0437, 0x0448, 0x044D, 0x0449, 0x0447, 0x044A,
0x042E, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413,
0x0425, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E,
0x041F, 0x042F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412,
0x042C, 0x042B, 0x0417, 0x0428, 0x042D, 0x0429, 0x0427, 0x042A } },
// next bits generated from tables on the Unicode 2.0 CD. we can
// use these tables since this is part of the transition to using
// unicode everywhere in qt.
// $ for A in 8 9 A B C D E F ; do for B in 0 1 2 3 4 5 6 7 8 9 A B C D E F ; do echo 0x${A}${B} 0xFFFD ; done ; done > /tmp/digits ; for a in 8859-* ; do (awk '/^0x[89ABCDEF]/{ print $1, $2 }' < $a ; cat /tmp/digits) | sort | uniq -w4 | cut -c6- | paste '-d ' - - - - - - - - | sed -e 's/ /, /g' -e 's/$/,/' -e '$ s/,$/} },/' -e '1 s/^/{ /' > ~/tmp/$a ; done
// then I inserted the files manually.
{ "ISO-8859-2", {"latin2", "iso-ir-101", "csISOLatin2", Q_NULLPTR }, 5,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x0104, 0x02D8, 0x0141, 0x00A4, 0x013D, 0x015A, 0x00A7,
0x00A8, 0x0160, 0x015E, 0x0164, 0x0179, 0x00AD, 0x017D, 0x017B,
0x00B0, 0x0105, 0x02DB, 0x0142, 0x00B4, 0x013E, 0x015B, 0x02C7,
0x00B8, 0x0161, 0x015F, 0x0165, 0x017A, 0x02DD, 0x017E, 0x017C,
0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7,
0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E,
0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7,
0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF,
0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7,
0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F,
0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7,
0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9} },
{ "ISO-8859-3", { "latin3", "iso-ir-109", "csISOLatin3", Q_NULLPTR }, 6,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x0126, 0x02D8, 0x00A3, 0x00A4, 0xFFFD, 0x0124, 0x00A7,
0x00A8, 0x0130, 0x015E, 0x011E, 0x0134, 0x00AD, 0xFFFD, 0x017B,
0x00B0, 0x0127, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x0125, 0x00B7,
0x00B8, 0x0131, 0x015F, 0x011F, 0x0135, 0x00BD, 0xFFFD, 0x017C,
0x00C0, 0x00C1, 0x00C2, 0xFFFD, 0x00C4, 0x010A, 0x0108, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0xFFFD, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x0120, 0x00D6, 0x00D7,
0x011C, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x016C, 0x015C, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0xFFFD, 0x00E4, 0x010B, 0x0109, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0xFFFD, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x0121, 0x00F6, 0x00F7,
0x011D, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x016D, 0x015D, 0x02D9} },
{ "ISO-8859-4", { "latin4", "iso-ir-110", "csISOLatin4", Q_NULLPTR }, 7,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x0104, 0x0138, 0x0156, 0x00A4, 0x0128, 0x013B, 0x00A7,
0x00A8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00AD, 0x017D, 0x00AF,
0x00B0, 0x0105, 0x02DB, 0x0157, 0x00B4, 0x0129, 0x013C, 0x02C7,
0x00B8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014A, 0x017E, 0x014B,
0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E,
0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x012A,
0x0110, 0x0145, 0x014C, 0x0136, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x0168, 0x016A, 0x00DF,
0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F,
0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x012B,
0x0111, 0x0146, 0x014D, 0x0137, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x0169, 0x016B, 0x02D9} },
{ "ISO-8859-5", { "cyrillic", "iso-ir-144", "csISOLatinCyrillic", Q_NULLPTR }, 8,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407,
0x0408, 0x0409, 0x040A, 0x040B, 0x040C, 0x00AD, 0x040E, 0x040F,
0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427,
0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437,
0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447,
0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457,
0x0458, 0x0459, 0x045A, 0x045B, 0x045C, 0x00A7, 0x045E, 0x045F} },
{ "ISO-8859-6", { "ISO-8859-6-I", "ECMA-114", "ASMO-708", "arabic", "iso-ir-127", "csISOLatinArabic", Q_NULLPTR }, 82,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A4, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x060C, 0x00AD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0x061B, 0xFFFD, 0xFFFD, 0xFFFD, 0x061F,
0xFFFD, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F,
0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637,
0x0638, 0x0639, 0x063A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647,
0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F,
0x0650, 0x0651, 0x0652, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD} },
{ "ISO-8859-7", { "ECMA-118", "greek", "iso-ir-126", "csISOLatinGreek", Q_NULLPTR }, 10,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x2018, 0x2019, 0x00A3, 0xFFFD, 0xFFFD, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0xFFFD, 0x2015,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x0385, 0x0386, 0x00B7,
0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F,
0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F,
0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7,
0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF,
0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7,
0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF,
0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7,
0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD} },
{ "ISO-8859-8", { "ISO 8859-8-I", "iso-ir-138", "hebrew", "csISOLatinHebrew", Q_NULLPTR }, 85,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x203E,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2017,
0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7,
0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7,
0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD} },
{ "ISO-8859-9", { "iso-ir-148", "latin5", "csISOLatin5", Q_NULLPTR }, 12,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, 0x015F, 0x00FF} },
{ "ISO-8859-10", { "iso-ir-157", "latin6", "ISO-8859-10:1992", "csISOLatin6", Q_NULLPTR }, 13,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x0104, 0x0112, 0x0122, 0x012A, 0x0128, 0x0136, 0x00A7,
0x013B, 0x0110, 0x0160, 0x0166, 0x017D, 0x00AD, 0x016A, 0x014A,
0x00B0, 0x0105, 0x0113, 0x0123, 0x012B, 0x0129, 0x0137, 0x00B7,
0x013C, 0x0111, 0x0161, 0x0167, 0x017E, 0x2015, 0x016B, 0x014B,
0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E,
0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x00CF,
0x00D0, 0x0145, 0x014C, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x0168,
0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F,
0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x00EF,
0x00F0, 0x0146, 0x014D, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x0169,
0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x0138} },
{ "ISO-8859-13", { Q_NULLPTR }, 109,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x201D, 0x00A2, 0x00A3, 0x00A4, 0x201E, 0x00A6, 0x00A7,
0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x201C, 0x00B5, 0x00B6, 0x00B7,
0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6,
0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112,
0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B,
0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7,
0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF,
0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113,
0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C,
0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7,
0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019} },
{ "ISO-8859-14", { "iso-ir-199", "latin8", "iso-celtic", Q_NULLPTR }, 110,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x1E02, 0x1E03, 0x00A3, 0x010A, 0x010B, 0x1E0A, 0x00A7,
0x1E80, 0x00A9, 0x1E82, 0x1E0B, 0x1EF2, 0x00AD, 0x00AE, 0x0178,
0x1E1E, 0x1E1F, 0x0120, 0x0121, 0x1E40, 0x1E41, 0x00B6, 0x1E56,
0x1E81, 0x1E57, 0x1E83, 0x1E60, 0x1EF3, 0x1E84, 0x1E85, 0x1E61,
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x0174, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x1E6A,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x0176, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x0175, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x1E6B,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x0177, 0x00FF} },
{ "ISO-8859-16", { "iso-ir-226", "latin10", Q_NULLPTR }, 112,
{ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7,
0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B,
0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7,
0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C,
0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A,
0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B,
0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF} },
// next bits generated again from tables on the Unicode 3.0 CD.
// $ for a in CP* ; do (awk '/^0x[89ABCDEF]/{ print $1, $2 }' < $a) | sort | sed -e 's/#UNDEF.*$/0xFFFD/' | cut -c6- | paste '-d ' - - - - - - - - | sed -e 's/ /, /g' -e 's/$/,/' -e '$ s/,$/} },/' -e '1 s/^/{ /' > ~/tmp/$a ; done
{ "IBM850", { "CP850", "csPC850Multilingual", Q_NULLPTR }, 2009,
{ 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7,
0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5,
0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9,
0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x0192,
0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA,
0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1, 0x00C2, 0x00C0,
0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510,
0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3,
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4,
0x00F0, 0x00D0, 0x00CA, 0x00CB, 0x00C8, 0x0131, 0x00CD, 0x00CE,
0x00CF, 0x2518, 0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580,
0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x00FE,
0x00DE, 0x00DA, 0x00DB, 0x00D9, 0x00FD, 0x00DD, 0x00AF, 0x00B4,
0x00AD, 0x00B1, 0x2017, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8,
0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2, 0x25A0, 0x00A0} },
{ "IBM874", { "CP874", Q_NULLPTR }, -874, //### what is the mib?
{ 0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0x00A0, 0x0E01, 0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07,
0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F,
0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, 0x0E16, 0x0E17,
0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F,
0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27,
0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F,
0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37,
0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F,
0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47,
0x0E48, 0x0E49, 0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F,
0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57,
0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD} },
{ "IBM866", { "CP866", "csIBM866", Q_NULLPTR }, 2086,
{ 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427,
0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437,
0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510,
0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567,
0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B,
0x256A, 0x2518, 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580,
0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447,
0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E,
0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4, 0x25A0, 0x00A0} },
{ "windows-1250", { "CP1250", Q_NULLPTR }, 2250,
{ 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021,
0xFFFD, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFD, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A,
0x00A0, 0x02C7, 0x02D8, 0x0141, 0x00A4, 0x0104, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B,
0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C,
0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7,
0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E,
0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7,
0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF,
0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7,
0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F,
0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7,
0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9} },
{ "windows-1251", { "CP1251", Q_NULLPTR }, 2251,
{ 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021,
0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F,
0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F,
0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7,
0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407,
0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7,
0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457,
0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427,
0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437,
0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447,
0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F} },
{ "windows-1252", { "CP1252", Q_NULLPTR }, 2252,
{ 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0x017D, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0x017E, 0x0178,
0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF} },
{ "windows-1253", {"CP1253", Q_NULLPTR }, 2253,
{ 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0x00A0, 0x0385, 0x0386, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5, 0x00B6, 0x00B7,
0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F,
0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F,
0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7,
0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF,
0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7,
0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF,
0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7,
0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD} },
{ "windows-1254", { "CP1254", Q_NULLPTR }, 2254,
{ 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178,
0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, 0x015F, 0x00FF} },
{ "windows-1255", { "CP1255", Q_NULLPTR }, 2255,
{ 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0x02C6, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x02DC, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x20AA, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7,
0x05B8, 0x05B9, 0xFFFD, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF,
0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3,
0x05F4, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7,
0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7,
0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD} },
{ "windows-1256", { "CP1256", Q_NULLPTR }, 2256,
{ 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688,
0x06AF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA,
0x00A0, 0x060C, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F,
0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F,
0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7,
0x0637, 0x0638, 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643,
0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0649, 0x064A, 0x00EE, 0x00EF,
0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7,
0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, 0x200F, 0x06D2} },
{ "windows-1257", { "CP1257", Q_NULLPTR }, 2257,
{ 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021,
0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0x00A8, 0x02C7, 0x00B8,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0x00AF, 0x02DB, 0xFFFD,
0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0xFFFD, 0x00A6, 0x00A7,
0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6,
0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112,
0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B,
0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7,
0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF,
0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113,
0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C,
0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7,
0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x02D9} },
{ "windows-1258", { "CP1258", Q_NULLPTR }, 2258,
{ 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0x02C6, 0x2030, 0xFFFD, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x02DC, 0x2122, 0xFFFD, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178,
0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF,
0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0301, 0x00ED, 0x00EE, 0x00EF,
0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, 0x20AB, 0x00FF} },
{ "Apple Roman", { "macintosh", "MacRoman", Q_NULLPTR }, -168,
{ 0x00C4, 0x00C5, 0x00C7, 0x00C9, 0x00D1, 0x00D6, 0x00DC, 0x00E1,
0x00E0, 0x00E2, 0x00E4, 0x00E3, 0x00E5, 0x00E7, 0x00E9, 0x00E8,
0x00EA, 0x00EB, 0x00ED, 0x00EC, 0x00EE, 0x00EF, 0x00F1, 0x00F3,
0x00F2, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x00F9, 0x00FB, 0x00FC,
0x2020, 0x00B0, 0x00A2, 0x00A3, 0x00A7, 0x2022, 0x00B6, 0x00DF,
0x00AE, 0x00A9, 0x2122, 0x00B4, 0x00A8, 0x2260, 0x00C6, 0x00D8,
0x221E, 0x00B1, 0x2264, 0x2265, 0x00A5, 0x00B5, 0x2202, 0x2211,
0x220F, 0x03C0, 0x222B, 0x00AA, 0x00BA, 0x03A9, 0x00E6, 0x00F8,
0x00BF, 0x00A1, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB,
0x00BB, 0x2026, 0x00A0, 0x00C0, 0x00C3, 0x00D5, 0x0152, 0x0153,
0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA,
0x00FF, 0x0178, 0x2044, 0x20AC, 0x2039, 0x203A, 0xFB01, 0xFB02,
0x2021, 0x00B7, 0x201A, 0x201E, 0x2030, 0x00C2, 0x00CA, 0x00C1,
0x00CB, 0x00C8, 0x00CD, 0x00CE, 0x00CF, 0x00CC, 0x00D3, 0x00D4,
0xF8FF, 0x00D2, 0x00DA, 0x00DB, 0x00D9, 0x0131, 0x02C6, 0x02DC,
0x00AF, 0x02D8, 0x02D9, 0x02DA, 0x00B8, 0x02DD, 0x02DB, 0x02C7} },
// This one is based on the charmap file
// /usr/share/i18n/charmaps/SAMI-WS2.gz, which is manually adapted
// to this format by Boerre Gaup <boerre@subdimension.com>
{ "WINSAMI2", { "WS2", Q_NULLPTR }, -165,
{ 0x20AC, 0xFFFD, 0x010C, 0x0192, 0x010D, 0x01B7, 0x0292, 0x01EE,
0x01EF, 0x0110, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x0111, 0x01E6, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178,
0x00A0, 0x01E7, 0x01E4, 0x00A3, 0x00A4, 0x01E5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x021E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x021F,
0x00B0, 0x00B1, 0x01E8, 0x01E9, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x014A, 0x014B, 0x0166, 0x00BB, 0x0167, 0x00BD, 0x017D, 0x017E,
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF} },
// this one is generated from the charmap file located in /usr/share/i18n/charmaps
// on most Linux distributions. The thai character set tis620 is byte by byte equivalent
// to iso8859-11, so we name it 8859-11 here, but recognise the name tis620 too.
// $ for A in 8 9 A B C D E F ; do for B in 0 1 2 3 4 5 6 7 8 9 A B C D E F ; do echo x${A}${B} 0xFFFD ; done ; done > /tmp/digits ; (cut -c25- < TIS-620 ; cat /tmp/digits) | awk '/^x[89ABCDEF]/{ print $1, $2 }' | sed -e 's/<U/0x/' -e 's/>//' | sort | uniq -w4 | cut -c5- | paste '-d ' - - - - - - - - | sed -e 's/ /, /g' -e 's/$/,/' -e '$ s/,$/} },/' -e '1 s/^/{ /' > ~/tmp/tis-620
{ "TIS-620", { "ISO 8859-11", Q_NULLPTR }, 2259, // Thai character set mib enum taken from tis620 (which is byte by byte equivalent)
{ 0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0x0E01, 0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07,
0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F,
0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, 0x0E16, 0x0E17,
0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F,
0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27,
0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F,
0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37,
0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F,
0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47,
0x0E48, 0x0E49, 0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F,
0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57,
0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD } },
/*
Name: hp-roman8 [HP-PCL5,RFC1345,KXS2]
MIBenum: 2004
Source: LaserJet IIP Printer User's Manual,
HP part no 33471-90901, Hewlet-Packard, June 1989.
Alias: roman8
Alias: r8
Alias: csHPRoman8
*/
{ "roman8", { "hp-roman8", "csHPRoman8", Q_NULLPTR }, 2004,
{ 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
0x00A0, 0x00C0, 0x00C2, 0x00C8, 0x00CA, 0x00CB, 0x00CE, 0x00CF,
0x00B4, 0x02CB, 0x02C6, 0x00A8, 0x02DC, 0x00D9, 0x00DB, 0x20A4,
0x00AF, 0x00DD, 0x00FD, 0x00B0, 0x00C7, 0x00E7, 0x00D1, 0x00F1,
0x00A1, 0x00BF, 0x00A4, 0x00A3, 0x00A5, 0x00A7, 0x0192, 0x00A2,
0x00E2, 0x00EA, 0x00F4, 0x00FB, 0x00E1, 0x00E9, 0x00F3, 0x00FA,
0x00E0, 0x00E8, 0x00F2, 0x00F9, 0x00E4, 0x00EB, 0x00F6, 0x00FC,
0x00C5, 0x00EE, 0x00D8, 0x00C6, 0x00E5, 0x00ED, 0x00F8, 0x00E6,
0x00C4, 0x00EC, 0x00D6, 0x00DC, 0x00C9, 0x00EF, 0x00DF, 0x00D4,
0x00C1, 0x00C3, 0x00E3, 0x00D0, 0x00F0, 0x00CD, 0x00CC, 0x00D3,
0x00D2, 0x00D5, 0x00F5, 0x0160, 0x0161, 0x00DA, 0x0178, 0x00FF,
0x00DE, 0x00FE, 0x00B7, 0x00B5, 0x00B6, 0x00BE, 0x2014, 0x00BC,
0x00BD, 0x00AA, 0x00BA, 0x00AB, 0x25A0, 0x00BB, 0x00B1, 0xFFFD } }
// if you add more chacater sets at the end, change LAST_MIB above
};
QSimpleTextCodec::QSimpleTextCodec(int i) : forwardIndex(i), reverseMap(Q_NULLPTR)
{
}
QSimpleTextCodec::~QSimpleTextCodec()
{
delete reverseMap;
}
static QByteArray *buildReverseMap(int forwardIndex)
{
QByteArray *map = new QByteArray();
int m = 0;
int i = 0;
while(i < 128) {
if (unicodevalues[forwardIndex].values[i] > m &&
unicodevalues[forwardIndex].values[i] < 0xfffd)
m = unicodevalues[forwardIndex].values[i];
i++;
}
m++;
map->resize(m);
for(i = 0; i < 128 && i < m; i++)
(*map)[i] = (char)i;
for(;i < m; i++)
(*map)[i] = 0;
for(i=128; i<256; i++) {
int u = unicodevalues[forwardIndex].values[i-128];
if (u < m)
(*map)[u] = (char)(unsigned char)(i);
}
return map;
}
QString QSimpleTextCodec::convertToUnicode(const char* chars, int len, ConverterState *) const
{
if (len <= 0 || !chars)
return QString();
const unsigned char * c = (const unsigned char *)chars;
QString r(len, Qt::Uninitialized);
QChar* uc = r.data();
for (int i = 0; i < len; i++) {
if (c[i] > 127)
uc[i] = unicodevalues[forwardIndex].values[c[i]-128];
else
uc[i] = QLatin1Char(c[i]);
}
return r;
}
QByteArray QSimpleTextCodec::convertFromUnicode(const QChar *in, int length, ConverterState *state) const
{
const char replacement = (state && state->flags & ConvertInvalidToNull) ? 0 : '?';
int invalid = 0;
if (!reverseMap){
QByteArray *tmp = buildReverseMap(this->forwardIndex);
if (!reverseMap.testAndSetOrdered(0, tmp))
delete tmp;
}
QByteArray r(length, Qt::Uninitialized);
int i = length;
int u;
const QChar* ucp = in;
unsigned char* rp = (unsigned char *)r.data();
const unsigned char* rmp = (const unsigned char *)reverseMap->constData();
int rmsize = (int) reverseMap->size();
while(i--)
{
u = ucp->unicode();
if (u < 128) {
*rp = (char)u;
} else {
*rp = ((u < rmsize) ? (*(rmp+u)) : 0);
if (*rp == 0) {
*rp = replacement;
++invalid;
}
}
rp++;
ucp++;
}
if (state) {
state->invalidChars += invalid;
}
return r;
}
QByteArray QSimpleTextCodec::name() const
{
return unicodevalues[forwardIndex].mime;
}
QList<QByteArray> QSimpleTextCodec::aliases() const
{
QList<QByteArray> list;
const char * const*a = unicodevalues[forwardIndex].aliases;
while (*a) {
list << *a;
++a;
}
return list;
}
int QSimpleTextCodec::mibEnum() const
{
return unicodevalues[forwardIndex].mib;
}
QT_END_NAMESPACE
#endif // QT_NO_TEXTCODEC

79
src/core/codecs/qsimplecodec_p.h
View file

@ -1,79 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QSIMPLECODEC_P_H
#define QSIMPLECODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include "QtCore/qtextcodec.h"
QT_BEGIN_NAMESPACE
#ifndef QT_NO_TEXTCODEC
template <typename T> class QAtomicPointer;
class QSimpleTextCodec: public QTextCodec
{
public:
enum { numSimpleCodecs = 30 };
explicit QSimpleTextCodec(int);
~QSimpleTextCodec();
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
private:
int forwardIndex;
mutable QAtomicPointer<QByteArray> reverseMap;
};
#endif // QT_NO_TEXTCODEC
QT_END_NAMESPACE
#endif // QSIMPLECODEC_P_H

374
src/core/codecs/qtextcodec.cpp
View file

@ -45,10 +45,7 @@
#include "qlocale_tools_p.h"
#include "qmutex.h"
#include "qhash.h"
#include "qiconvcodec_p.h"
#include "qutfcodec_p.h"
#include "qsimplecodec_p.h"
#include "qlatincodec_p.h"
#include "qicucodec_p.h"
#ifndef QT_NO_LIBRARY
# include "qcoreapplication.h"
@ -56,14 +53,6 @@
# include "qfactoryloader_p.h"
#endif
#ifndef QT_NO_CODECS
# include "qtsciicodec_p.h"
# include "qisciicodec_p.h"
# if defined(Q_WS_X11) && !defined(QT_BOOTSTRAPPED)
# include "qfontlaocodec_p.h"
# endif
#endif // QT_NO_CODECS
#include <stdlib.h>
#include <ctype.h>
#include <locale.h>
@ -133,7 +122,8 @@ static QTextCodec *createForName(const QByteArray &name)
#else
Q_UNUSED(name);
#endif
return 0;
return new QIcuCodec(name.constData());
}
static QTextCodec *createForMib(int mib)
@ -146,16 +136,16 @@ static QTextCodec *createForMib(int mib)
#else
Q_UNUSED(mib);
#endif
return 0;
return Q_NULLPTR;
}
static QList<QTextCodec*> *all = 0;
static QList<QTextCodec*> *all = Q_NULLPTR;
#ifdef Q_DEBUG_TEXTCODEC
static bool destroying_is_ok = false;
#endif
static QTextCodec *localeMapper = 0;
QTextCodec *QTextCodec::cftr = 0;
static QTextCodec *localeMapper = Q_NULLPTR;
QTextCodec *QTextCodec::cftr = Q_NULLPTR;
class QTextCodecCleanup
@ -179,13 +169,13 @@ QTextCodecCleanup::~QTextCodecCleanup()
#endif
QList<QTextCodec *> *myAll = all;
all = 0; // Otherwise the d'tor destroys the iterator
all = Q_NULLPTR; // Otherwise the d'tor destroys the iterator
for (QList<QTextCodec *>::const_iterator it = myAll->constBegin()
; it != myAll->constEnd(); ++it) {
delete *it;
}
delete myAll;
localeMapper = 0;
localeMapper = Q_NULLPTR;
#ifdef Q_DEBUG_TEXTCODEC
destroying_is_ok = false;
@ -194,107 +184,6 @@ QTextCodecCleanup::~QTextCodecCleanup()
Q_GLOBAL_STATIC(QTextCodecCleanup, createQTextCodecCleanup)
/* locale names mostly copied from XFree86 */
static const char * const iso8859_2locales[] = {
"croatian", "cs", "cs_CS", "cs_CZ","cz", "cz_CZ", "czech", "hr",
"hr_HR", "hu", "hu_HU", "hungarian", "pl", "pl_PL", "polish", "ro",
"ro_RO", "rumanian", "serbocroatian", "sh", "sh_SP", "sh_YU", "sk",
"sk_SK", "sl", "sl_CS", "sl_SI", "slovak", "slovene", "sr_SP", 0 };
static const char * const iso8859_3locales[] = {
"eo", 0 };
static const char * const iso8859_4locales[] = {
"ee", "ee_EE", 0 };
static const char * const iso8859_5locales[] = {
"mk", "mk_MK", "sp", "sp_YU", 0 };
static const char * const cp_1251locales[] = {
"be", "be_BY", "bg", "bg_BG", "bulgarian", 0 };
static const char * const pt_154locales[] = {
"ba_RU", "ky", "ky_KG", "kk", "kk_KZ", 0 };
static const char * const iso8859_6locales[] = {
"ar_AA", "ar_SA", "arabic", 0 };
static const char * const iso8859_7locales[] = {
"el", "el_GR", "greek", 0 };
static const char * const iso8859_8locales[] = {
"hebrew", "he", "he_IL", "iw", "iw_IL", 0 };
static const char * const iso8859_9locales[] = {
"tr", "tr_TR", "turkish", 0 };
static const char * const iso8859_13locales[] = {
"lt", "lt_LT", "lv", "lv_LV", 0 };
static const char * const iso8859_15locales[] = {
"et", "et_EE",
// Euro countries
"br_FR", "ca_ES", "de", "de_AT", "de_BE", "de_DE", "de_LU", "en_IE",
"es", "es_ES", "eu_ES", "fi", "fi_FI", "finnish", "fr", "fr_FR",
"fr_BE", "fr_LU", "french", "ga_IE", "gl_ES", "it", "it_IT", "oc_FR",
"nl", "nl_BE", "nl_NL", "pt", "pt_PT", "sv_FI", "wa_BE",
0 };
static const char * const koi8_ulocales[] = {
"uk", "uk_UA", "ru_UA", "ukrainian", 0 };
static const char * const tis_620locales[] = {
"th", "th_TH", "thai", 0 };
// static const char * const tcvnlocales[] = {
// "vi", "vi_VN", 0 };
static bool try_locale_list(const char * const locale[], const QByteArray &lang)
{
int i;
for(i=0; locale[i] && lang != locale[i]; i++)
;
return locale[i] != 0;
}
// For the probably_koi8_locales we have to look. the standard says
// these are 8859-5, but almost all Russian users use KOI8-R and
// incorrectly set $LANG to ru_RU. We'll check tolower() to see what
// it thinks ru_RU means.
// If you read the history, it seems that many Russians blame ISO and
// Perestroika for the confusion.
//
// The real bug is that some programs break if the user specifies
// ru_RU.KOI8-R.
static const char * const probably_koi8_rlocales[] = {
"ru", "ru_SU", "ru_RU", "russian", 0 };
static QTextCodec * ru_RU_hack(const char * i) {
QTextCodec * ru_RU_codec = 0;
const QByteArray origlocale(setlocale(LC_CTYPE, i));
// unicode koi8r latin5 name
// 0x044E 0xC0 0xEE CYRILLIC SMALL LETTER YU
// 0x042E 0xE0 0xCE CYRILLIC CAPITAL LETTER YU
int latin5 = tolower(0xCE);
int koi8r = tolower(0xE0);
if (koi8r == 0xC0 && latin5 != 0xEE) {
ru_RU_codec = QTextCodec::codecForName("KOI8-R");
} else if (koi8r != 0xC0 && latin5 == 0xEE) {
ru_RU_codec = QTextCodec::codecForName("ISO 8859-5");
} else {
// something else again... let's assume... *throws dice*
ru_RU_codec = QTextCodec::codecForName("KOI8-R");
qWarning("QTextCodec: Using KOI8-R, probe failed (%02x %02x %s)",
koi8r, latin5, i);
}
setlocale(LC_CTYPE, origlocale.constData());
return ru_RU_codec;
}
static QTextCodec *checkForCodec(const QByteArray &name) {
QTextCodec *c = QTextCodec::codecForName(name);
if (!c) {
@ -311,24 +200,9 @@ static QTextCodec *checkForCodec(const QByteArray &name) {
*/
static void setupLocaleMapper()
{
#ifndef QT_NO_ICONV
localeMapper = QTextCodec::codecForName("System");
#endif
#if defined (_XOPEN_UNIX) && !defined(Q_OS_OSF)
if (!localeMapper) {
char *charset = nl_langinfo (CODESET);
if (charset)
localeMapper = QTextCodec::codecForName(charset);
}
#endif
if (!localeMapper) {
// Very poorly defined and followed standards causes lots of
// code to try to get all the cases... This logic is
// duplicated in QIconvCodec, so if you change it here, change
// it there too.
// code to try to get all the cases...
// Try to determine locale codeset from locale name assigned to
// LC_CTYPE category.
@ -354,8 +228,6 @@ static void setupLocaleMapper()
// 3. ctype (maybe the locale is named "ISO-8859-1" or something)
// 4. locale (ditto)
// 5. check for "@euro"
// 6. guess locale from ctype unless ctype is "C"
// 7. guess locale from lang
// 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
int indexOfDot = ctype.indexOf('.');
@ -379,52 +251,22 @@ static void setupLocaleMapper()
// 5. "@euro"
if ((!localeMapper && ctype.contains("@euro")) || lang.contains("@euro"))
localeMapper = checkForCodec("ISO 8859-15");
// 6. guess locale from ctype unless ctype is "C"
// 7. guess locale from lang
const QByteArray &try_by_name = (!ctype.isEmpty() && ctype != "C") ? lang : ctype;
// Now do the guessing.
if (!lang.isEmpty() && !localeMapper && !try_by_name.isEmpty()) {
if (try_locale_list(iso8859_15locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-15");
else if (try_locale_list(iso8859_2locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-2");
else if (try_locale_list(iso8859_3locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-3");
else if (try_locale_list(iso8859_4locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-4");
else if (try_locale_list(iso8859_5locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-5");
else if (try_locale_list(iso8859_6locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-6");
else if (try_locale_list(iso8859_7locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-7");
else if (try_locale_list(iso8859_8locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-8-I");
else if (try_locale_list(iso8859_9locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-9");
else if (try_locale_list(iso8859_13locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-13");
else if (try_locale_list(tis_620locales, lang))
localeMapper = QTextCodec::codecForName("ISO 8859-11");
else if (try_locale_list(koi8_ulocales, lang))
localeMapper = QTextCodec::codecForName("KOI8-U");
else if (try_locale_list(cp_1251locales, lang))
localeMapper = QTextCodec::codecForName("CP 1251");
else if (try_locale_list(pt_154locales, lang))
localeMapper = QTextCodec::codecForName("PT 154");
else if (try_locale_list(probably_koi8_rlocales, lang))
localeMapper = ru_RU_hack(lang.constData());
}
localeMapper = checkForCodec("ISO-8859-15");
}
#if defined (_XOPEN_UNIX) && !defined(Q_OS_OSF)
if (!localeMapper) {
char *charset = nl_langinfo (CODESET);
if (charset)
localeMapper = QTextCodec::codecForName(charset);
}
#endif
// If everything failed, we default to 8859-1
// We could perhaps default to 8859-15.
if (!localeMapper)
localeMapper = QTextCodec::codecForName("ISO 8859-1");
localeMapper = QTextCodec::codecForName("ISO-8859-1");
}
#ifndef QT_NO_THREAD
@ -446,34 +288,12 @@ static void setup()
// create the cleanup object to cleanup all codecs on exit
(void) createQTextCodecCleanup();
#ifndef QT_NO_CODECS
(void)new QTsciiCodec;
for (int i = 0; i < 9; ++i)
(void)new QIsciiCodec(i);
for (int i = 0; i < QSimpleTextCodec::numSimpleCodecs; ++i)
(void)new QSimpleTextCodec(i);
# if defined(Q_WS_X11) && !defined(QT_BOOTSTRAPPED)
// no font codecs when bootstrapping
(void)new QFontLaoCodec;
# endif // Q_WS_X11
#endif // QT_NO_CODECS
(void)new QUtf16Codec;
(void)new QUtf16BECodec;
(void)new QUtf16LECodec;
(void)new QUtf32Codec;
(void)new QUtf32BECodec;
(void)new QUtf32LECodec;
(void)new QLatin15Codec;
(void)new QLatin1Codec;
(void)new QUtf8Codec;
#if !defined(QT_NO_ICONV) && !defined(QT_BOOTSTRAPPED)
// QIconvCodec depends on the UTF-16 codec, so it needs to be created last
(void) new QIconvCodec();
#if !defined(QT_NO_TEXTCODEC)
// codecs have to be explicitly created for use in QString
(void) new QIcuCodec("US-ASCII");
(void) new QIcuCodec("UTF-8");
(void) new QIcuCodec("UTF-16");
(void) new QIcuCodec("UTF-32");
#endif
if (!localeMapper)
@ -487,8 +307,6 @@ static void setup()
\value ConvertInvalidToNull If this flag is set, each invalid input
character is output as a null character.
\value IgnoreHeader Ignore any Unicode byte-order mark and don't generate any.
\omitvalue FreeFunction
*/
/*!
@ -502,8 +320,8 @@ static void setup()
*/
QTextCodec::ConverterState::~ConverterState()
{
if (flags & FreeFunction)
(QTextCodecUnalignedPointer::decode(state_data))(this);
if (d)
ucnv_close(static_cast<UConverter *>(d));
}
/*!
@ -524,31 +342,108 @@ QTextCodec::ConverterState::~ConverterState()
The supported encodings are:
\list
\o Apple Roman
\o IBM 850
\o IBM 866
\o IBM 874
\o ISO 8859-1 to 10
\o ISO 8859-13 to 16
\o Iscii-Bng, Dev, Gjr, Knd, Mlm, Ori, Pnj, Tlg, and Tml
\o Adobe-Standard-Encoding
\o Big5
\o Big5-HKSCS
\o BOCU-1
\o CESU-8
\o cp851
\o EUC-JP
\o EUC-KR
\o GB18030
\o GB2312
\o GB_2312-80
\o GBK
\o hp-roman8
\o HZ-GB-2312
\o IBM00858
\o IBM01140
\o IBM01141
\o IBM01142
\o IBM01143
\o IBM01144
\o IBM01145
\o IBM01146
\o IBM01147
\o IBM01148
\o IBM01149
\o IBM037
\o IBM1026
\o IBM1047
\o IBM273
\o IBM277
\o IBM278
\o IBM280
\o IBM284
\o IBM285
\o IBM290
\o IBM297
\o IBM420
\o IBM424
\o IBM437
\o IBM500
\o IBM775
\o IBM850
\o IBM852
\o IBM855
\o IBM857
\o IBM860
\o IBM861
\o IBM862
\o IBM863
\o IBM864
\o IBM865
\o IBM866
\o IBM868
\o IBM869
\o IBM870
\o IBM871
\o IBM918
\o IBM-Thai
\o ISO-2022-CN
\o ISO-2022-CN-EXT
\o ISO-2022-JP
\o ISO-2022-JP-1
\o ISO-2022-JP-2
\o ISO-2022-KR
\o ISO-8859-1
\o ISO-8859-10
\o ISO-8859-13
\o ISO-8859-14
\o ISO-8859-15
\o ISO-8859-2
\o ISO-8859-3
\o ISO-8859-4
\o ISO-8859-5
\o ISO-8859-6
\o ISO-8859-7
\o ISO-8859-8
\o ISO-8859-9
\o KOI8-R
\o KOI8-U
\o MuleLao-1
\o ROMAN8
\o macintosh
\o SCSU
\o Shift_JIS
\o TIS-620
\o \l{TSCII Text Codec}{TSCII}
\o UTF-8
\o US-ASCII
\o UTF-16
\o UTF-16BE
\o UTF-16LE
\o UTF-32
\o UTF-32BE
\o UTF-32LE
\o Windows-1250 to 1258
\o WINSAMI2
\o UTF-7
\o UTF-8
\o windows-1250
\o windows-1251
\o windows-1252
\o windows-1253
\o windows-1254
\o windows-1255
\o windows-1256
\o windows-1257
\o windows-1258
\endlist
QTextCodecs can be used as follows to convert some locally encoded
@ -686,7 +581,12 @@ QTextCodec::~QTextCodec()
QTextCodec *QTextCodec::codecForName(const QByteArray &name)
{
if (name.isEmpty())
return 0;
return Q_NULLPTR;
if (name == "System")
return QTextCodec::codecForLocale();
else if (name == "Latin1")
return QTextCodec::codecForName("US-ASCII");
#ifndef QT_NO_THREAD
QMutexLocker locker(textCodecsMutex());
@ -778,6 +678,7 @@ QList<QByteArray> QTextCodec::availableCodecs()
setup();
QList<QByteArray> codecs;
codecs << "System" << "Latin1";
for (int i = 0; i < all->size(); ++i) {
codecs += all->at(i)->name();
codecs += all->at(i)->aliases();
@ -860,11 +761,6 @@ void QTextCodec::setCodecForLocale(QTextCodec *c)
/*!
Returns a pointer to the codec most suitable for this locale.
On Windows, the codec will be based on a system locale. On Unix
systems, starting with Qt 4.2, the codec will be using the \e
iconv library. Note that in both cases the codec's name will be
"System".
*/
QTextCodec* QTextCodec::codecForLocale()
@ -875,7 +771,7 @@ QTextCodec* QTextCodec::codecForLocale()
#ifndef QT_NO_THREAD
QMutexLocker locker(textCodecsMutex());
#endif
setup();
setupLocaleMapper();
return localeMapper;
}
@ -1181,19 +1077,7 @@ QString QTextDecoder::toUnicode(const char *chars, int len)
void QTextDecoder::toUnicode(QString *target, const char *chars, int len)
{
Q_ASSERT(target);
switch (c->mibEnum()) {
case 106: // utf8
static_cast<const QUtf8Codec*>(c)->convertToUnicode(target, chars, len, &state);
break;
case 4: { // latin1
target->resize(len);
ushort *data = (ushort*)target->data();
for (int i = len; i >=0; --i)
data[i] = (uchar) chars[i];
} break;
default:
*target = c->toUnicode(chars, len, &state);
}
*target = c->toUnicode(chars, len, &state);
}

11
src/core/codecs/qtextcodec.h
View file

@ -86,19 +86,20 @@ public:
enum ConversionFlag {
DefaultConversion,
ConvertInvalidToNull = 0x80000000,
IgnoreHeader = 0x1,
FreeFunction = 0x2
IgnoreHeader = 0x1
};
Q_DECLARE_FLAGS(ConversionFlags, ConversionFlag)
struct Q_CORE_EXPORT ConverterState {
ConverterState(ConversionFlags f = DefaultConversion)
: flags(f), remainingChars(0), invalidChars(0) { state_data[0] = state_data[1] = state_data[2] = 0; }
: flags(f), remainingChars(0), invalidChars(0), d(Q_NULLPTR)
{
}
~ConverterState();
ConversionFlags flags;
int remainingChars;
int invalidChars;
uint state_data[3];
void *d;
private:
Q_DISABLE_COPY(ConverterState)
};
@ -122,8 +123,6 @@ protected:
QTextCodec();
virtual ~QTextCodec();
public:
private:
friend class QTextCodecCleanup;
static QTextCodec *cftr;

28
src/core/codecs/qtextcodec_p.h
View file

@ -50,26 +50,7 @@
QT_BEGIN_NAMESPACE
#ifndef QT_NO_TEXTCODEC
typedef void (*QTextCodecStateFreeFunction)(QTextCodec::ConverterState*);
struct QTextCodecUnalignedPointer
{
static inline QTextCodecStateFreeFunction decode(const uint *src)
{
quintptr data;
memcpy(&data, src, sizeof(data));
return reinterpret_cast<QTextCodecStateFreeFunction>(data);
}
static inline void encode(uint *dst, QTextCodecStateFreeFunction fn)
{
quintptr data = reinterpret_cast<quintptr>(fn);
memcpy(dst, &data, sizeof(data));
}
};
#else
#ifdef QT_NO_TEXTCODEC
class QTextCodec
{
@ -77,19 +58,18 @@ public:
enum ConversionFlag {
DefaultConversion,
ConvertInvalidToNull = 0x80000000,
IgnoreHeader = 0x1,
FreeFunction = 0x2
IgnoreHeader = 0x1
};
Q_DECLARE_FLAGS(ConversionFlags, ConversionFlag)
struct ConverterState {
ConverterState(ConversionFlags f = DefaultConversion)
: flags(f), remainingChars(0), invalidChars(0), { state_data[0] = state_data[1] = state_data[2] = 0; }
: flags(f), remainingChars(0), invalidChars(0), d(Q_NULLPTR) { }
~ConverterState() { }
ConversionFlags flags;
int remainingChars;
int invalidChars;
uint state_data[3];
void *d;
private:
Q_DISABLE_COPY(ConverterState)
};

490
src/core/codecs/qtsciicodec.cpp
View file

@ -1,490 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
// Most of the code here was originally written by Hans Petter Bieker,
// and is included in Qt with the author's permission, and the grateful
// thanks of the Qt team.
#include "qtsciicodec_p.h"
#include "qlist.h"
#ifndef QT_NO_CODECS
QT_BEGIN_NAMESPACE
static unsigned char qt_UnicodeToTSCII(ushort u1, ushort u2, ushort u3);
static unsigned int qt_TSCIIToUnicode(unsigned int code, uint *s);
#define IsTSCIIChar(c) (((c) >= 0x80) && ((c) <= 0xfd))
/*! \class QTsciiCodec
\reentrant
\internal
*/
/*!
Destroys the text codec object.
*/
QTsciiCodec::~QTsciiCodec()
{
}
/*!
Converts the first \a len characters in \a uc from Unicode to this
encoding, and returns the result in a byte array. The \a state contains
some conversion flags, and is used by the codec to maintain state
information.
*/
QByteArray QTsciiCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const
{
char replacement = '?';
if (state) {
if (state->flags & ConvertInvalidToNull)
replacement = 0;
}
int invalid = 0;
QByteArray rstr(len, Qt::Uninitialized);
uchar* cursor = (uchar*)rstr.data();
for (int i = 0; i < len; i++) {
QChar ch = uc[i];
uchar j;
if (ch.row() == 0x00 && ch.cell() < 0x80) {
// ASCII
j = ch.cell();
} else if ((j = qt_UnicodeToTSCII(uc[i].unicode(),
uc[i + 1].unicode(),
uc[i + 2].unicode()))) {
// We have to check the combined chars first!
i += 2;
} else if ((j = qt_UnicodeToTSCII(uc[i].unicode(),
uc[i + 1].unicode(), 0))) {
i++;
} else if ((j = qt_UnicodeToTSCII(uc[i].unicode(), 0, 0))) {
} else {
// Error
j = replacement;
++invalid;
}
*cursor++ = j;
}
rstr.resize(cursor - (const uchar*)rstr.constData());
if (state) {
state->invalidChars += invalid;
}
return rstr;
}
/*!
Converts the first \a len characters in \a chars from this encoding
to Unicode, and returns the result in a QString. The \a state contains
some conversion flags, and is used by the codec to maintain state
information.
*/
QString QTsciiCodec::convertToUnicode(const char* chars, int len, ConverterState *state) const
{
QChar replacement = QChar::ReplacementCharacter;
if (state) {
if (state->flags & ConvertInvalidToNull)
replacement = QChar::Null;
}
int invalid = 0;
QString result;
for (int i = 0; i < len; i++) {
uchar ch = chars[i];
if (ch < 0x80) {
// ASCII
result += QLatin1Char(ch);
} else if (IsTSCIIChar(ch)) {
// TSCII
uint s[3];
uint u = qt_TSCIIToUnicode(ch, s);
uint *p = s;
while (u--) {
uint c = *p++;
if (c)
result += QChar(c);
else {
result += replacement;
++invalid;
}
}
} else {
// Invalid
result += replacement;
++invalid;
}
}
if (state) {
state->invalidChars += invalid;
}
return result;
}
/*!
Returns the official name for the encoding that is handled by the codec.
\sa QTextCodec::name()
*/
QByteArray QTsciiCodec::name() const
{
return "TSCII";
}
/*!
Returns the MIB enum for the encoding.
\sa QTextCodec::mibEnum()
*/
int QTsciiCodec::mibEnum() const
{
return 2107;
}
static const int UnToTsLast = 124; // 125 items -- so the last will be 124
static const ushort UnToTs [][4] = {
// *Sorted* list of TSCII maping for unicode chars
//FIRST SECOND THIRD TSCII
{0x00A0, 0x0000, 0x0000, 0xA0},
{0x00A9, 0x0000, 0x0000, 0xA9},
{0x0B83, 0x0000, 0x0000, 0xB7},
{0x0B85, 0x0000, 0x0000, 0xAB},
{0x0B86, 0x0000, 0x0000, 0xAC},
{0x0B87, 0x0000, 0x0000, 0xAD},
{0x0B88, 0x0000, 0x0000, 0xAE},
{0x0B89, 0x0000, 0x0000, 0xAF},
{0x0B8A, 0x0000, 0x0000, 0xB0},
{0x0B8E, 0x0000, 0x0000, 0xB1},
{0x0B8F, 0x0000, 0x0000, 0xB2},
{0x0B90, 0x0000, 0x0000, 0xB3},
{0x0B92, 0x0000, 0x0000, 0xB4},
{0x0B93, 0x0000, 0x0000, 0xB5},
{0x0B94, 0x0000, 0x0000, 0xB6},
{0x0B95, 0x0000, 0x0000, 0xB8},
{0x0B95, 0x0B82, 0x0000, 0xEC},
{0x0B95, 0x0BC1, 0x0000, 0xCC},
{0x0B95, 0x0BC2, 0x0000, 0xDC},
{0x0B99, 0x0000, 0x0000, 0xB9},
{0x0B99, 0x0B82, 0x0000, 0xED},
{0x0B99, 0x0BC1, 0x0000, 0x99},
{0x0B99, 0x0BC2, 0x0000, 0x9B},
{0x0B9A, 0x0000, 0x0000, 0xBA},
{0x0B9A, 0x0B82, 0x0000, 0xEE},
{0x0B9A, 0x0BC1, 0x0000, 0xCD},
{0x0B9A, 0x0BC2, 0x0000, 0xDD},
{0x0B9C, 0x0000, 0x0000, 0x83},
{0x0B9C, 0x0B82, 0x0000, 0x88},
{0x0B9E, 0x0000, 0x0000, 0xBB},
{0x0B9E, 0x0B82, 0x0000, 0xEF},
{0x0B9E, 0x0BC1, 0x0000, 0x9A},
{0x0B9E, 0x0BC2, 0x0000, 0x9C},
{0x0B9F, 0x0000, 0x0000, 0xBC},
{0x0B9F, 0x0B82, 0x0000, 0xF0},
{0x0B9F, 0x0BBF, 0x0000, 0xCA},
{0x0B9F, 0x0BC0, 0x0000, 0xCB},
{0x0B9F, 0x0BC1, 0x0000, 0xCE},
{0x0B9F, 0x0BC2, 0x0000, 0xDE},
{0x0BA1, 0x0B82, 0x0000, 0xF2},
{0x0BA3, 0x0000, 0x0000, 0xBD},
{0x0BA3, 0x0B82, 0x0000, 0xF1},
{0x0BA3, 0x0BC1, 0x0000, 0xCF},
{0x0BA3, 0x0BC2, 0x0000, 0xDF},
{0x0BA4, 0x0000, 0x0000, 0xBE},
{0x0BA4, 0x0BC1, 0x0000, 0xD0},
{0x0BA4, 0x0BC2, 0x0000, 0xE0},
{0x0BA8, 0x0000, 0x0000, 0xBF},
{0x0BA8, 0x0B82, 0x0000, 0xF3},
{0x0BA8, 0x0BC1, 0x0000, 0xD1},
{0x0BA8, 0x0BC2, 0x0000, 0xE1},
{0x0BA9, 0x0000, 0x0000, 0xC9},
{0x0BA9, 0x0B82, 0x0000, 0xFD},
{0x0BA9, 0x0BC1, 0x0000, 0xDB},
{0x0BA9, 0x0BC2, 0x0000, 0xEB},
{0x0BAA, 0x0000, 0x0000, 0xC0},
{0x0BAA, 0x0B82, 0x0000, 0xF4},
{0x0BAA, 0x0BC1, 0x0000, 0xD2},
{0x0BAA, 0x0BC2, 0x0000, 0xE2},
{0x0BAE, 0x0000, 0x0000, 0xC1},
{0x0BAE, 0x0B82, 0x0000, 0xF5},
{0x0BAE, 0x0BC1, 0x0000, 0xD3},
{0x0BAE, 0x0BC2, 0x0000, 0xE3},
{0x0BAF, 0x0000, 0x0000, 0xC2},
{0x0BAF, 0x0B82, 0x0000, 0xF6},
{0x0BAF, 0x0BC1, 0x0000, 0xD4},
{0x0BAF, 0x0BC2, 0x0000, 0xE4},
{0x0BB0, 0x0000, 0x0000, 0xC3},
{0x0BB0, 0x0B82, 0x0000, 0xF7},
{0x0BB0, 0x0BC1, 0x0000, 0xD5},
{0x0BB0, 0x0BC2, 0x0000, 0xE5},
{0x0BB1, 0x0000, 0x0000, 0xC8},
{0x0BB1, 0x0B82, 0x0000, 0xFC},
{0x0BB1, 0x0BC1, 0x0000, 0xDA},
{0x0BB1, 0x0BC2, 0x0000, 0xEA},
{0x0BB2, 0x0000, 0x0000, 0xC4},
{0x0BB2, 0x0B82, 0x0000, 0xF8},
{0x0BB2, 0x0BC1, 0x0000, 0xD6},
{0x0BB2, 0x0BC2, 0x0000, 0xE6},
{0x0BB3, 0x0000, 0x0000, 0xC7},
{0x0BB3, 0x0B82, 0x0000, 0xFB},
{0x0BB3, 0x0BC1, 0x0000, 0xD9},
{0x0BB3, 0x0BC2, 0x0000, 0xE9},
{0x0BB4, 0x0000, 0x0000, 0xC6},
{0x0BB4, 0x0B82, 0x0000, 0xFA},
{0x0BB4, 0x0BC1, 0x0000, 0xD8},
{0x0BB4, 0x0BC2, 0x0000, 0xE8},
{0x0BB5, 0x0000, 0x0000, 0xC5},
{0x0BB5, 0x0B82, 0x0000, 0xF9},
{0x0BB5, 0x0BC1, 0x0000, 0xD7},
{0x0BB5, 0x0BC2, 0x0000, 0xE7},
{0x0BB7, 0x0000, 0x0000, 0x84},
{0x0BB7, 0x0B82, 0x0000, 0x89},
{0x0BB8, 0x0000, 0x0000, 0x85},
{0x0BB8, 0x0B82, 0x0000, 0x8A},
{0x0BB9, 0x0000, 0x0000, 0x86},
{0x0BB9, 0x0B82, 0x0000, 0x8B},
{0x0BBE, 0x0000, 0x0000, 0xA1},
{0x0BBF, 0x0000, 0x0000, 0xA2},
{0x0BC0, 0x0000, 0x0000, 0xA3},
{0x0BC1, 0x0000, 0x0000, 0xA4},
{0x0BC2, 0x0000, 0x0000, 0xA5},
{0x0BC6, 0x0000, 0x0000, 0xA6},
{0x0BC7, 0x0000, 0x0000, 0xA7},
{0x0BC8, 0x0000, 0x0000, 0xA8},
{0x0BCC, 0x0000, 0x0000, 0xAA},
{0x0BE6, 0x0000, 0x0000, 0x80},
{0x0BE7, 0x0000, 0x0000, 0x81},
{0x0BE7, 0x0BB7, 0x0000, 0x87},
{0x0BE7, 0x0BB7, 0x0B82, 0x8C},
{0x0BE8, 0x0000, 0x0000, 0x8D},
{0x0BE9, 0x0000, 0x0000, 0x8E},
{0x0BEA, 0x0000, 0x0000, 0x8F},
{0x0BEB, 0x0000, 0x0000, 0x90},
{0x0BEC, 0x0000, 0x0000, 0x95},
{0x0BED, 0x0000, 0x0000, 0x96},
{0x0BEE, 0x0000, 0x0000, 0x97},
{0x0BEF, 0x0000, 0x0000, 0x98},
{0x0BF0, 0x0000, 0x0000, 0x9D},
{0x0BF1, 0x0000, 0x0000, 0x9E},
{0x0BF2, 0x0000, 0x0000, 0x9F},
{0x2018, 0x0000, 0x0000, 0x91},
{0x2019, 0x0000, 0x0000, 0x92},
{0x201C, 0x0000, 0x0000, 0x93},
{0x201C, 0x0000, 0x0000, 0x94}
};
static const ushort TsToUn [][3] = {
// Starting at 0x80
{0x0BE6, 0x0000, 0x0000},
{0x0BE7, 0x0000, 0x0000},
{0x0000, 0x0000, 0x0000}, // unknown
{0x0B9C, 0x0000, 0x0000},
{0x0BB7, 0x0000, 0x0000},
{0x0BB8, 0x0000, 0x0000},
{0x0BB9, 0x0000, 0x0000},
{0x0BE7, 0x0BB7, 0x0000},
{0x0B9C, 0x0B82, 0x0000},
{0x0BB7, 0x0B82, 0x0000},
{0x0BB8, 0x0B82, 0x0000},
{0x0BB9, 0x0B82, 0x0000},
{0x0BE7, 0x0BB7, 0x0B82},
{0x0BE8, 0x0000, 0x0000},
{0x0BE9, 0x0000, 0x0000},
{0x0BEA, 0x0000, 0x0000},
{0x0BEB, 0x0000, 0x0000},
{0x2018, 0x0000, 0x0000},
{0x2019, 0x0000, 0x0000},
{0x201C, 0x0000, 0x0000},
{0x201C, 0x0000, 0x0000}, // two of the same??
{0x0BEC, 0x0000, 0x0000},
{0x0BED, 0x0000, 0x0000},
{0x0BEE, 0x0000, 0x0000},
{0x0BEF, 0x0000, 0x0000},
{0x0B99, 0x0BC1, 0x0000},
{0x0B9E, 0x0BC1, 0x0000},
{0x0B99, 0x0BC2, 0x0000},
{0x0B9E, 0x0BC2, 0x0000},
{0x0BF0, 0x0000, 0x0000},
{0x0BF1, 0x0000, 0x0000},
{0x0BF2, 0x0000, 0x0000},
{0x00A0, 0x0000, 0x0000},
{0x0BBE, 0x0000, 0x0000},
{0x0BBF, 0x0000, 0x0000},
{0x0BC0, 0x0000, 0x0000},
{0x0BC1, 0x0000, 0x0000},
{0x0BC2, 0x0000, 0x0000},
{0x0BC6, 0x0000, 0x0000},
{0x0BC7, 0x0000, 0x0000},
{0x0BC8, 0x0000, 0x0000},
{0x00A9, 0x0000, 0x0000},
{0x0BCC, 0x0000, 0x0000},
{0x0B85, 0x0000, 0x0000},
{0x0B86, 0x0000, 0x0000},
{0x0B87, 0x0000, 0x0000},
{0x0B88, 0x0000, 0x0000},
{0x0B89, 0x0000, 0x0000},
{0x0B8A, 0x0000, 0x0000},
{0x0B8E, 0x0000, 0x0000},
{0x0B8F, 0x0000, 0x0000},
{0x0B90, 0x0000, 0x0000},
{0x0B92, 0x0000, 0x0000},
{0x0B93, 0x0000, 0x0000},
{0x0B94, 0x0000, 0x0000},
{0x0B83, 0x0000, 0x0000},
{0x0B95, 0x0000, 0x0000},
{0x0B99, 0x0000, 0x0000},
{0x0B9A, 0x0000, 0x0000},
{0x0B9E, 0x0000, 0x0000},
{0x0B9F, 0x0000, 0x0000},
{0x0BA3, 0x0000, 0x0000},
{0x0BA4, 0x0000, 0x0000},
{0x0BA8, 0x0000, 0x0000},
{0x0BAA, 0x0000, 0x0000},
{0x0BAE, 0x0000, 0x0000},
{0x0BAF, 0x0000, 0x0000},
{0x0BB0, 0x0000, 0x0000},
{0x0BB2, 0x0000, 0x0000},
{0x0BB5, 0x0000, 0x0000},
{0x0BB4, 0x0000, 0x0000},
{0x0BB3, 0x0000, 0x0000},
{0x0BB1, 0x0000, 0x0000},
{0x0BA9, 0x0000, 0x0000},
{0x0B9F, 0x0BBF, 0x0000},
{0x0B9F, 0x0BC0, 0x0000},
{0x0B95, 0x0BC1, 0x0000},
{0x0B9A, 0x0BC1, 0x0000},
{0x0B9F, 0x0BC1, 0x0000},
{0x0BA3, 0x0BC1, 0x0000},
{0x0BA4, 0x0BC1, 0x0000},
{0x0BA8, 0x0BC1, 0x0000},
{0x0BAA, 0x0BC1, 0x0000},
{0x0BAE, 0x0BC1, 0x0000},
{0x0BAF, 0x0BC1, 0x0000},
{0x0BB0, 0x0BC1, 0x0000},
{0x0BB2, 0x0BC1, 0x0000},
{0x0BB5, 0x0BC1, 0x0000},
{0x0BB4, 0x0BC1, 0x0000},
{0x0BB3, 0x0BC1, 0x0000},
{0x0BB1, 0x0BC1, 0x0000},
{0x0BA9, 0x0BC1, 0x0000},
{0x0B95, 0x0BC2, 0x0000},
{0x0B9A, 0x0BC2, 0x0000},
{0x0B9F, 0x0BC2, 0x0000},
{0x0BA3, 0x0BC2, 0x0000},
{0x0BA4, 0x0BC2, 0x0000},
{0x0BA8, 0x0BC2, 0x0000},
{0x0BAA, 0x0BC2, 0x0000},
{0x0BAE, 0x0BC2, 0x0000},
{0x0BAF, 0x0BC2, 0x0000},
{0x0BB0, 0x0BC2, 0x0000},
{0x0BB2, 0x0BC2, 0x0000},
{0x0BB5, 0x0BC2, 0x0000},
{0x0BB4, 0x0BC2, 0x0000},
{0x0BB3, 0x0BC2, 0x0000},
{0x0BB1, 0x0BC2, 0x0000},
{0x0BA9, 0x0BC2, 0x0000},
{0x0B95, 0x0B82, 0x0000},
{0x0B99, 0x0B82, 0x0000},
{0x0B9A, 0x0B82, 0x0000},
{0x0B9E, 0x0B82, 0x0000},
{0x0B9F, 0x0B82, 0x0000},
{0x0BA3, 0x0B82, 0x0000},
{0x0BA1, 0x0B82, 0x0000},
{0x0BA8, 0x0B82, 0x0000},
{0x0BAA, 0x0B82, 0x0000},
{0x0BAE, 0x0B82, 0x0000},
{0x0BAF, 0x0B82, 0x0000},
{0x0BB0, 0x0B82, 0x0000},
{0x0BB2, 0x0B82, 0x0000},
{0x0BB5, 0x0B82, 0x0000},
{0x0BB4, 0x0B82, 0x0000},
{0x0BB3, 0x0B82, 0x0000},
{0x0BB1, 0x0B82, 0x0000},
{0x0BA9, 0x0B82, 0x0000}
};
static int cmp(const ushort *s1, const ushort *s2, size_t len)
{
int diff = 0;
while (len-- && (diff = *s1++ - *s2++) == 0)
;
return diff;
}
static unsigned char qt_UnicodeToTSCII(ushort u1, ushort u2, ushort u3)
{
ushort s[3];
s[0] = u1;
s[1] = u2;
s[2] = u3;
int a = 0; // start pos
int b = UnToTsLast; // end pos
// do a binary search for the composed unicode in the list
while (a <= b) {
int w = (a + b) / 2;
int j = cmp(UnToTs[w], s, 3);
if (j == 0)
// found it
return UnToTs[w][3];
if (j < 0)
a = w + 1;
else
b = w - 1;
}
return 0;
}
static unsigned int qt_TSCIIToUnicode(uint code, uint *s)
{
int len = 0;
for (int i = 0; i < 3; i++) {
uint u = TsToUn[code & 0x7f][i];
s[i] = u;
if (s[i]) len = i + 1;
}
return len;
}
QT_END_NAMESPACE
#endif // QT_NO_CODECS

98
src/core/codecs/qtsciicodec_p.h
View file

@ -1,98 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
// Most of the code here was originally written by Hans Petter Bieker,
// and is included in Qt with the author's permission, and the grateful
// thanks of the Qt team.
/*
* Copyright (C) 2000 Hans Petter Bieker. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef QTSCIICODEC_P_H
#define QTSCIICODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists for the convenience
// of other Qt classes. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include "QtCore/qtextcodec.h"
QT_BEGIN_NAMESPACE
#ifndef QT_NO_CODECS
class QTsciiCodec : public QTextCodec {
public:
~QTsciiCodec();
QByteArray name() const;
int mibEnum() const;
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
};
#endif // QT_NO_CODECS
QT_END_NAMESPACE
#endif // QTSCIICODEC_P_H

658
src/core/codecs/qutfcodec.cpp
View file

@ -1,658 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qutfcodec_p.h"
#include "qlist.h"
#include "qendian.h"
#include "qchar.h"
QT_BEGIN_NAMESPACE
enum { Endian = 0, Data = 1 };
static inline bool isUnicodeNonCharacter(uint ucs4)
{
// Unicode has a couple of "non-characters" that one can use internally,
// but are not allowed to be used for text interchange.
//
// Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF,
// U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and
// U+FDEF (inclusive)
return (ucs4 & 0xfffe) == 0xfffe || (ucs4 - 0xfdd0U) < 16;
}
QByteArray QUtf8::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state)
{
uchar replacement = '?';
int rlen = 3*len;
int surrogate_high = -1;
if (state) {
if (state->flags & QTextCodec::ConvertInvalidToNull)
replacement = 0;
if (!(state->flags & QTextCodec::IgnoreHeader))
rlen += 3;
if (state->remainingChars)
surrogate_high = state->state_data[0];
}
QByteArray rstr(rlen, Qt::Uninitialized);
uchar* cursor = (uchar*)rstr.data();
const QChar *ch = uc;
int invalid = 0;
if (state && !(state->flags & QTextCodec::IgnoreHeader)) {
*cursor++ = 0xef;
*cursor++ = 0xbb;
*cursor++ = 0xbf;
}
const QChar *end = ch + len;
while (ch < end) {
uint u = ch->unicode();
if (surrogate_high >= 0) {
if (ch->isLowSurrogate()) {
u = QChar::surrogateToUcs4(surrogate_high, u);
surrogate_high = -1;
} else {
// high surrogate without low
*cursor = replacement;
++ch;
++invalid;
surrogate_high = -1;
continue;
}
} else if (ch->isLowSurrogate()) {
// low surrogate without high
*cursor = replacement;
++ch;
++invalid;
continue;
} else if (ch->isHighSurrogate()) {
surrogate_high = u;
++ch;
continue;
}
if (u < 0x80) {
*cursor++ = (uchar)u;
} else {
if (u < 0x0800) {
*cursor++ = 0xc0 | ((uchar) (u >> 6));
} else {
// is it one of the Unicode non-characters?
if (isUnicodeNonCharacter(u)) {
*cursor++ = replacement;
++ch;
++invalid;
continue;
}
if (u > 0xffff) {
*cursor++ = 0xf0 | ((uchar) (u >> 18));
*cursor++ = 0x80 | (((uchar) (u >> 12)) & 0x3f);
} else {
*cursor++ = 0xe0 | (((uchar) (u >> 12)) & 0x3f);
}
*cursor++ = 0x80 | (((uchar) (u >> 6)) & 0x3f);
}
*cursor++ = 0x80 | ((uchar) (u&0x3f));
}
++ch;
}
rstr.resize(cursor - (const uchar*)rstr.constData());
if (state) {
state->invalidChars += invalid;
state->flags |= QTextCodec::IgnoreHeader;
state->remainingChars = 0;
if (surrogate_high >= 0) {
state->remainingChars = 1;
state->state_data[0] = surrogate_high;
}
}
return rstr;
}
QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state)
{
bool headerdone = false;
ushort replacement = QChar::ReplacementCharacter;
int need = 0;
int error = -1;
uint uc = 0;
uint min_uc = 0;
if (state) {
if (state->flags & QTextCodec::IgnoreHeader)
headerdone = true;
if (state->flags & QTextCodec::ConvertInvalidToNull)
replacement = QChar::Null;
need = state->remainingChars;
if (need) {
uc = state->state_data[0];
min_uc = state->state_data[1];
}
}
if (!headerdone && len > 3
&& (uchar)chars[0] == 0xef && (uchar)chars[1] == 0xbb && (uchar)chars[2] == 0xbf) {
// starts with a byte order mark
chars += 3;
len -= 3;
headerdone = true;
}
QString result(need + len + 1, Qt::Uninitialized); // worst case
ushort *qch = (ushort *)result.unicode();
uchar ch;
int invalid = 0;
for (int i = 0; i < len; ++i) {
ch = chars[i];
if (need) {
if ((ch&0xc0) == 0x80) {
uc = (uc << 6) | (ch & 0x3f);
--need;
if (!need) {
// utf-8 bom composes into 0xfeff code point
bool nonCharacter;
if (!headerdone && uc == 0xfeff) {
// don't do anything, just skip the BOM
} else if (!(nonCharacter = isUnicodeNonCharacter(uc)) && uc > 0xffff && uc < 0x110000) {
// surrogate pair
Q_ASSERT((qch - (ushort*)result.unicode()) + 2 < result.length());
*qch++ = QChar::highSurrogate(uc);
*qch++ = QChar::lowSurrogate(uc);
} else if ((uc < min_uc) || (uc >= 0xd800 && uc <= 0xdfff) || nonCharacter || uc >= 0x110000) {
// error: overlong sequence, UTF16 surrogate or non-character
*qch++ = replacement;
++invalid;
} else {
*qch++ = uc;
}
headerdone = true;
}
} else {
// error
i = error;
*qch++ = replacement;
++invalid;
need = 0;
headerdone = true;
}
} else {
if (ch < 128) {
*qch++ = ushort(ch);
headerdone = true;
} else if ((ch & 0xe0) == 0xc0) {
uc = ch & 0x1f;
need = 1;
error = i;
min_uc = 0x80;
headerdone = true;
} else if ((ch & 0xf0) == 0xe0) {
uc = ch & 0x0f;
need = 2;
error = i;
min_uc = 0x800;
} else if ((ch&0xf8) == 0xf0) {
uc = ch & 0x07;
need = 3;
error = i;
min_uc = 0x10000;
headerdone = true;
} else {
// error
*qch++ = replacement;
++invalid;
headerdone = true;
}
}
}
if (!state && need > 0) {
// unterminated UTF sequence
for (int i = error; i < len; ++i) {
*qch++ = replacement;
++invalid;
}
}
result.truncate(qch - (ushort *)result.unicode());
if (state) {
state->invalidChars += invalid;
state->remainingChars = need;
if (headerdone)
state->flags |= QTextCodec::IgnoreHeader;
state->state_data[0] = need ? uc : 0;
state->state_data[1] = need ? min_uc : 0;
}
return result;
}
QByteArray QUtf16::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state, DataEndianness e)
{
DataEndianness endian = e;
int length = 2*len;
if (!state || (!(state->flags & QTextCodec::IgnoreHeader))) {
length += 2;
}
if (e == DetectEndianness) {
endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness;
}
QByteArray result(length, Qt::Uninitialized);
char *data = result.data();
if (!state || !(state->flags & QTextCodec::IgnoreHeader)) {
QChar bom(QChar::ByteOrderMark);
if (endian == BigEndianness) {
data[0] = bom.row();
data[1] = bom.cell();
} else {
data[0] = bom.cell();
data[1] = bom.row();
}
data += 2;
}
if (endian == BigEndianness) {
for (int i = 0; i < len; ++i) {
*(data++) = uc[i].row();
*(data++) = uc[i].cell();
}
} else {
for (int i = 0; i < len; ++i) {
*(data++) = uc[i].cell();
*(data++) = uc[i].row();
}
}
if (state) {
state->remainingChars = 0;
state->flags |= QTextCodec::IgnoreHeader;
}
return result;
}
QString QUtf16::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state, DataEndianness e)
{
DataEndianness endian = e;
bool half = false;
uchar buf = 0;
bool headerdone = false;
if (state) {
headerdone = state->flags & QTextCodec::IgnoreHeader;
if (endian == DetectEndianness)
endian = (DataEndianness)state->state_data[Endian];
if (state->remainingChars) {
half = true;
buf = state->state_data[Data];
}
}
if (headerdone && endian == DetectEndianness)
endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness;
QString result(len, Qt::Uninitialized); // worst case
QChar *qch = (QChar *)result.unicode();
while (len--) {
if (half) {
QChar ch;
if (endian == LittleEndianness) {
ch.setRow(*chars++);
ch.setCell(buf);
} else {
ch.setRow(buf);
ch.setCell(*chars++);
}
if (!headerdone) {
headerdone = true;
if (endian == DetectEndianness) {
if (ch == QChar::ByteOrderSwapped) {
endian = LittleEndianness;
} else if (ch == QChar::ByteOrderMark) {
endian = BigEndianness;
} else {
if (QSysInfo::ByteOrder == QSysInfo::BigEndian) {
endian = BigEndianness;
} else {
endian = LittleEndianness;
ch = QChar((ch.unicode() >> 8) | ((ch.unicode() & 0xff) << 8));
}
*qch++ = ch;
}
} else if (ch != QChar::ByteOrderMark) {
*qch++ = ch;
}
} else {
*qch++ = ch;
}
half = false;
} else {
buf = *chars++;
half = true;
}
}
result.truncate(qch - result.unicode());
if (state) {
if (headerdone)
state->flags |= QTextCodec::IgnoreHeader;
state->state_data[Endian] = endian;
if (half) {
state->remainingChars = 1;
state->state_data[Data] = buf;
} else {
state->remainingChars = 0;
state->state_data[Data] = 0;
}
}
return result;
}
QByteArray QUtf32::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state, DataEndianness e)
{
DataEndianness endian = e;
int length = 4*len;
if (!state || (!(state->flags & QTextCodec::IgnoreHeader))) {
length += 4;
}
if (e == DetectEndianness) {
endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness;
}
QByteArray d(length, Qt::Uninitialized);
char *data = d.data();
if (!state || !(state->flags & QTextCodec::IgnoreHeader)) {
if (endian == BigEndianness) {
data[0] = 0;
data[1] = 0;
data[2] = (char)0xfe;
data[3] = (char)0xff;
} else {
data[0] = (char)0xff;
data[1] = (char)0xfe;
data[2] = 0;
data[3] = 0;
}
data += 4;
}
if (endian == BigEndianness) {
for (int i = 0; i < len; ++i) {
uint cp = uc[i].unicode();
if (uc[i].isHighSurrogate() && i < len - 1)
cp = QChar::surrogateToUcs4(cp, uc[++i].unicode());
*(data++) = cp >> 24;
*(data++) = (cp >> 16) & 0xff;
*(data++) = (cp >> 8) & 0xff;
*(data++) = cp & 0xff;
}
} else {
for (int i = 0; i < len; ++i) {
uint cp = uc[i].unicode();
if (uc[i].isHighSurrogate() && i < len - 1)
cp = QChar::surrogateToUcs4(cp, uc[++i].unicode());
*(data++) = cp & 0xff;
*(data++) = (cp >> 8) & 0xff;
*(data++) = (cp >> 16) & 0xff;
*(data++) = cp >> 24;
}
}
if (state) {
state->remainingChars = 0;
state->flags |= QTextCodec::IgnoreHeader;
}
return d;
}
QString QUtf32::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state, DataEndianness e)
{
DataEndianness endian = e;
uchar tuple[4];
int num = 0;
bool headerdone = false;
if (state) {
headerdone = state->flags & QTextCodec::IgnoreHeader;
if (endian == DetectEndianness) {
endian = (DataEndianness)state->state_data[Endian];
}
num = state->remainingChars;
memcpy(tuple, &state->state_data[Data], 4);
}
if (headerdone && endian == DetectEndianness)
endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness;
QString result((num + len) >> 2 << 1, Qt::Uninitialized); // worst case
QChar *qch = (QChar *)result.unicode();
const char *end = chars + len;
while (chars < end) {
tuple[num++] = *chars++;
if (num == 4) {
if (!headerdone) {
if (endian == DetectEndianness) {
if (endian == DetectEndianness) {
if (tuple[0] == 0xff && tuple[1] == 0xfe && tuple[2] == 0 && tuple[3] == 0 && endian != BigEndianness) {
endian = LittleEndianness;
num = 0;
continue;
} else if (tuple[0] == 0 && tuple[1] == 0 && tuple[2] == 0xfe && tuple[3] == 0xff && endian != LittleEndianness) {
endian = BigEndianness;
num = 0;
continue;
} else if (QSysInfo::ByteOrder == QSysInfo::BigEndian) {
endian = BigEndianness;
} else {
endian = LittleEndianness;
}
}
} else if (((endian == BigEndianness) ? qFromBigEndian<quint32>(tuple) : qFromLittleEndian<quint32>(tuple)) == QChar::ByteOrderMark) {
num = 0;
continue;
}
}
uint code = (endian == BigEndianness) ? qFromBigEndian<quint32>(tuple) : qFromLittleEndian<quint32>(tuple);
if (code >= 0x10000) {
*qch++ = QChar::highSurrogate(code);
*qch++ = QChar::lowSurrogate(code);
} else {
*qch++ = code;
}
num = 0;
}
}
result.truncate(qch - result.unicode());
if (state) {
if (headerdone)
state->flags |= QTextCodec::IgnoreHeader;
state->state_data[Endian] = endian;
state->remainingChars = num;
memcpy(&state->state_data[Data], tuple, 4);
}
return result;
}
#ifndef QT_NO_TEXTCODEC
QUtf8Codec::~QUtf8Codec()
{
}
QByteArray QUtf8Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const
{
return QUtf8::convertFromUnicode(uc, len, state);
}
void QUtf8Codec::convertToUnicode(QString *target, const char *chars, int len, ConverterState *state) const
{
*target += QUtf8::convertToUnicode(chars, len, state);
}
QString QUtf8Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const
{
return QUtf8::convertToUnicode(chars, len, state);
}
QByteArray QUtf8Codec::name() const
{
return "UTF-8";
}
int QUtf8Codec::mibEnum() const
{
return 106;
}
QUtf16Codec::~QUtf16Codec()
{
}
QByteArray QUtf16Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const
{
return QUtf16::convertFromUnicode(uc, len, state, e);
}
QString QUtf16Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const
{
return QUtf16::convertToUnicode(chars, len, state, e);
}
int QUtf16Codec::mibEnum() const
{
return 1015;
}
QByteArray QUtf16Codec::name() const
{
return "UTF-16";
}
QList<QByteArray> QUtf16Codec::aliases() const
{
return QList<QByteArray>();
}
int QUtf16BECodec::mibEnum() const
{
return 1013;
}
QByteArray QUtf16BECodec::name() const
{
return "UTF-16BE";
}
QList<QByteArray> QUtf16BECodec::aliases() const
{
QList<QByteArray> list;
return list;
}
int QUtf16LECodec::mibEnum() const
{
return 1014;
}
QByteArray QUtf16LECodec::name() const
{
return "UTF-16LE";
}
QList<QByteArray> QUtf16LECodec::aliases() const
{
QList<QByteArray> list;
return list;
}
QUtf32Codec::~QUtf32Codec()
{
}
QByteArray QUtf32Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const
{
return QUtf32::convertFromUnicode(uc, len, state, e);
}
QString QUtf32Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const
{
return QUtf32::convertToUnicode(chars, len, state, e);
}
int QUtf32Codec::mibEnum() const
{
return 1017;
}
QByteArray QUtf32Codec::name() const
{
return "UTF-32";
}
QList<QByteArray> QUtf32Codec::aliases() const
{
QList<QByteArray> list;
return list;
}
int QUtf32BECodec::mibEnum() const
{
return 1018;
}
QByteArray QUtf32BECodec::name() const
{
return "UTF-32BE";
}
QList<QByteArray> QUtf32BECodec::aliases() const
{
QList<QByteArray> list;
return list;
}
int QUtf32LECodec::mibEnum() const
{
return 1019;
}
QByteArray QUtf32LECodec::name() const
{
return "UTF-32LE";
}
QList<QByteArray> QUtf32LECodec::aliases() const
{
QList<QByteArray> list;
return list;
}
#endif //QT_NO_TEXTCODEC
QT_END_NAMESPACE

162
src/core/codecs/qutfcodec_p.h
View file

@ -1,162 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2015 The Qt Company Ltd.
** Copyright (C) 2016-2019 Ivailo Monev
**
** This file is part of the QtCore module of the Katie Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 or version 3 as published by the Free
** Software Foundation and appearing in the file LICENSE.LGPLv21 and
** LICENSE.LGPLv3 included in the packaging of this file. Please review the
** following information to ensure the GNU Lesser General Public License
** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** As a special exception, The Qt Company gives you certain additional
** rights. These rights are described in The Qt Company LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QUTFCODEC_P_H
#define QUTFCODEC_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Katie API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include "QtCore/qtextcodec.h"
#include "qtextcodec_p.h"
QT_BEGIN_NAMESPACE
enum DataEndianness
{
DetectEndianness,
BigEndianness,
LittleEndianness
};
struct QUtf8
{
static QString convertToUnicode(const char *, int, QTextCodec::ConverterState *);
static QByteArray convertFromUnicode(const QChar *, int, QTextCodec::ConverterState *);
};
struct QUtf16
{
static QString convertToUnicode(const char *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness);
static QByteArray convertFromUnicode(const QChar *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness);
};
struct QUtf32
{
static QString convertToUnicode(const char *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness);
static QByteArray convertFromUnicode(const QChar *, int, QTextCodec::ConverterState *, DataEndianness = DetectEndianness);
};
#ifndef QT_NO_TEXTCODEC
class QUtf8Codec : public QTextCodec {
public:
~QUtf8Codec();
QByteArray name() const;
int mibEnum() const;
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
void convertToUnicode(QString *target, const char *, int, ConverterState *) const;
};
class QUtf16Codec : public QTextCodec {
protected:
public:
QUtf16Codec() { e = DetectEndianness; }
~QUtf16Codec();
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
protected:
DataEndianness e;
};
class QUtf16BECodec : public QUtf16Codec {
public:
QUtf16BECodec() : QUtf16Codec() { e = BigEndianness; }
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
};
class QUtf16LECodec : public QUtf16Codec {
public:
QUtf16LECodec() : QUtf16Codec() { e = LittleEndianness; }
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
};
class QUtf32Codec : public QTextCodec {
public:
QUtf32Codec() { e = DetectEndianness; }
~QUtf32Codec();
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
QString convertToUnicode(const char *, int, ConverterState *) const;
QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
protected:
DataEndianness e;
};
class QUtf32BECodec : public QUtf32Codec {
public:
QUtf32BECodec() : QUtf32Codec() { e = BigEndianness; }
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
};
class QUtf32LECodec : public QUtf32Codec {
public:
QUtf32LECodec() : QUtf32Codec() { e = LittleEndianness; }
QByteArray name() const;
QList<QByteArray> aliases() const;
int mibEnum() const;
};
#endif // QT_NO_TEXTCODEC
QT_END_NAMESPACE
#endif // QUTFCODEC_P_H

2
src/core/global/qconfig.h.cmake
View file

@ -89,6 +89,7 @@
#define QT_NO_RAWFONT
#define QT_NO_IMAGE_TEXT
#define QT_NO_GLIB
#define QT_NO_ICONV
/* Qt build specs */
#ifndef QT_EDITION
@ -179,7 +180,6 @@
#cmakedefine QT_NO_GROUPBOX
#cmakedefine QT_NO_HTTP
#cmakedefine QT_NO_ICON
#cmakedefine QT_NO_ICONV
#cmakedefine QT_NO_IDENTITYPROXYMODEL
#cmakedefine QT_NO_IMAGEFORMATPLUGIN
#cmakedefine QT_NO_IMAGEFORMAT_BMP

3
src/core/io/qtextstream.cpp
View file

@ -457,9 +457,6 @@ static void copyConverterStateHelper(QTextCodec::ConverterState *dest,
// private. This function copies the structure manually.
dest->flags = src->flags;
dest->invalidChars = src->invalidChars;
dest->state_data[0] = src->state_data[0];
dest->state_data[1] = src->state_data[1];
dest->state_data[2] = src->state_data[2];
}
#endif

17
src/core/tools/qstring.cpp
View file

@ -36,7 +36,7 @@
#ifndef QT_NO_TEXTCODEC
#include "qtextcodec.h"
#endif
#include "qutfcodec_p.h"
#include "qicucodec_p.h"
#include "qdatastream.h"
#include "qlist.h"
#include "qlocale.h"
@ -3455,7 +3455,8 @@ QByteArray QString::toUtf8() const
if (isNull())
return QByteArray();
return QUtf8::convertFromUnicode(constData(), length(), Q_NULLPTR);
QTextCodec *c = QTextCodec::codecForName("UTF-8");
return c->fromUnicode(constData(), length(), Q_NULLPTR);
}
/*!
@ -3618,7 +3619,8 @@ QString QString::fromUtf8(const char *str, int size)
if (size < 0)
size = qstrlen(str);
return QUtf8::convertToUnicode(str, size, Q_NULLPTR);
QTextCodec *c = QTextCodec::codecForName("UTF-8");
return c->toUnicode(str, size, Q_NULLPTR);
}
/*!
@ -3647,7 +3649,8 @@ QString QString::fromUtf16(const ushort *unicode, int size)
while (unicode[size] != 0)
++size;
}
return QUtf16::convertToUnicode((const char *)unicode, size*2, Q_NULLPTR);
QTextCodec *c = QTextCodec::codecForName("UTF-16");
return c->toUnicode((const char *)unicode, size, Q_NULLPTR);
}
@ -3671,7 +3674,8 @@ QString QString::fromUcs4(const uint *unicode, int size)
while (unicode[size] != 0)
++size;
}
return QUtf32::convertToUnicode((const char *)unicode, size*4, Q_NULLPTR);
QTextCodec *c = QTextCodec::codecForName("UTF-32");
return c->toUnicode((const char *)unicode, size, Q_NULLPTR);
}
/*!
@ -8535,7 +8539,8 @@ QByteArray QStringRef::toUtf8() const
if (isNull())
return QByteArray();
return QUtf8::convertFromUnicode(constData(), length(), Q_NULLPTR);
QTextCodec *c = QTextCodec::codecForName("UTF-8");
return c->fromUnicode(constData(), length(), Q_NULLPTR);
}
/*!

4
src/gui/kernel/qapplication_x11.cpp
View file

@ -886,10 +886,10 @@ static void qt_set_input_encoding()
qt_input_mapper = QTextCodec::codecForName((char *)data);
// make sure we have an input codec
if(!qt_input_mapper)
qt_input_mapper = QTextCodec::codecForName("ISO 8859-1");
qt_input_mapper = QTextCodec::codecForName("ISO-8859-1");
}
if (qt_input_mapper && qt_input_mapper->mibEnum() == 11) // 8859-8
qt_input_mapper = QTextCodec::codecForName("ISO 8859-8-I");
qt_input_mapper = QTextCodec::codecForName("ISO-8859-8-I");
if(data)
XFree((char *)data);
}

6
src/tools/moc/CMakeLists.txt
View file

@ -42,12 +42,8 @@ set(BOOTSTRAP_DEFINITIONS
)
set(BOOTSTRAP_SOURCES
${CMAKE_SOURCE_DIR}/src/core/codecs/qisciicodec.cpp
${CMAKE_SOURCE_DIR}/src/core/codecs/qlatincodec.cpp
${CMAKE_SOURCE_DIR}/src/core/codecs/qsimplecodec.cpp
${CMAKE_SOURCE_DIR}/src/core/codecs/qtextcodec.cpp
${CMAKE_SOURCE_DIR}/src/core/codecs/qtsciicodec.cpp
${CMAKE_SOURCE_DIR}/src/core/codecs/qutfcodec.cpp
${CMAKE_SOURCE_DIR}/src/core/codecs/qicucodec.cpp
${CMAKE_SOURCE_DIR}/src/core/global/qglobal.cpp
${CMAKE_SOURCE_DIR}/src/core/global/qnumeric.cpp
${CMAKE_SOURCE_DIR}/src/core/io/qabstractfileengine.cpp

6
tests/benchmarks/core/tools/qstring/main.cpp
View file

@ -656,10 +656,6 @@ int fromUtf8_qt47(ushort *dst, const char *chars, int len)
if (state->flags & QTextCodec::ConvertInvalidToNull)
replacement = QChar::Null;
need = state->remainingChars;
if (need) {
uc = state->state_data[0];
min_uc = state->state_data[1];
}
}
if (!headerdone && len > 3
&& (uchar)chars[0] == 0xef && (uchar)chars[1] == 0xbb && (uchar)chars[2] == 0xbf) {
@ -750,8 +746,6 @@ int fromUtf8_qt47(ushort *dst, const char *chars, int len)
state->remainingChars = need;
if (headerdone)
state->flags |= QTextCodec::IgnoreHeader;
state->state_data[0] = need ? uc : 0;
state->state_data[1] = need ? min_uc : 0;
}
//return result;
return qch - dst;