source: trunk/src/corelib/codecs/qiconvcodec.cpp@ 561

Last change on this file since 561 was 561, checked in by Dmitry A. Kuminov, 15 years ago

trunk: Merged in qt 4.6.1 sources.
File size: 17.7 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
4** All rights reserved.
5** Contact: Nokia Corporation ([email protected])
6**
7** This file is part of the QtCore module of the Qt Toolkit.
8**
9** $QT_BEGIN_LICENSE:LGPL$
10** Commercial Usage
11** Licensees holding valid Qt Commercial licenses may use this file in
12** accordance with the Qt Commercial License Agreement provided with the
13** Software or, alternatively, in accordance with the terms contained in
14** a written agreement between you and Nokia.
15**
16** GNU Lesser General Public License Usage
17** Alternatively, this file may be used under the terms of the GNU Lesser
18** General Public License version 2.1 as published by the Free Software
19** Foundation and appearing in the file LICENSE.LGPL included in the
20** packaging of this file. Please review the following information to
21** ensure the GNU Lesser General Public License version 2.1 requirements
22** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
23**
24** In addition, as a special exception, Nokia gives you certain additional
25** rights. These rights are described in the Nokia Qt LGPL Exception
26** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you have questions regarding the use of this file, please contact
37** Nokia at [email protected].
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42#include "qiconvcodec_p.h"
43#include "qtextcodec_p.h"
44#include <qlibrary.h>
45#include <qdebug.h>
46#include <qthreadstorage.h>
47
48#include <errno.h>
49#include <locale.h>
50#include <stdio.h>
51#include <dlfcn.h>
52
53// unistd.h is needed for the _XOPEN_UNIX macro
54#include <unistd.h>
55#if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)
56# include <langinfo.h>
57#endif
58
59#if defined(Q_OS_HPUX)
60# define NO_BOM
61# define UTF16 "ucs2"
62#elif defined(Q_OS_AIX)
63# define NO_BOM
64# define UTF16 "UCS-2"
65#elif defined(Q_OS_FREEBSD) || defined(Q_OS_MAC)
66# define NO_BOM
67# if Q_BYTE_ORDER == Q_BIG_ENDIAN
68# define UTF16 "UTF-16BE"
69# else
70# define UTF16 "UTF-16LE"
71# endif
72#else
73# define UTF16 "UTF-16"
74#endif
75
76#if defined(Q_OS_MAC)
77#ifndef GNU_LIBICONV
78#define GNU_LIBICONV
79#endif
80typedef iconv_t (*Ptr_iconv_open) (const char*, const char*);
81typedef size_t (*Ptr_iconv) (iconv_t, const char **, size_t *, char **, size_t *);
82typedef int (*Ptr_iconv_close) (iconv_t);
83
84static Ptr_iconv_open ptr_iconv_open = 0;
85static Ptr_iconv ptr_iconv = 0;
86static Ptr_iconv_close ptr_iconv_close = 0;
87#endif
88
89QT_BEGIN_NAMESPACE
90
91extern bool qt_locale_initialized;
92
93QIconvCodec::QIconvCodec()
94 : utf16Codec(0)
95{
96 utf16Codec = QTextCodec::codecForMib(1015);
97 Q_ASSERT_X(utf16Codec != 0,
98 "QIconvCodec::convertToUnicode",
99 "internal error, UTF-16 codec not found");
100 if (!utf16Codec) {
101 fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n");
102 utf16Codec = reinterpret_cast<QTextCodec *>(~0);
103 }
104#if defined(Q_OS_MAC)
105 if (ptr_iconv_open == 0) {
106 QLibrary libiconv(QLatin1String("/usr/lib/libiconv"));
107 libiconv.setLoadHints(QLibrary::ExportExternalSymbolsHint);
108
109 ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("libiconv_open"));
110 if (!ptr_iconv_open)
111 ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("iconv_open"));
112 ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("libiconv"));
113 if (!ptr_iconv)
114 ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("iconv"));
115 ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("libiconv_close"));
116 if (!ptr_iconv_close)
117 ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("iconv_close"));
118
119 Q_ASSERT_X(ptr_iconv_open && ptr_iconv && ptr_iconv_close,
120 "QIconvCodec::QIconvCodec()",
121 "internal error, could not resolve the iconv functions");
122
123# undef iconv_open
124# define iconv_open ptr_iconv_open
125# undef iconv
126# define iconv ptr_iconv
127# undef iconv_close
128# define iconv_close ptr_iconv_close
129 }
130#endif
131}
132
133QIconvCodec::~QIconvCodec()
134{
135}
136
137QIconvCodec::IconvState::IconvState(iconv_t x)
138 : buffer(array), bufferLen(sizeof array), cd(x)
139{
140}
141
142QIconvCodec::IconvState::~IconvState()
143{
144 if (cd != reinterpret_cast<iconv_t>(-1))
145 iconv_close(cd);
146 if (buffer != array)
147 delete[] buffer;
148}
149
150void QIconvCodec::IconvState::saveChars(const char *c, int count)
151{
152 if (count > bufferLen) {
153 if (buffer != array)
154 delete[] buffer;
155 buffer = new char[bufferLen = count];
156 }
157
158 memcpy(buffer, c, count);
159}
160
161static void qIconvCodecStateFree(QTextCodec::ConverterState *state)
162{
163 delete reinterpret_cast<QIconvCodec::IconvState *>(state->d);
164}
165
166Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, toUnicodeState)
167
168QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const
169{
170 if (utf16Codec == reinterpret_cast<QTextCodec *>(~0))
171 return QString::fromAscii(chars, len);
172
173 int invalidCount = 0;
174 int remainingCount = 0;
175 char *remainingBuffer = 0;
176 IconvState **pstate;
177
178 if (convState) {
179 // stateful conversion
180 pstate = reinterpret_cast<IconvState **>(&convState->d);
181 if (convState->d) {
182 // restore state
183 remainingCount = convState->remainingChars;
184 remainingBuffer = (*pstate)->buffer;
185 } else {
186 // first time
187 convState->flags |= FreeFunction;
188 QTextCodecUnalignedPointer::encode(convState->state_data, qIconvCodecStateFree);
189 }
190 } else {
191 QThreadStorage<QIconvCodec::IconvState *> *ts = toUnicodeState();
192 if (!qt_locale_initialized || !ts) {
193 // we're running after the Q_GLOBAL_STATIC has been deleted
194 // or before the QCoreApplication initialization
195 // bad programmer, no cookie for you
196 return QString::fromLatin1(chars, len);
197 }
198
199 // stateless conversion -- use thread-local data
200 pstate = &toUnicodeState()->localData();
201 }
202
203 if (!*pstate) {
204 // first time, create the state
205 iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0);
206 if (cd == reinterpret_cast<iconv_t>(-1)) {
207 static int reported = 0;
208 if (!reported++) {
209 fprintf(stderr,
210 "QIconvCodec::convertToUnicode: using ASCII for conversion, iconv_open failed\n");
211 }
212 return QString::fromAscii(chars, len);
213 }
214
215 *pstate = new IconvState(cd);
216 }
217
218 IconvState *state = *pstate;
219 size_t inBytesLeft = len;
220 // best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM
221#ifdef GNU_LIBICONV
222 // GNU doesn't disagree with POSIX :/
223 const char *inBytes = chars;
224#else
225 char *inBytes = const_cast<char *>(chars);
226#endif
227
228 QByteArray in;
229 if (remainingCount) {
230 // we have to prepend the remaining bytes from the previous conversion
231 inBytesLeft += remainingCount;
232 in.resize(inBytesLeft);
233 inBytes = in.data();
234
235 memcpy(in.data(), remainingBuffer, remainingCount);
236 memcpy(in.data() + remainingCount, chars, len);
237
238 remainingCount = 0;
239 }
240
241 size_t outBytesLeft = len * 2 + 2;
242 QByteArray ba(outBytesLeft, Qt::Uninitialized);
243 char *outBytes = ba.data();
244 do {
245 size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft);
246 if (ret == (size_t) -1) {
247 if (errno == E2BIG) {
248 int offset = ba.size() - outBytesLeft;
249 ba.resize(ba.size() * 2);
250 outBytes = ba.data() + offset;
251 outBytesLeft = ba.size() - offset;
252
253 continue;
254 }
255
256 if (errno == EILSEQ) {
257 // conversion stopped because of an invalid character in the sequence
258 ++invalidCount;
259 } else if (errno == EINVAL && convState) {
260 // conversion stopped because the remaining inBytesLeft make up
261 // an incomplete multi-byte sequence; save them for later
262 state->saveChars(inBytes, inBytesLeft);
263 remainingCount = inBytesLeft;
264 break;
265 }
266
267 if (errno == EILSEQ || errno == EINVAL) {
268 // skip the next character
269 ++inBytes;
270 --inBytesLeft;
271 continue;
272 }
273
274 // some other error
275 // note, cannot use qWarning() since we are implementing the codecForLocale :)
276 perror("QIconvCodec::convertToUnicode: using ASCII for conversion, iconv failed");
277
278 if (!convState) {
279 // reset state
280 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
281 }
282
283 return QString::fromAscii(chars, len);
284 }
285 } while (inBytesLeft != 0);
286
287 QString s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft);
288
289 if (convState) {
290 convState->invalidChars = invalidCount;
291 convState->remainingChars = remainingCount;
292 } else {
293 // reset state
294 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
295 }
296
297 return s;
298}
299
300Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, fromUnicodeState)
301
302QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const
303{
304 char *inBytes;
305 char *outBytes;
306 size_t inBytesLeft;
307
308#if defined(GNU_LIBICONV)
309 const char **inBytesPtr = const_cast<const char **>(&inBytes);
310#else
311 char **inBytesPtr = &inBytes;
312#endif
313
314 QThreadStorage<QIconvCodec::IconvState *> *ts = fromUnicodeState();
315 if (!qt_locale_initialized || !ts) {
316 // we're running after the Q_GLOBAL_STATIC has been deleted
317 // or before the QCoreApplication initialization
318 // bad programmer, no cookie for you
319 if (!len)
320 // this is a special case - zero-sized string should be
321 // translated to empty but not-null QByteArray.
322 return QByteArray("");
323 return QString::fromRawData(uc, len).toLatin1();
324 }
325 IconvState *&state = ts->localData();
326 if (!state) {
327 state = new IconvState(QIconvCodec::createIconv_t(0, UTF16));
328 if (state->cd != reinterpret_cast<iconv_t>(-1)) {
329 size_t outBytesLeft = len + 3; // +3 for the BOM
330 QByteArray ba(outBytesLeft, Qt::Uninitialized);
331 outBytes = ba.data();
332
333#if !defined(NO_BOM)
334 // give iconv() a BOM
335 QChar bom[] = { QChar(QChar::ByteOrderMark) };
336 inBytes = reinterpret_cast<char *>(bom);
337 inBytesLeft = sizeof(bom);
338 if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
339 perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed for BOM");
340
341 iconv_close(state->cd);
342 state->cd = reinterpret_cast<iconv_t>(-1);
343
344 return QString(uc, len).toAscii();
345 }
346#endif // NO_BOM
347 }
348 }
349 if (state->cd == reinterpret_cast<iconv_t>(-1)) {
350 static int reported = 0;
351 if (!reported++) {
352 fprintf(stderr,
353 "QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv_open failed\n");
354 }
355 return QString(uc, len).toAscii();
356 }
357
358 size_t outBytesLeft = len;
359 QByteArray ba(outBytesLeft, Qt::Uninitialized);
360 outBytes = ba.data();
361
362 // now feed iconv() the real data
363 inBytes = const_cast<char *>(reinterpret_cast<const char *>(uc));
364 inBytesLeft = len * sizeof(QChar);
365
366 QByteArray in;
367 if (convState && convState->remainingChars) {
368 // we have one surrogate char to be prepended
369 in.resize(sizeof(QChar) + len);
370 inBytes = in.data();
371
372 QChar remaining = convState->state_data[0];
373 memcpy(in.data(), &remaining, sizeof(QChar));
374 memcpy(in.data() + sizeof(QChar), uc, inBytesLeft);
375
376 inBytesLeft += sizeof(QChar);
377 convState->remainingChars = 0;
378 }
379
380 int invalidCount = 0;
381 while (inBytesLeft != 0) {
382 if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
383 if (errno == EINVAL && convState) {
384 // buffer ends in a surrogate
385 Q_ASSERT(inBytesLeft == 2);
386 convState->remainingChars = 1;
387 convState->state_data[0] = uc[len - 1].unicode();
388 break;
389 }
390
391 switch (errno) {
392 case EILSEQ:
393 ++invalidCount;
394 // fall through
395 case EINVAL:
396 {
397 inBytes += sizeof(QChar);
398 inBytesLeft -= sizeof(QChar);
399 break;
400 }
401 case E2BIG:
402 {
403 int offset = ba.size() - outBytesLeft;
404 ba.resize(ba.size() * 2);
405 outBytes = ba.data() + offset;
406 outBytesLeft = ba.size() - offset;
407 break;
408 }
409 default:
410 {
411 // note, cannot use qWarning() since we are implementing the codecForLocale :)
412 perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed");
413
414 // reset to initial state
415 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
416
417 return QString(uc, len).toAscii();
418 }
419 }
420 }
421 }
422
423 // reset to initial state
424 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
425
426 ba.resize(ba.size() - outBytesLeft);
427
428 if (convState)
429 convState->invalidChars = invalidCount;
430
431 return ba;
432}
433
434QByteArray QIconvCodec::name() const
435{
436 return "System";
437}
438
439int QIconvCodec::mibEnum() const
440{
441 return 0;
442}
443
444iconv_t QIconvCodec::createIconv_t(const char *to, const char *from)
445{
446 Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0));
447
448 iconv_t cd = (iconv_t) -1;
449#if defined(__GLIBC__) || defined(GNU_LIBICONV)
450 // both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string
451 static const char empty_codeset[] = "";
452 const char *codeset = empty_codeset;
453 cd = iconv_open(to ? to : codeset, from ? from : codeset);
454#else
455 char *codeset = 0;
456#endif
457
458#if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)
459 if (cd == (iconv_t) -1) {
460 codeset = nl_langinfo(CODESET);
461 if (codeset)
462 cd = iconv_open(to ? to : codeset, from ? from : codeset);
463 }
464#endif
465
466 if (cd == (iconv_t) -1) {
467 // Very poorly defined and followed standards causes lots of
468 // code to try to get all the cases... This logic is
469 // duplicated in QTextCodec, so if you change it here, change
470 // it there too.
471
472 // Try to determine locale codeset from locale name assigned to
473 // LC_CTYPE category.
474
475 // First part is getting that locale name. First try setlocale() which
476 // definitely knows it, but since we cannot fully trust it, get ready
477 // to fall back to environment variables.
478 char * ctype = qstrdup(setlocale(LC_CTYPE, 0));
479
480 // Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG
481 // environment variables.
482 char * lang = qstrdup(qgetenv("LC_ALL").constData());
483 if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
484 if (lang) delete [] lang;
485 lang = qstrdup(qgetenv("LC_CTYPE").constData());
486 }
487 if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
488 if (lang) delete [] lang;
489 lang = qstrdup(qgetenv("LANG").constData());
490 }
491
492 // Now try these in order:
493 // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
494 // 2. CODESET from lang if it contains a .CODESET part
495 // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
496 // 4. locale (ditto)
497 // 5. check for "@euro"
498
499 // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
500 codeset = ctype ? strchr(ctype, '.') : 0;
501 if (codeset && *codeset == '.') {
502 ++codeset;
503 cd = iconv_open(to ? to : codeset, from ? from : codeset);
504 }
505
506 // 2. CODESET from lang if it contains a .CODESET part
507 codeset = lang ? strchr(lang, '.') : 0;
508 if (cd == (iconv_t) -1 && codeset && *codeset == '.') {
509 ++codeset;
510 cd = iconv_open(to ? to : codeset, from ? from : codeset);
511 }
512
513 // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
514 if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0)
515 cd = iconv_open(to ? to : ctype, from ? from : ctype);
516
517
518 // 4. locale (ditto)
519 if (cd == (iconv_t) -1 && lang && *lang != 0)
520 cd = iconv_open(to ? to : lang, from ? from : lang);
521
522 // 5. "@euro"
523 if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro")))
524 cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15");
525
526 delete [] ctype;
527 delete [] lang;
528 }
529
530 return cd;
531}
532
533QT_END_NAMESPACE
Note: See TracBrowser for help on using the repository browser.