source: trunk/src/corelib/codecs/qiconvcodec.cpp@ 890

Last change on this file since 890 was 846, checked in by Dmitry A. Kuminov, 15 years ago

trunk: Merged in qt 4.7.2 sources from branches/vendor/nokia/qt.
File size: 17.9 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
4** All rights reserved.
5** Contact: Nokia Corporation ([email protected])
6**
7** This file is part of the QtCore module of the Qt Toolkit.
8**
9** $QT_BEGIN_LICENSE:LGPL$
10** Commercial Usage
11** Licensees holding valid Qt Commercial licenses may use this file in
12** accordance with the Qt Commercial License Agreement provided with the
13** Software or, alternatively, in accordance with the terms contained in
14** a written agreement between you and Nokia.
15**
16** GNU Lesser General Public License Usage
17** Alternatively, this file may be used under the terms of the GNU Lesser
18** General Public License version 2.1 as published by the Free Software
19** Foundation and appearing in the file LICENSE.LGPL included in the
20** packaging of this file. Please review the following information to
21** ensure the GNU Lesser General Public License version 2.1 requirements
22** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
23**
24** In addition, as a special exception, Nokia gives you certain additional
25** rights. These rights are described in the Nokia Qt LGPL Exception
26** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you have questions regarding the use of this file, please contact
37** Nokia at [email protected].
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42#include "qiconvcodec_p.h"
43#include "qtextcodec_p.h"
44#include <qlibrary.h>
45#include <qdebug.h>
46#include <qthreadstorage.h>
47
48#include <errno.h>
49#include <locale.h>
50#include <stdio.h>
51#include <dlfcn.h>
52
53// unistd.h is needed for the _XOPEN_UNIX macro
54#include <unistd.h>
55#if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)
56# include <langinfo.h>
57#endif
58
59#if defined(Q_OS_HPUX)
60# define NO_BOM
61# define UTF16 "ucs2"
62#elif defined(Q_OS_AIX)
63# define NO_BOM
64# define UTF16 "UCS-2"
65#elif defined(Q_OS_FREEBSD) || defined(Q_OS_MAC)
66# define NO_BOM
67# if Q_BYTE_ORDER == Q_BIG_ENDIAN
68# define UTF16 "UTF-16BE"
69# else
70# define UTF16 "UTF-16LE"
71# endif
72#else
73# define UTF16 "UTF-16"
74#endif
75
76#if defined(Q_OS_MAC)
77#ifndef GNU_LIBICONV
78#define GNU_LIBICONV
79#endif
80typedef iconv_t (*Ptr_iconv_open) (const char*, const char*);
81typedef size_t (*Ptr_iconv) (iconv_t, const char **, size_t *, char **, size_t *);
82typedef int (*Ptr_iconv_close) (iconv_t);
83
84static Ptr_iconv_open ptr_iconv_open = 0;
85static Ptr_iconv ptr_iconv = 0;
86static Ptr_iconv_close ptr_iconv_close = 0;
87#endif
88
89QT_BEGIN_NAMESPACE
90
91extern bool qt_locale_initialized;
92
93QIconvCodec::QIconvCodec()
94 : utf16Codec(0)
95{
96 utf16Codec = QTextCodec::codecForMib(1015);
97 Q_ASSERT_X(utf16Codec != 0,
98 "QIconvCodec::convertToUnicode",
99 "internal error, UTF-16 codec not found");
100 if (!utf16Codec) {
101 fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n");
102 utf16Codec = reinterpret_cast<QTextCodec *>(~0);
103 }
104#if defined(Q_OS_MAC)
105 if (ptr_iconv_open == 0) {
106 QLibrary libiconv(QLatin1String("/usr/lib/libiconv"));
107 libiconv.setLoadHints(QLibrary::ExportExternalSymbolsHint);
108
109 ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("libiconv_open"));
110 if (!ptr_iconv_open)
111 ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("iconv_open"));
112 ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("libiconv"));
113 if (!ptr_iconv)
114 ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("iconv"));
115 ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("libiconv_close"));
116 if (!ptr_iconv_close)
117 ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("iconv_close"));
118
119 Q_ASSERT_X(ptr_iconv_open && ptr_iconv && ptr_iconv_close,
120 "QIconvCodec::QIconvCodec()",
121 "internal error, could not resolve the iconv functions");
122
123# undef iconv_open
124# define iconv_open ptr_iconv_open
125# undef iconv
126# define iconv ptr_iconv
127# undef iconv_close
128# define iconv_close ptr_iconv_close
129 }
130#endif
131}
132
133QIconvCodec::~QIconvCodec()
134{
135}
136
137QIconvCodec::IconvState::IconvState(iconv_t x)
138 : buffer(array), bufferLen(sizeof array), cd(x)
139{
140}
141
142QIconvCodec::IconvState::~IconvState()
143{
144 if (cd != reinterpret_cast<iconv_t>(-1))
145 iconv_close(cd);
146 if (buffer != array)
147 delete[] buffer;
148}
149
150void QIconvCodec::IconvState::saveChars(const char *c, int count)
151{
152 if (count > bufferLen) {
153 if (buffer != array)
154 delete[] buffer;
155 buffer = new char[bufferLen = count];
156 }
157
158 memcpy(buffer, c, count);
159}
160
161static void qIconvCodecStateFree(QTextCodec::ConverterState *state)
162{
163 delete reinterpret_cast<QIconvCodec::IconvState *>(state->d);
164}
165
166Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, toUnicodeState)
167
168QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const
169{
170 if (utf16Codec == reinterpret_cast<QTextCodec *>(~0))
171 return QString::fromLatin1(chars, len);
172
173 int invalidCount = 0;
174 int remainingCount = 0;
175 char *remainingBuffer = 0;
176 IconvState **pstate;
177
178 if (convState) {
179 // stateful conversion
180 pstate = reinterpret_cast<IconvState **>(&convState->d);
181 if (convState->d) {
182 // restore state
183 remainingCount = convState->remainingChars;
184 remainingBuffer = (*pstate)->buffer;
185 } else {
186 // first time
187 convState->flags |= FreeFunction;
188 QTextCodecUnalignedPointer::encode(convState->state_data, qIconvCodecStateFree);
189 }
190 } else {
191 QThreadStorage<QIconvCodec::IconvState *> *ts = toUnicodeState();
192 if (!qt_locale_initialized || !ts) {
193 // we're running after the Q_GLOBAL_STATIC has been deleted
194 // or before the QCoreApplication initialization
195 // bad programmer, no cookie for you
196 return QString::fromLatin1(chars, len);
197 }
198
199 // stateless conversion -- use thread-local data
200 pstate = &toUnicodeState()->localData();
201 }
202
203 if (!*pstate) {
204 // first time, create the state
205 iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0);
206 if (cd == reinterpret_cast<iconv_t>(-1)) {
207 static int reported = 0;
208 if (!reported++) {
209 fprintf(stderr,
210 "QIconvCodec::convertToUnicode: using Latin-1 for conversion, iconv_open failed\n");
211 }
212 return QString::fromLatin1(chars, len);
213 }
214
215 *pstate = new IconvState(cd);
216 }
217
218 IconvState *state = *pstate;
219 size_t inBytesLeft = len;
220 // best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM
221#ifdef GNU_LIBICONV
222 // GNU doesn't disagree with POSIX :/
223 const char *inBytes = chars;
224#else
225 char *inBytes = const_cast<char *>(chars);
226#endif
227
228 QByteArray in;
229 if (remainingCount) {
230 // we have to prepend the remaining bytes from the previous conversion
231 inBytesLeft += remainingCount;
232 in.resize(inBytesLeft);
233 inBytes = in.data();
234
235 memcpy(in.data(), remainingBuffer, remainingCount);
236 memcpy(in.data() + remainingCount, chars, len);
237
238 remainingCount = 0;
239 }
240
241 size_t outBytesLeft = len * 2 + 2;
242 QByteArray ba(outBytesLeft, Qt::Uninitialized);
243 char *outBytes = ba.data();
244 do {
245 size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft);
246 if (ret == (size_t) -1) {
247 if (errno == E2BIG) {
248 int offset = ba.size() - outBytesLeft;
249 ba.resize(ba.size() * 2);
250 outBytes = ba.data() + offset;
251 outBytesLeft = ba.size() - offset;
252
253 continue;
254 }
255
256 if (errno == EILSEQ) {
257 // conversion stopped because of an invalid character in the sequence
258 ++invalidCount;
259 } else if (errno == EINVAL && convState) {
260 // conversion stopped because the remaining inBytesLeft make up
261 // an incomplete multi-byte sequence; save them for later
262 state->saveChars(inBytes, inBytesLeft);
263 remainingCount = inBytesLeft;
264 break;
265 }
266
267 if (errno == EILSEQ || errno == EINVAL) {
268 // skip the next character
269 ++inBytes;
270 --inBytesLeft;
271 continue;
272 }
273
274 // some other error
275 // note, cannot use qWarning() since we are implementing the codecForLocale :)
276 perror("QIconvCodec::convertToUnicode: using Latin-1 for conversion, iconv failed");
277
278 if (!convState) {
279 // reset state
280 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
281 }
282
283 return QString::fromLatin1(chars, len);
284 }
285 } while (inBytesLeft != 0);
286
287 QString s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft);
288
289 if (convState) {
290 convState->invalidChars = invalidCount;
291 convState->remainingChars = remainingCount;
292 } else {
293 // reset state
294 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
295 }
296
297 return s;
298}
299
300Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, fromUnicodeState)
301
302static bool setByteOrder(iconv_t cd)
303{
304#if !defined(NO_BOM)
305 // give iconv() a BOM
306 char buf[4];
307 ushort bom[] = { QChar::ByteOrderMark };
308
309 char *outBytes = buf;
310 char *inBytes = reinterpret_cast<char *>(bom);
311 size_t outBytesLeft = sizeof buf;
312 size_t inBytesLeft = sizeof bom;
313
314#if defined(GNU_LIBICONV)
315 const char **inBytesPtr = const_cast<const char **>(&inBytes);
316#else
317 char **inBytesPtr = &inBytes;
318#endif
319
320 if (iconv(cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
321 return false;
322 }
323#endif // NO_BOM
324
325 return true;
326}
327
328QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const
329{
330 char *inBytes;
331 char *outBytes;
332 size_t inBytesLeft;
333
334#if defined(GNU_LIBICONV)
335 const char **inBytesPtr = const_cast<const char **>(&inBytes);
336#else
337 char **inBytesPtr = &inBytes;
338#endif
339
340 QThreadStorage<QIconvCodec::IconvState *> *ts = fromUnicodeState();
341 if (!qt_locale_initialized || !ts) {
342 // we're running after the Q_GLOBAL_STATIC has been deleted
343 // or before the QCoreApplication initialization
344 // bad programmer, no cookie for you
345 if (!len)
346 // this is a special case - zero-sized string should be
347 // translated to empty but not-null QByteArray.
348 return QByteArray("");
349 return QString::fromRawData(uc, len).toLatin1();
350 }
351 IconvState *&state = ts->localData();
352 if (!state) {
353 state = new IconvState(QIconvCodec::createIconv_t(0, UTF16));
354 if (state->cd == reinterpret_cast<iconv_t>(-1)) {
355 if (!setByteOrder(state->cd)) {
356 perror("QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv failed for BOM");
357
358 iconv_close(state->cd);
359 state->cd = reinterpret_cast<iconv_t>(-1);
360
361 return QString(uc, len).toLatin1();
362 }
363 }
364 }
365 if (state->cd == reinterpret_cast<iconv_t>(-1)) {
366 static int reported = 0;
367 if (!reported++) {
368 fprintf(stderr,
369 "QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv_open failed\n");
370 }
371 return QString(uc, len).toLatin1();
372 }
373
374 size_t outBytesLeft = len;
375 QByteArray ba(outBytesLeft, Qt::Uninitialized);
376 outBytes = ba.data();
377
378 // now feed iconv() the real data
379 inBytes = const_cast<char *>(reinterpret_cast<const char *>(uc));
380 inBytesLeft = len * sizeof(QChar);
381
382 QByteArray in;
383 if (convState && convState->remainingChars) {
384 // we have one surrogate char to be prepended
385 in.resize(sizeof(QChar) + len);
386 inBytes = in.data();
387
388 QChar remaining = convState->state_data[0];
389 memcpy(in.data(), &remaining, sizeof(QChar));
390 memcpy(in.data() + sizeof(QChar), uc, inBytesLeft);
391
392 inBytesLeft += sizeof(QChar);
393 convState->remainingChars = 0;
394 }
395
396 int invalidCount = 0;
397 while (inBytesLeft != 0) {
398 if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
399 if (errno == EINVAL && convState) {
400 // buffer ends in a surrogate
401 Q_ASSERT(inBytesLeft == 2);
402 convState->remainingChars = 1;
403 convState->state_data[0] = uc[len - 1].unicode();
404 break;
405 }
406
407 switch (errno) {
408 case EILSEQ:
409 ++invalidCount;
410 // fall through
411 case EINVAL:
412 {
413 inBytes += sizeof(QChar);
414 inBytesLeft -= sizeof(QChar);
415 break;
416 }
417 case E2BIG:
418 {
419 int offset = ba.size() - outBytesLeft;
420 ba.resize(ba.size() * 2);
421 outBytes = ba.data() + offset;
422 outBytesLeft = ba.size() - offset;
423 break;
424 }
425 default:
426 {
427 // note, cannot use qWarning() since we are implementing the codecForLocale :)
428 perror("QIconvCodec::convertFromUnicode: using Latin-1 for conversion, iconv failed");
429
430 // reset to initial state
431 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
432
433 return QString(uc, len).toLatin1();
434 }
435 }
436 }
437 }
438
439 // reset to initial state
440 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
441 setByteOrder(state->cd);
442
443 ba.resize(ba.size() - outBytesLeft);
444
445 if (convState)
446 convState->invalidChars = invalidCount;
447
448 return ba;
449}
450
451QByteArray QIconvCodec::name() const
452{
453 return "System";
454}
455
456int QIconvCodec::mibEnum() const
457{
458 return 0;
459}
460
461iconv_t QIconvCodec::createIconv_t(const char *to, const char *from)
462{
463 Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0));
464
465 iconv_t cd = (iconv_t) -1;
466#if defined(__GLIBC__) || defined(GNU_LIBICONV)
467 // both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string
468 static const char empty_codeset[] = "";
469 const char *codeset = empty_codeset;
470 cd = iconv_open(to ? to : codeset, from ? from : codeset);
471#else
472 char *codeset = 0;
473#endif
474
475#if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)
476 if (cd == (iconv_t) -1) {
477 codeset = nl_langinfo(CODESET);
478 if (codeset)
479 cd = iconv_open(to ? to : codeset, from ? from : codeset);
480 }
481#endif
482
483 if (cd == (iconv_t) -1) {
484 // Very poorly defined and followed standards causes lots of
485 // code to try to get all the cases... This logic is
486 // duplicated in QTextCodec, so if you change it here, change
487 // it there too.
488
489 // Try to determine locale codeset from locale name assigned to
490 // LC_CTYPE category.
491
492 // First part is getting that locale name. First try setlocale() which
493 // definitely knows it, but since we cannot fully trust it, get ready
494 // to fall back to environment variables.
495 char * ctype = qstrdup(setlocale(LC_CTYPE, 0));
496
497 // Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG
498 // environment variables.
499 char * lang = qstrdup(qgetenv("LC_ALL").constData());
500 if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
501 if (lang) delete [] lang;
502 lang = qstrdup(qgetenv("LC_CTYPE").constData());
503 }
504 if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
505 if (lang) delete [] lang;
506 lang = qstrdup(qgetenv("LANG").constData());
507 }
508
509 // Now try these in order:
510 // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
511 // 2. CODESET from lang if it contains a .CODESET part
512 // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
513 // 4. locale (ditto)
514 // 5. check for "@euro"
515
516 // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
517 codeset = ctype ? strchr(ctype, '.') : 0;
518 if (codeset && *codeset == '.') {
519 ++codeset;
520 cd = iconv_open(to ? to : codeset, from ? from : codeset);
521 }
522
523 // 2. CODESET from lang if it contains a .CODESET part
524 codeset = lang ? strchr(lang, '.') : 0;
525 if (cd == (iconv_t) -1 && codeset && *codeset == '.') {
526 ++codeset;
527 cd = iconv_open(to ? to : codeset, from ? from : codeset);
528 }
529
530 // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
531 if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0)
532 cd = iconv_open(to ? to : ctype, from ? from : ctype);
533
534
535 // 4. locale (ditto)
536 if (cd == (iconv_t) -1 && lang && *lang != 0)
537 cd = iconv_open(to ? to : lang, from ? from : lang);
538
539 // 5. "@euro"
540 if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro")))
541 cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15");
542
543 delete [] ctype;
544 delete [] lang;
545 }
546
547 return cd;
548}
549
550QT_END_NAMESPACE
Note: See TracBrowser for help on using the repository browser.