source: trunk/src/corelib/codecs/qiconvcodec.cpp@ 551

Last change on this file since 551 was 2, checked in by Dmitry A. Kuminov, 16 years ago

Initially imported qt-all-opensource-src-4.5.1 from Trolltech.
File size: 17.7 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
4** Contact: Qt Software Information ([email protected])
5**
6** This file is part of the QtCore module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial Usage
10** Licensees holding valid Qt Commercial licenses may use this file in
11** accordance with the Qt Commercial License Agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and Nokia.
14**
15** GNU Lesser General Public License Usage
16** Alternatively, this file may be used under the terms of the GNU Lesser
17** General Public License version 2.1 as published by the Free Software
18** Foundation and appearing in the file LICENSE.LGPL included in the
19** packaging of this file. Please review the following information to
20** ensure the GNU Lesser General Public License version 2.1 requirements
21** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
22**
23** In addition, as a special exception, Nokia gives you certain
24** additional rights. These rights are described in the Nokia Qt LGPL
25** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
26** package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you are unsure which license is appropriate for your use, please
37** contact the sales department at [email protected].
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42#include "qiconvcodec_p.h"
43#include "qtextcodec_p.h"
44#include <qlibrary.h>
45#include <qdebug.h>
46#include <qthreadstorage.h>
47
48#include <errno.h>
49#include <locale.h>
50#include <stdio.h>
51#include <dlfcn.h>
52
53// unistd.h is needed for the _XOPEN_UNIX macro
54#include <unistd.h>
55#if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX6) && !defined(Q_OS_OSF)
56# include <langinfo.h>
57#endif
58
59#if defined(Q_OS_HPUX)
60# define NO_BOM
61# define UTF16 "ucs2"
62#elif defined(Q_OS_AIX)
63# define NO_BOM
64# define UTF16 "UCS-2"
65#elif defined(Q_OS_MAC)
66# define NO_BOM
67# if Q_BYTE_ORDER == Q_BIG_ENDIAN
68# define UTF16 "UTF-16BE"
69# else
70# define UTF16 "UTF-16LE"
71# endif
72#else
73# define UTF16 "UTF-16"
74#endif
75
76#if defined(Q_OS_MAC)
77#ifndef GNU_LIBICONV
78#define GNU_LIBICONV
79#endif
80typedef iconv_t (*Ptr_iconv_open) (const char*, const char*);
81typedef size_t (*Ptr_iconv) (iconv_t, const char **, size_t *, char **, size_t *);
82typedef int (*Ptr_iconv_close) (iconv_t);
83
84static Ptr_iconv_open ptr_iconv_open = 0;
85static Ptr_iconv ptr_iconv = 0;
86static Ptr_iconv_close ptr_iconv_close = 0;
87#endif
88
89QT_BEGIN_NAMESPACE
90
91extern bool qt_locale_initialized;
92
93QIconvCodec::QIconvCodec()
94 : utf16Codec(0)
95{
96 utf16Codec = QTextCodec::codecForMib(1015);
97 Q_ASSERT_X(utf16Codec != 0,
98 "QIconvCodec::convertToUnicode",
99 "internal error, UTF-16 codec not found");
100 if (!utf16Codec) {
101 fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n");
102 utf16Codec = reinterpret_cast<QTextCodec *>(~0);
103 }
104#if defined(Q_OS_MAC)
105 if (ptr_iconv_open == 0) {
106 QLibrary libiconv(QLatin1String("/usr/lib/libiconv"));
107 libiconv.setLoadHints(QLibrary::ExportExternalSymbolsHint);
108
109 ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("libiconv_open"));
110 if (!ptr_iconv_open)
111 ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("iconv_open"));
112 ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("libiconv"));
113 if (!ptr_iconv)
114 ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("iconv"));
115 ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("libiconv_close"));
116 if (!ptr_iconv_close)
117 ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("iconv_close"));
118
119 Q_ASSERT_X(ptr_iconv_open && ptr_iconv && ptr_iconv_close,
120 "QIconvCodec::QIconvCodec()",
121 "internal error, could not resolve the iconv functions");
122
123# undef iconv_open
124# define iconv_open ptr_iconv_open
125# undef iconv
126# define iconv ptr_iconv
127# undef iconv_close
128# define iconv_close ptr_iconv_close
129 }
130#endif
131}
132
133QIconvCodec::~QIconvCodec()
134{
135}
136
137QIconvCodec::IconvState::IconvState(iconv_t x)
138 : buffer(array), bufferLen(sizeof array), cd(x)
139{
140}
141
142QIconvCodec::IconvState::~IconvState()
143{
144 if (cd != reinterpret_cast<iconv_t>(-1))
145 iconv_close(cd);
146 if (buffer != array)
147 delete[] buffer;
148}
149
150void QIconvCodec::IconvState::saveChars(const char *c, int count)
151{
152 if (count > bufferLen) {
153 if (buffer != array)
154 delete[] buffer;
155 buffer = new char[bufferLen = count];
156 }
157
158 memcpy(buffer, c, count);
159}
160
161static void qIconvCodecStateFree(QTextCodec::ConverterState *state)
162{
163 delete reinterpret_cast<QIconvCodec::IconvState *>(state->d);
164}
165
166Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, toUnicodeState)
167
168QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const
169{
170 if (utf16Codec == reinterpret_cast<QTextCodec *>(~0))
171 return QString::fromAscii(chars, len);
172
173 int invalidCount = 0;
174 int remainingCount = 0;
175 char *remainingBuffer = 0;
176 IconvState **pstate;
177
178 if (convState) {
179 // stateful conversion
180 pstate = reinterpret_cast<IconvState **>(&convState->d);
181 if (convState->d) {
182 // restore state
183 remainingCount = convState->remainingChars;
184 remainingBuffer = (*pstate)->buffer;
185 } else {
186 // first time
187 convState->flags |= FreeFunction;
188 QTextCodecUnalignedPointer::encode(convState->state_data, qIconvCodecStateFree);
189 }
190 } else {
191 QThreadStorage<QIconvCodec::IconvState *> *ts = toUnicodeState();
192 if (!qt_locale_initialized || !ts) {
193 // we're running after the Q_GLOBAL_STATIC has been deleted
194 // or before the QCoreApplication initialization
195 // bad programmer, no cookie for you
196 return QString::fromLatin1(chars, len);
197 }
198
199 // stateless conversion -- use thread-local data
200 pstate = &toUnicodeState()->localData();
201 }
202
203 if (!*pstate) {
204 // first time, create the state
205 iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0);
206 if (cd == reinterpret_cast<iconv_t>(-1)) {
207 static int reported = 0;
208 if (!reported++) {
209 fprintf(stderr,
210 "QIconvCodec::convertToUnicode: using ASCII for conversion, iconv_open failed\n");
211 }
212 return QString::fromAscii(chars, len);
213 }
214
215 *pstate = new IconvState(cd);
216 }
217
218 IconvState *state = *pstate;
219 size_t inBytesLeft = len;
220 // best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM
221#ifdef GNU_LIBICONV
222 // GNU doesn't disagree with POSIX :/
223 const char *inBytes = chars;
224#else
225 char *inBytes = const_cast<char *>(chars);
226#endif
227
228 QByteArray in;
229 if (remainingCount) {
230 // we have to prepend the remaining bytes from the previous conversion
231 inBytesLeft += remainingCount;
232 in.resize(inBytesLeft);
233 inBytes = in.data();
234
235 memcpy(in.data(), remainingBuffer, remainingCount);
236 memcpy(in.data() + remainingCount, chars, len);
237
238 remainingCount = 0;
239 }
240
241 QByteArray ba;
242 size_t outBytesLeft = len * 2 + 2;
243 ba.resize(outBytesLeft);
244 char *outBytes = ba.data();
245 do {
246 size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft);
247 if (ret == (size_t) -1) {
248 if (errno == E2BIG) {
249 int offset = ba.size() - outBytesLeft;
250 ba.resize(ba.size() * 2);
251 outBytes = ba.data() + offset;
252 outBytesLeft = ba.size() - offset;
253
254 continue;
255 }
256
257 if (errno == EILSEQ) {
258 // conversion stopped because of an invalid character in the sequence
259 ++invalidCount;
260 } else if (errno == EINVAL && convState) {
261 // conversion stopped because the remaining inBytesLeft make up
262 // an incomplete multi-byte sequence; save them for later
263 state->saveChars(inBytes, inBytesLeft);
264 remainingCount = inBytesLeft;
265 break;
266 }
267
268 if (errno == EILSEQ || errno == EINVAL) {
269 // skip the next character
270 ++inBytes;
271 --inBytesLeft;
272 continue;
273 }
274
275 // some other error
276 // note, cannot use qWarning() since we are implementing the codecForLocale :)
277 perror("QIconvCodec::convertToUnicode: using ASCII for conversion, iconv failed");
278
279 if (!convState) {
280 // reset state
281 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
282 }
283
284 return QString::fromAscii(chars, len);
285 }
286 } while (inBytesLeft != 0);
287
288 QString s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft);
289
290 if (convState) {
291 convState->invalidChars = invalidCount;
292 convState->remainingChars = remainingCount;
293 } else {
294 // reset state
295 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
296 }
297
298 return s;
299}
300
301Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, fromUnicodeState)
302
303QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const
304{
305 char *inBytes;
306 char *outBytes;
307 size_t inBytesLeft;
308
309#if defined(GNU_LIBICONV)
310 const char **inBytesPtr = const_cast<const char **>(&inBytes);
311#else
312 char **inBytesPtr = &inBytes;
313#endif
314
315 QThreadStorage<QIconvCodec::IconvState *> *ts = fromUnicodeState();
316 if (!qt_locale_initialized || !ts) {
317 // we're running after the Q_GLOBAL_STATIC has been deleted
318 // or before the QCoreApplication initialization
319 // bad programmer, no cookie for you
320 if (!len)
321 // this is a special case - zero-sized string should be
322 // translated to empty but not-null QByteArray.
323 return QByteArray("");
324 return QString::fromRawData(uc, len).toLatin1();
325 }
326 IconvState *&state = ts->localData();
327 if (!state) {
328 state = new IconvState(QIconvCodec::createIconv_t(0, UTF16));
329 if (state->cd != reinterpret_cast<iconv_t>(-1)) {
330 size_t outBytesLeft = len + 3; // +3 for the BOM
331 QByteArray ba;
332 ba.resize(outBytesLeft);
333 outBytes = ba.data();
334
335#if !defined(NO_BOM)
336 // give iconv() a BOM
337 QChar bom[] = { QChar(QChar::ByteOrderMark) };
338 inBytes = reinterpret_cast<char *>(bom);
339 inBytesLeft = sizeof(bom);
340 if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
341 perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed for BOM");
342
343 iconv_close(state->cd);
344 state->cd = reinterpret_cast<iconv_t>(-1);
345
346 return QString(uc, len).toAscii();
347 }
348#endif // NO_BOM
349 }
350 }
351 if (state->cd == reinterpret_cast<iconv_t>(-1)) {
352 static int reported = 0;
353 if (!reported++) {
354 fprintf(stderr,
355 "QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv_open failed\n");
356 }
357 return QString(uc, len).toAscii();
358 }
359
360 size_t outBytesLeft = len;
361 QByteArray ba;
362 ba.resize(outBytesLeft);
363 outBytes = ba.data();
364
365 // now feed iconv() the real data
366 inBytes = const_cast<char *>(reinterpret_cast<const char *>(uc));
367 inBytesLeft = len * sizeof(QChar);
368
369 QByteArray in;
370 if (convState && convState->remainingChars) {
371 // we have one surrogate char to be prepended
372 in.resize(sizeof(QChar) + len);
373 inBytes = in.data();
374
375 QChar remaining = convState->state_data[0];
376 memcpy(in.data(), &remaining, sizeof(QChar));
377 memcpy(in.data() + sizeof(QChar), uc, inBytesLeft);
378
379 inBytesLeft += sizeof(QChar);
380 convState->remainingChars = 0;
381 }
382
383 int invalidCount = 0;
384 do {
385 if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
386 if (errno == EINVAL && convState) {
387 // buffer ends in a surrogate
388 Q_ASSERT(inBytesLeft == 2);
389 convState->remainingChars = 1;
390 convState->state_data[0] = uc[len - 1].unicode();
391 break;
392 }
393
394 switch (errno) {
395 case EILSEQ:
396 ++invalidCount;
397 // fall through
398 case EINVAL:
399 {
400 inBytes += sizeof(QChar);
401 inBytesLeft -= sizeof(QChar);
402 break;
403 }
404 case E2BIG:
405 {
406 int offset = ba.size() - outBytesLeft;
407 ba.resize(ba.size() * 2);
408 outBytes = ba.data() + offset;
409 outBytesLeft = ba.size() - offset;
410 break;
411 }
412 default:
413 {
414 // note, cannot use qWarning() since we are implementing the codecForLocale :)
415 perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed");
416
417 // reset to initial state
418 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
419
420 return QString(uc, len).toAscii();
421 }
422 }
423 }
424 } while (inBytesLeft != 0);
425
426 // reset to initial state
427 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
428
429 ba.resize(ba.size() - outBytesLeft);
430
431 if (convState)
432 convState->invalidChars = invalidCount;
433
434 return ba;
435}
436
437QByteArray QIconvCodec::name() const
438{
439 return "System";
440}
441
442int QIconvCodec::mibEnum() const
443{
444 return 0;
445}
446
447iconv_t QIconvCodec::createIconv_t(const char *to, const char *from)
448{
449 Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0));
450
451 iconv_t cd = (iconv_t) -1;
452#if defined(__GLIBC__) || defined(GNU_LIBICONV)
453 // both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string
454 static const char empty_codeset[] = "";
455 const char *codeset = empty_codeset;
456 cd = iconv_open(to ? to : codeset, from ? from : codeset);
457#else
458 char *codeset = 0;
459#endif
460
461#if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX6) && !defined(Q_OS_OSF)
462 if (cd == (iconv_t) -1) {
463 codeset = nl_langinfo(CODESET);
464 if (codeset)
465 cd = iconv_open(to ? to : codeset, from ? from : codeset);
466 }
467#endif
468
469 if (cd == (iconv_t) -1) {
470 // Very poorly defined and followed standards causes lots of
471 // code to try to get all the cases... This logic is
472 // duplicated in QTextCodec, so if you change it here, change
473 // it there too.
474
475 // Try to determine locale codeset from locale name assigned to
476 // LC_CTYPE category.
477
478 // First part is getting that locale name. First try setlocale() which
479 // definitely knows it, but since we cannot fully trust it, get ready
480 // to fall back to environment variables.
481 char * ctype = qstrdup(setlocale(LC_CTYPE, 0));
482
483 // Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG
484 // environment variables.
485 char * lang = qstrdup(qgetenv("LC_ALL").constData());
486 if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
487 if (lang) delete [] lang;
488 lang = qstrdup(qgetenv("LC_CTYPE").constData());
489 }
490 if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
491 if (lang) delete [] lang;
492 lang = qstrdup(qgetenv("LANG").constData());
493 }
494
495 // Now try these in order:
496 // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
497 // 2. CODESET from lang if it contains a .CODESET part
498 // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
499 // 4. locale (ditto)
500 // 5. check for "@euro"
501
502 // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
503 codeset = ctype ? strchr(ctype, '.') : 0;
504 if (codeset && *codeset == '.') {
505 ++codeset;
506 cd = iconv_open(to ? to : codeset, from ? from : codeset);
507 }
508
509 // 2. CODESET from lang if it contains a .CODESET part
510 codeset = lang ? strchr(lang, '.') : 0;
511 if (cd == (iconv_t) -1 && codeset && *codeset == '.') {
512 ++codeset;
513 cd = iconv_open(to ? to : codeset, from ? from : codeset);
514 }
515
516 // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
517 if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0)
518 cd = iconv_open(to ? to : ctype, from ? from : ctype);
519
520
521 // 4. locale (ditto)
522 if (cd == (iconv_t) -1 && lang && *lang != 0)
523 cd = iconv_open(to ? to : lang, from ? from : lang);
524
525 // 5. "@euro"
526 if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro")))
527 cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15");
528
529 delete [] ctype;
530 delete [] lang;
531 }
532
533 return cd;
534}
535
536QT_END_NAMESPACE
Note: See TracBrowser for help on using the repository browser.