source: vendor/python/2.5/Modules/cjkcodecs/multibytecodec.c@ 3609

Last change on this file since 3609 was 3225, checked in by bird, 19 years ago

Python 2.5
File size: 43.3 KB
Line 
1/*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <[email protected]>
5 */
6
7#define PY_SSIZE_T_CLEAN
8#include "Python.h"
9#include "structmember.h"
10#include "multibytecodec.h"
11
12typedef struct {
13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;
14 unsigned char *outbuf, *outbuf_end;
15 PyObject *excobj, *outobj;
16} MultibyteEncodeBuffer;
17
18typedef struct {
19 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
20 Py_UNICODE *outbuf, *outbuf_end;
21 PyObject *excobj, *outobj;
22} MultibyteDecodeBuffer;
23
24PyDoc_STRVAR(MultibyteCodec_Encode__doc__,
25"I.encode(unicode[, errors]) -> (string, length consumed)\n\
26\n\
27Return an encoded string version of `unicode'. errors may be given to\n\
28set a different error handling scheme. Default is 'strict' meaning that\n\
29encoding errors raise a UnicodeEncodeError. Other possible values are\n\
30'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\
31registered with codecs.register_error that can handle UnicodeEncodeErrors.");
32
33PyDoc_STRVAR(MultibyteCodec_Decode__doc__,
34"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\
35\n\
36Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\
37to set a different error handling scheme. Default is 'strict' meaning\n\
38that encoding errors raise a UnicodeDecodeError. Other possible values\n\
39are 'ignore' and 'replace' as well as any other name registerd with\n\
40codecs.register_error that is able to handle UnicodeDecodeErrors.");
41
42static char *codeckwarglist[] = {"input", "errors", NULL};
43static char *incnewkwarglist[] = {"errors", NULL};
44static char *incrementalkwarglist[] = {"input", "final", NULL};
45static char *streamkwarglist[] = {"stream", "errors", NULL};
46
47static PyObject *multibytecodec_encode(MultibyteCodec *,
48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,
49 PyObject *, int);
50
51#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
52
53static PyObject *
54make_tuple(PyObject *object, Py_ssize_t len)
55{
56 PyObject *v, *w;
57
58 if (object == NULL)
59 return NULL;
60
61 v = PyTuple_New(2);
62 if (v == NULL) {
63 Py_DECREF(object);
64 return NULL;
65 }
66 PyTuple_SET_ITEM(v, 0, object);
67
68 w = PyInt_FromSsize_t(len);
69 if (w == NULL) {
70 Py_DECREF(v);
71 return NULL;
72 }
73 PyTuple_SET_ITEM(v, 1, w);
74
75 return v;
76}
77
78static PyObject *
79internal_error_callback(const char *errors)
80{
81 if (errors == NULL || strcmp(errors, "strict") == 0)
82 return ERROR_STRICT;
83 else if (strcmp(errors, "ignore") == 0)
84 return ERROR_IGNORE;
85 else if (strcmp(errors, "replace") == 0)
86 return ERROR_REPLACE;
87 else
88 return PyString_FromString(errors);
89}
90
91static PyObject *
92call_error_callback(PyObject *errors, PyObject *exc)
93{
94 PyObject *args, *cb, *r;
95
96 assert(PyString_Check(errors));
97 cb = PyCodec_LookupError(PyString_AS_STRING(errors));
98 if (cb == NULL)
99 return NULL;
100
101 args = PyTuple_New(1);
102 if (args == NULL) {
103 Py_DECREF(cb);
104 return NULL;
105 }
106
107 PyTuple_SET_ITEM(args, 0, exc);
108 Py_INCREF(exc);
109
110 r = PyObject_CallObject(cb, args);
111 Py_DECREF(args);
112 Py_DECREF(cb);
113 return r;
114}
115
116static PyObject *
117codecctx_errors_get(MultibyteStatefulCodecContext *self)
118{
119 const char *errors;
120
121 if (self->errors == ERROR_STRICT)
122 errors = "strict";
123 else if (self->errors == ERROR_IGNORE)
124 errors = "ignore";
125 else if (self->errors == ERROR_REPLACE)
126 errors = "replace";
127 else {
128 Py_INCREF(self->errors);
129 return self->errors;
130 }
131
132 return PyString_FromString(errors);
133}
134
135static int
136codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,
137 void *closure)
138{
139 PyObject *cb;
140
141 if (!PyString_Check(value)) {
142 PyErr_SetString(PyExc_TypeError, "errors must be a string");
143 return -1;
144 }
145
146 cb = internal_error_callback(PyString_AS_STRING(value));
147 if (cb == NULL)
148 return -1;
149
150 ERROR_DECREF(self->errors);
151 self->errors = cb;
152 return 0;
153}
154
155/* This getset handlers list is used by all the stateful codec objects */
156static PyGetSetDef codecctx_getsets[] = {
157 {"errors", (getter)codecctx_errors_get,
158 (setter)codecctx_errors_set,
159 PyDoc_STR("how to treat errors")},
160 {NULL,}
161};
162
163static int
164expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
165{
166 Py_ssize_t orgpos, orgsize;
167
168 orgpos = (Py_ssize_t)((char *)buf->outbuf -
169 PyString_AS_STRING(buf->outobj));
170 orgsize = PyString_GET_SIZE(buf->outobj);
171 if (_PyString_Resize(&buf->outobj, orgsize + (
172 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
173 return -1;
174
175 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;
176 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)
177 + PyString_GET_SIZE(buf->outobj);
178
179 return 0;
180}
181#define REQUIRE_ENCODEBUFFER(buf, s) { \
182 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
183 if (expand_encodebuffer(buf, s) == -1) \
184 goto errorexit; \
185}
186
187static int
188expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)
189{
190 Py_ssize_t orgpos, orgsize;
191
192 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));
193 orgsize = PyUnicode_GET_SIZE(buf->outobj);
194 if (PyUnicode_Resize(&buf->outobj, orgsize + (
195 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
196 return -1;
197
198 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;
199 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)
200 + PyUnicode_GET_SIZE(buf->outobj);
201
202 return 0;
203}
204#define REQUIRE_DECODEBUFFER(buf, s) { \
205 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
206 if (expand_decodebuffer(buf, s) == -1) \
207 goto errorexit; \
208}
209
210
211/**
212 * MultibyteCodec object
213 */
214
215static int
216multibytecodec_encerror(MultibyteCodec *codec,
217 MultibyteCodec_State *state,
218 MultibyteEncodeBuffer *buf,
219 PyObject *errors, Py_ssize_t e)
220{
221 PyObject *retobj = NULL, *retstr = NULL, *tobj;
222 Py_ssize_t retstrsize, newpos;
223 Py_ssize_t esize, start, end;
224 const char *reason;
225
226 if (e > 0) {
227 reason = "illegal multibyte sequence";
228 esize = e;
229 }
230 else {
231 switch (e) {
232 case MBERR_TOOSMALL:
233 REQUIRE_ENCODEBUFFER(buf, -1);
234 return 0; /* retry it */
235 case MBERR_TOOFEW:
236 reason = "incomplete multibyte sequence";
237 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
238 break;
239 case MBERR_INTERNAL:
240 PyErr_SetString(PyExc_RuntimeError,
241 "internal codec error");
242 return -1;
243 default:
244 PyErr_SetString(PyExc_RuntimeError,
245 "unknown runtime error");
246 return -1;
247 }
248 }
249
250 if (errors == ERROR_REPLACE) {
251 const Py_UNICODE replchar = '?', *inbuf = &replchar;
252 Py_ssize_t r;
253
254 for (;;) {
255 Py_ssize_t outleft;
256
257 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
258 r = codec->encode(state, codec->config, &inbuf, 1,
259 &buf->outbuf, outleft, 0);
260 if (r == MBERR_TOOSMALL) {
261 REQUIRE_ENCODEBUFFER(buf, -1);
262 continue;
263 }
264 else
265 break;
266 }
267
268 if (r != 0) {
269 REQUIRE_ENCODEBUFFER(buf, 1);
270 *buf->outbuf++ = '?';
271 }
272 }
273 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
274 buf->inbuf += esize;
275 return 0;
276 }
277
278 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
279 end = start + esize;
280
281 /* use cached exception object if available */
282 if (buf->excobj == NULL) {
283 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,
284 buf->inbuf_top,
285 buf->inbuf_end - buf->inbuf_top,
286 start, end, reason);
287 if (buf->excobj == NULL)
288 goto errorexit;
289 }
290 else
291 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
292 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
293 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
294 goto errorexit;
295
296 if (errors == ERROR_STRICT) {
297 PyCodec_StrictErrors(buf->excobj);
298 goto errorexit;
299 }
300
301 retobj = call_error_callback(errors, buf->excobj);
302 if (retobj == NULL)
303 goto errorexit;
304
305 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
306 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||
307 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
308 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
309 PyErr_SetString(PyExc_TypeError,
310 "encoding error handler must return "
311 "(unicode, int) tuple");
312 goto errorexit;
313 }
314
315 {
316 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);
317
318 retstr = multibytecodec_encode(codec, state, &uraw,
319 PyUnicode_GET_SIZE(tobj), ERROR_STRICT,
320 MBENC_FLUSH);
321 if (retstr == NULL)
322 goto errorexit;
323 }
324
325 retstrsize = PyString_GET_SIZE(retstr);
326 REQUIRE_ENCODEBUFFER(buf, retstrsize);
327
328 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
329 buf->outbuf += retstrsize;
330
331 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
332 if (newpos < 0 && !PyErr_Occurred())
333 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
334 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
335 PyErr_Clear();
336 PyErr_Format(PyExc_IndexError,
337 "position %zd from error handler out of bounds",
338 newpos);
339 goto errorexit;
340 }
341 buf->inbuf = buf->inbuf_top + newpos;
342
343 Py_DECREF(retobj);
344 Py_DECREF(retstr);
345 return 0;
346
347errorexit:
348 Py_XDECREF(retobj);
349 Py_XDECREF(retstr);
350 return -1;
351}
352
353static int
354multibytecodec_decerror(MultibyteCodec *codec,
355 MultibyteCodec_State *state,
356 MultibyteDecodeBuffer *buf,
357 PyObject *errors, Py_ssize_t e)
358{
359 PyObject *retobj = NULL, *retuni = NULL;
360 Py_ssize_t retunisize, newpos;
361 const char *reason;
362 Py_ssize_t esize, start, end;
363
364 if (e > 0) {
365 reason = "illegal multibyte sequence";
366 esize = e;
367 }
368 else {
369 switch (e) {
370 case MBERR_TOOSMALL:
371 REQUIRE_DECODEBUFFER(buf, -1);
372 return 0; /* retry it */
373 case MBERR_TOOFEW:
374 reason = "incomplete multibyte sequence";
375 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
376 break;
377 case MBERR_INTERNAL:
378 PyErr_SetString(PyExc_RuntimeError,
379 "internal codec error");
380 return -1;
381 default:
382 PyErr_SetString(PyExc_RuntimeError,
383 "unknown runtime error");
384 return -1;
385 }
386 }
387
388 if (errors == ERROR_REPLACE) {
389 REQUIRE_DECODEBUFFER(buf, 1);
390 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;
391 }
392 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
393 buf->inbuf += esize;
394 return 0;
395 }
396
397 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
398 end = start + esize;
399
400 /* use cached exception object if available */
401 if (buf->excobj == NULL) {
402 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
403 (const char *)buf->inbuf_top,
404 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
405 start, end, reason);
406 if (buf->excobj == NULL)
407 goto errorexit;
408 }
409 else
410 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
411 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
412 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
413 goto errorexit;
414
415 if (errors == ERROR_STRICT) {
416 PyCodec_StrictErrors(buf->excobj);
417 goto errorexit;
418 }
419
420 retobj = call_error_callback(errors, buf->excobj);
421 if (retobj == NULL)
422 goto errorexit;
423
424 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
425 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
426 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
427 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
428 PyErr_SetString(PyExc_TypeError,
429 "decoding error handler must return "
430 "(unicode, int) tuple");
431 goto errorexit;
432 }
433
434 retunisize = PyUnicode_GET_SIZE(retuni);
435 if (retunisize > 0) {
436 REQUIRE_DECODEBUFFER(buf, retunisize);
437 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),
438 retunisize * Py_UNICODE_SIZE);
439 buf->outbuf += retunisize;
440 }
441
442 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
443 if (newpos < 0 && !PyErr_Occurred())
444 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
445 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
446 PyErr_Clear();
447 PyErr_Format(PyExc_IndexError,
448 "position %zd from error handler out of bounds",
449 newpos);
450 goto errorexit;
451 }
452 buf->inbuf = buf->inbuf_top + newpos;
453 Py_DECREF(retobj);
454 return 0;
455
456errorexit:
457 Py_XDECREF(retobj);
458 return -1;
459}
460
461static PyObject *
462multibytecodec_encode(MultibyteCodec *codec,
463 MultibyteCodec_State *state,
464 const Py_UNICODE **data, Py_ssize_t datalen,
465 PyObject *errors, int flags)
466{
467 MultibyteEncodeBuffer buf;
468 Py_ssize_t finalsize, r = 0;
469
470 if (datalen == 0)
471 return PyString_FromString("");
472
473 buf.excobj = NULL;
474 buf.inbuf = buf.inbuf_top = *data;
475 buf.inbuf_end = buf.inbuf_top + datalen;
476 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);
477 if (buf.outobj == NULL)
478 goto errorexit;
479 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);
480 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);
481
482 while (buf.inbuf < buf.inbuf_end) {
483 Py_ssize_t inleft, outleft;
484
485 /* we don't reuse inleft and outleft here.
486 * error callbacks can relocate the cursor anywhere on buffer*/
487 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
488 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
489 r = codec->encode(state, codec->config, &buf.inbuf, inleft,
490 &buf.outbuf, outleft, flags);
491 *data = buf.inbuf;
492 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
493 break;
494 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
495 goto errorexit;
496 else if (r == MBERR_TOOFEW)
497 break;
498 }
499
500 if (codec->encreset != NULL)
501 for (;;) {
502 Py_ssize_t outleft;
503
504 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
505 r = codec->encreset(state, codec->config, &buf.outbuf,
506 outleft);
507 if (r == 0)
508 break;
509 else if (multibytecodec_encerror(codec, state,
510 &buf, errors, r))
511 goto errorexit;
512 }
513
514 finalsize = (Py_ssize_t)((char *)buf.outbuf -
515 PyString_AS_STRING(buf.outobj));
516
517 if (finalsize != PyString_GET_SIZE(buf.outobj))
518 if (_PyString_Resize(&buf.outobj, finalsize) == -1)
519 goto errorexit;
520
521 Py_XDECREF(buf.excobj);
522 return buf.outobj;
523
524errorexit:
525 Py_XDECREF(buf.excobj);
526 Py_XDECREF(buf.outobj);
527 return NULL;
528}
529
530static PyObject *
531MultibyteCodec_Encode(MultibyteCodecObject *self,
532 PyObject *args, PyObject *kwargs)
533{
534 MultibyteCodec_State state;
535 Py_UNICODE *data;
536 PyObject *errorcb, *r, *arg, *ucvt;
537 const char *errors = NULL;
538 Py_ssize_t datalen;
539
540 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",
541 codeckwarglist, &arg, &errors))
542 return NULL;
543
544 if (PyUnicode_Check(arg))
545 ucvt = NULL;
546 else {
547 arg = ucvt = PyObject_Unicode(arg);
548 if (arg == NULL)
549 return NULL;
550 else if (!PyUnicode_Check(arg)) {
551 PyErr_SetString(PyExc_TypeError,
552 "couldn't convert the object to unicode.");
553 Py_DECREF(ucvt);
554 return NULL;
555 }
556 }
557
558 data = PyUnicode_AS_UNICODE(arg);
559 datalen = PyUnicode_GET_SIZE(arg);
560
561 errorcb = internal_error_callback(errors);
562 if (errorcb == NULL) {
563 Py_XDECREF(ucvt);
564 return NULL;
565 }
566
567 if (self->codec->encinit != NULL &&
568 self->codec->encinit(&state, self->codec->config) != 0)
569 goto errorexit;
570 r = multibytecodec_encode(self->codec, &state,
571 (const Py_UNICODE **)&data, datalen, errorcb,
572 MBENC_FLUSH | MBENC_RESET);
573 if (r == NULL)
574 goto errorexit;
575
576 ERROR_DECREF(errorcb);
577 Py_XDECREF(ucvt);
578 return make_tuple(r, datalen);
579
580errorexit:
581 ERROR_DECREF(errorcb);
582 Py_XDECREF(ucvt);
583 return NULL;
584}
585
586static PyObject *
587MultibyteCodec_Decode(MultibyteCodecObject *self,
588 PyObject *args, PyObject *kwargs)
589{
590 MultibyteCodec_State state;
591 MultibyteDecodeBuffer buf;
592 PyObject *errorcb;
593 const char *data, *errors = NULL;
594 Py_ssize_t datalen, finalsize;
595
596 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|z:decode",
597 codeckwarglist, &data, &datalen, &errors))
598 return NULL;
599
600 errorcb = internal_error_callback(errors);
601 if (errorcb == NULL)
602 return NULL;
603
604 if (datalen == 0) {
605 ERROR_DECREF(errorcb);
606 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);
607 }
608
609 buf.excobj = NULL;
610 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
611 buf.inbuf_end = buf.inbuf_top + datalen;
612 buf.outobj = PyUnicode_FromUnicode(NULL, datalen);
613 if (buf.outobj == NULL)
614 goto errorexit;
615 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);
616 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);
617
618 if (self->codec->decinit != NULL &&
619 self->codec->decinit(&state, self->codec->config) != 0)
620 goto errorexit;
621
622 while (buf.inbuf < buf.inbuf_end) {
623 Py_ssize_t inleft, outleft, r;
624
625 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
626 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
627
628 r = self->codec->decode(&state, self->codec->config,
629 &buf.inbuf, inleft, &buf.outbuf, outleft);
630 if (r == 0)
631 break;
632 else if (multibytecodec_decerror(self->codec, &state,
633 &buf, errorcb, r))
634 goto errorexit;
635 }
636
637 finalsize = (Py_ssize_t)(buf.outbuf -
638 PyUnicode_AS_UNICODE(buf.outobj));
639
640 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
641 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
642 goto errorexit;
643
644 Py_XDECREF(buf.excobj);
645 ERROR_DECREF(errorcb);
646 return make_tuple(buf.outobj, datalen);
647
648errorexit:
649 ERROR_DECREF(errorcb);
650 Py_XDECREF(buf.excobj);
651 Py_XDECREF(buf.outobj);
652
653 return NULL;
654}
655
656static struct PyMethodDef multibytecodec_methods[] = {
657 {"encode", (PyCFunction)MultibyteCodec_Encode,
658 METH_VARARGS | METH_KEYWORDS,
659 MultibyteCodec_Encode__doc__},
660 {"decode", (PyCFunction)MultibyteCodec_Decode,
661 METH_VARARGS | METH_KEYWORDS,
662 MultibyteCodec_Decode__doc__},
663 {NULL, NULL},
664};
665
666static void
667multibytecodec_dealloc(MultibyteCodecObject *self)
668{
669 PyObject_Del(self);
670}
671
672static PyTypeObject MultibyteCodec_Type = {
673 PyObject_HEAD_INIT(NULL)
674 0, /* ob_size */
675 "MultibyteCodec", /* tp_name */
676 sizeof(MultibyteCodecObject), /* tp_basicsize */
677 0, /* tp_itemsize */
678 /* methods */
679 (destructor)multibytecodec_dealloc, /* tp_dealloc */
680 0, /* tp_print */
681 0, /* tp_getattr */
682 0, /* tp_setattr */
683 0, /* tp_compare */
684 0, /* tp_repr */
685 0, /* tp_as_number */
686 0, /* tp_as_sequence */
687 0, /* tp_as_mapping */
688 0, /* tp_hash */
689 0, /* tp_call */
690 0, /* tp_str */
691 PyObject_GenericGetAttr, /* tp_getattro */
692 0, /* tp_setattro */
693 0, /* tp_as_buffer */
694 Py_TPFLAGS_DEFAULT, /* tp_flags */
695 0, /* tp_doc */
696 0, /* tp_traverse */
697 0, /* tp_clear */
698 0, /* tp_richcompare */
699 0, /* tp_weaklistoffset */
700 0, /* tp_iter */
701 0, /* tp_iterext */
702 multibytecodec_methods, /* tp_methods */
703};
704
705
706/**
707 * Utility functions for stateful codec mechanism
708 */
709
710#define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
711#define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
712
713static PyObject *
714encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
715 PyObject *unistr, int final)
716{
717 PyObject *ucvt, *r = NULL;
718 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;
719 Py_ssize_t datalen, origpending;
720
721 if (PyUnicode_Check(unistr))
722 ucvt = NULL;
723 else {
724 unistr = ucvt = PyObject_Unicode(unistr);
725 if (unistr == NULL)
726 return NULL;
727 else if (!PyUnicode_Check(unistr)) {
728 PyErr_SetString(PyExc_TypeError,
729 "couldn't convert the object to unicode.");
730 Py_DECREF(ucvt);
731 return NULL;
732 }
733 }
734
735 datalen = PyUnicode_GET_SIZE(unistr);
736 origpending = ctx->pendingsize;
737
738 if (origpending > 0) {
739 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);
740 if (inbuf_tmp == NULL)
741 goto errorexit;
742 memcpy(inbuf_tmp, ctx->pending,
743 Py_UNICODE_SIZE * ctx->pendingsize);
744 memcpy(inbuf_tmp + ctx->pendingsize,
745 PyUnicode_AS_UNICODE(unistr),
746 Py_UNICODE_SIZE * datalen);
747 datalen += ctx->pendingsize;
748 ctx->pendingsize = 0;
749 inbuf = inbuf_tmp;
750 }
751 else
752 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);
753
754 inbuf_end = inbuf + datalen;
755
756 r = multibytecodec_encode(ctx->codec, &ctx->state,
757 (const Py_UNICODE **)&inbuf,
758 datalen, ctx->errors, final ? MBENC_FLUSH : 0);
759 if (r == NULL) {
760 /* recover the original pending buffer */
761 if (origpending > 0)
762 memcpy(ctx->pending, inbuf_tmp,
763 Py_UNICODE_SIZE * origpending);
764 ctx->pendingsize = origpending;
765 goto errorexit;
766 }
767
768 if (inbuf < inbuf_end) {
769 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);
770 if (ctx->pendingsize > MAXENCPENDING) {
771 /* normal codecs can't reach here */
772 ctx->pendingsize = 0;
773 PyErr_SetString(PyExc_UnicodeError,
774 "pending buffer overflow");
775 goto errorexit;
776 }
777 memcpy(ctx->pending, inbuf,
778 ctx->pendingsize * Py_UNICODE_SIZE);
779 }
780
781 if (inbuf_tmp != NULL)
782 PyMem_Del(inbuf_tmp);
783 Py_XDECREF(ucvt);
784 return r;
785
786errorexit:
787 if (inbuf_tmp != NULL)
788 PyMem_Del(inbuf_tmp);
789 Py_XDECREF(r);
790 Py_XDECREF(ucvt);
791 return NULL;
792}
793
794static int
795decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
796 MultibyteDecodeBuffer *buf)
797{
798 Py_ssize_t npendings;
799
800 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
801 if (npendings + ctx->pendingsize > MAXDECPENDING) {
802 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");
803 return -1;
804 }
805 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
806 ctx->pendingsize += npendings;
807 return 0;
808}
809
810static int
811decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
812 Py_ssize_t size)
813{
814 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
815 buf->inbuf_end = buf->inbuf_top + size;
816 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */
817 buf->outobj = PyUnicode_FromUnicode(NULL, size);
818 if (buf->outobj == NULL)
819 return -1;
820 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);
821 buf->outbuf_end = buf->outbuf +
822 PyUnicode_GET_SIZE(buf->outobj);
823 }
824
825 return 0;
826}
827
828static int
829decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
830 MultibyteDecodeBuffer *buf)
831{
832 while (buf->inbuf < buf->inbuf_end) {
833 Py_ssize_t inleft, outleft;
834 Py_ssize_t r;
835
836 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
837 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
838
839 r = ctx->codec->decode(&ctx->state, ctx->codec->config,
840 &buf->inbuf, inleft, &buf->outbuf, outleft);
841 if (r == 0 || r == MBERR_TOOFEW)
842 break;
843 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
844 buf, ctx->errors, r))
845 return -1;
846 }
847 return 0;
848}
849
850
851/**
852 * MultibyteIncrementalEncoder object
853 */
854
855static PyObject *
856mbiencoder_encode(MultibyteIncrementalEncoderObject *self,
857 PyObject *args, PyObject *kwargs)
858{
859 PyObject *data;
860 int final = 0;
861
862 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",
863 incrementalkwarglist, &data, &final))
864 return NULL;
865
866 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);
867}
868
869static PyObject *
870mbiencoder_reset(MultibyteIncrementalEncoderObject *self)
871{
872 if (self->codec->decreset != NULL &&
873 self->codec->decreset(&self->state, self->codec->config) != 0)
874 return NULL;
875 self->pendingsize = 0;
876
877 Py_RETURN_NONE;
878}
879
880static struct PyMethodDef mbiencoder_methods[] = {
881 {"encode", (PyCFunction)mbiencoder_encode,
882 METH_VARARGS | METH_KEYWORDS, NULL},
883 {"reset", (PyCFunction)mbiencoder_reset,
884 METH_NOARGS, NULL},
885 {NULL, NULL},
886};
887
888static PyObject *
889mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
890{
891 MultibyteIncrementalEncoderObject *self;
892 PyObject *codec = NULL;
893 char *errors = NULL;
894
895 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
896 incnewkwarglist, &errors))
897 return NULL;
898
899 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
900 if (self == NULL)
901 return NULL;
902
903 codec = PyObject_GetAttrString((PyObject *)type, "codec");
904 if (codec == NULL)
905 goto errorexit;
906 if (!MultibyteCodec_Check(codec)) {
907 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
908 goto errorexit;
909 }
910
911 self->codec = ((MultibyteCodecObject *)codec)->codec;
912 self->pendingsize = 0;
913 self->errors = internal_error_callback(errors);
914 if (self->errors == NULL)
915 goto errorexit;
916 if (self->codec->encinit != NULL &&
917 self->codec->encinit(&self->state, self->codec->config) != 0)
918 goto errorexit;
919
920 Py_DECREF(codec);
921 return (PyObject *)self;
922
923errorexit:
924 Py_XDECREF(self);
925 Py_XDECREF(codec);
926 return NULL;
927}
928
929static int
930mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
931{
932 return 0;
933}
934
935static int
936mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,
937 visitproc visit, void *arg)
938{
939 if (ERROR_ISCUSTOM(self->errors))
940 Py_VISIT(self->errors);
941 return 0;
942}
943
944static void
945mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)
946{
947 PyObject_GC_UnTrack(self);
948 ERROR_DECREF(self->errors);
949 self->ob_type->tp_free(self);
950}
951
952static PyTypeObject MultibyteIncrementalEncoder_Type = {
953 PyObject_HEAD_INIT(NULL)
954 0, /* ob_size */
955 "MultibyteIncrementalEncoder", /* tp_name */
956 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */
957 0, /* tp_itemsize */
958 /* methods */
959 (destructor)mbiencoder_dealloc, /* tp_dealloc */
960 0, /* tp_print */
961 0, /* tp_getattr */
962 0, /* tp_setattr */
963 0, /* tp_compare */
964 0, /* tp_repr */
965 0, /* tp_as_number */
966 0, /* tp_as_sequence */
967 0, /* tp_as_mapping */
968 0, /* tp_hash */
969 0, /* tp_call */
970 0, /* tp_str */
971 PyObject_GenericGetAttr, /* tp_getattro */
972 0, /* tp_setattro */
973 0, /* tp_as_buffer */
974 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
975 | Py_TPFLAGS_BASETYPE, /* tp_flags */
976 0, /* tp_doc */
977 (traverseproc)mbiencoder_traverse, /* tp_traverse */
978 0, /* tp_clear */
979 0, /* tp_richcompare */
980 0, /* tp_weaklistoffset */
981 0, /* tp_iter */
982 0, /* tp_iterext */
983 mbiencoder_methods, /* tp_methods */
984 0, /* tp_members */
985 codecctx_getsets, /* tp_getset */
986 0, /* tp_base */
987 0, /* tp_dict */
988 0, /* tp_descr_get */
989 0, /* tp_descr_set */
990 0, /* tp_dictoffset */
991 mbiencoder_init, /* tp_init */
992 0, /* tp_alloc */
993 mbiencoder_new, /* tp_new */
994};
995
996
997/**
998 * MultibyteIncrementalDecoder object
999 */
1000
1001static PyObject *
1002mbidecoder_decode(MultibyteIncrementalDecoderObject *self,
1003 PyObject *args, PyObject *kwargs)
1004{
1005 MultibyteDecodeBuffer buf;
1006 char *data, *wdata;
1007 Py_ssize_t wsize, finalsize = 0, size, origpending;
1008 int final = 0;
1009
1010 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "t#|i:decode",
1011 incrementalkwarglist, &data, &size, &final))
1012 return NULL;
1013
1014 buf.outobj = buf.excobj = NULL;
1015 origpending = self->pendingsize;
1016
1017 if (self->pendingsize == 0) {
1018 wsize = size;
1019 wdata = data;
1020 }
1021 else {
1022 wsize = size + self->pendingsize;
1023 wdata = PyMem_Malloc(wsize);
1024 if (wdata == NULL)
1025 goto errorexit;
1026 memcpy(wdata, self->pending, self->pendingsize);
1027 memcpy(wdata + self->pendingsize, data, size);
1028 self->pendingsize = 0;
1029 }
1030
1031 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1032 goto errorexit;
1033
1034 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1035 goto errorexit;
1036
1037 if (final && buf.inbuf < buf.inbuf_end) {
1038 if (multibytecodec_decerror(self->codec, &self->state,
1039 &buf, self->errors, MBERR_TOOFEW)) {
1040 /* recover the original pending buffer */
1041 memcpy(self->pending, wdata, origpending);
1042 self->pendingsize = origpending;
1043 goto errorexit;
1044 }
1045 }
1046
1047 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1048 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1049 goto errorexit;
1050 }
1051
1052 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));
1053 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1054 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1055 goto errorexit;
1056
1057 if (wdata != data)
1058 PyMem_Del(wdata);
1059 Py_XDECREF(buf.excobj);
1060 return buf.outobj;
1061
1062errorexit:
1063 if (wdata != NULL && wdata != data)
1064 PyMem_Del(wdata);
1065 Py_XDECREF(buf.excobj);
1066 Py_XDECREF(buf.outobj);
1067 return NULL;
1068}
1069
1070static PyObject *
1071mbidecoder_reset(MultibyteIncrementalDecoderObject *self)
1072{
1073 if (self->codec->decreset != NULL &&
1074 self->codec->decreset(&self->state, self->codec->config) != 0)
1075 return NULL;
1076 self->pendingsize = 0;
1077
1078 Py_RETURN_NONE;
1079}
1080
1081static struct PyMethodDef mbidecoder_methods[] = {
1082 {"decode", (PyCFunction)mbidecoder_decode,
1083 METH_VARARGS | METH_KEYWORDS, NULL},
1084 {"reset", (PyCFunction)mbidecoder_reset,
1085 METH_NOARGS, NULL},
1086 {NULL, NULL},
1087};
1088
1089static PyObject *
1090mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1091{
1092 MultibyteIncrementalDecoderObject *self;
1093 PyObject *codec = NULL;
1094 char *errors = NULL;
1095
1096 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1097 incnewkwarglist, &errors))
1098 return NULL;
1099
1100 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1101 if (self == NULL)
1102 return NULL;
1103
1104 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1105 if (codec == NULL)
1106 goto errorexit;
1107 if (!MultibyteCodec_Check(codec)) {
1108 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1109 goto errorexit;
1110 }
1111
1112 self->codec = ((MultibyteCodecObject *)codec)->codec;
1113 self->pendingsize = 0;
1114 self->errors = internal_error_callback(errors);
1115 if (self->errors == NULL)
1116 goto errorexit;
1117 if (self->codec->decinit != NULL &&
1118 self->codec->decinit(&self->state, self->codec->config) != 0)
1119 goto errorexit;
1120
1121 Py_DECREF(codec);
1122 return (PyObject *)self;
1123
1124errorexit:
1125 Py_XDECREF(self);
1126 Py_XDECREF(codec);
1127 return NULL;
1128}
1129
1130static int
1131mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1132{
1133 return 0;
1134}
1135
1136static int
1137mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,
1138 visitproc visit, void *arg)
1139{
1140 if (ERROR_ISCUSTOM(self->errors))
1141 Py_VISIT(self->errors);
1142 return 0;
1143}
1144
1145static void
1146mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)
1147{
1148 PyObject_GC_UnTrack(self);
1149 ERROR_DECREF(self->errors);
1150 self->ob_type->tp_free(self);
1151}
1152
1153static PyTypeObject MultibyteIncrementalDecoder_Type = {
1154 PyObject_HEAD_INIT(NULL)
1155 0, /* ob_size */
1156 "MultibyteIncrementalDecoder", /* tp_name */
1157 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */
1158 0, /* tp_itemsize */
1159 /* methods */
1160 (destructor)mbidecoder_dealloc, /* tp_dealloc */
1161 0, /* tp_print */
1162 0, /* tp_getattr */
1163 0, /* tp_setattr */
1164 0, /* tp_compare */
1165 0, /* tp_repr */
1166 0, /* tp_as_number */
1167 0, /* tp_as_sequence */
1168 0, /* tp_as_mapping */
1169 0, /* tp_hash */
1170 0, /* tp_call */
1171 0, /* tp_str */
1172 PyObject_GenericGetAttr, /* tp_getattro */
1173 0, /* tp_setattro */
1174 0, /* tp_as_buffer */
1175 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1176 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1177 0, /* tp_doc */
1178 (traverseproc)mbidecoder_traverse, /* tp_traverse */
1179 0, /* tp_clear */
1180 0, /* tp_richcompare */
1181 0, /* tp_weaklistoffset */
1182 0, /* tp_iter */
1183 0, /* tp_iterext */
1184 mbidecoder_methods, /* tp_methods */
1185 0, /* tp_members */
1186 codecctx_getsets, /* tp_getset */
1187 0, /* tp_base */
1188 0, /* tp_dict */
1189 0, /* tp_descr_get */
1190 0, /* tp_descr_set */
1191 0, /* tp_dictoffset */
1192 mbidecoder_init, /* tp_init */
1193 0, /* tp_alloc */
1194 mbidecoder_new, /* tp_new */
1195};
1196
1197
1198/**
1199 * MultibyteStreamReader object
1200 */
1201
1202static PyObject *
1203mbstreamreader_iread(MultibyteStreamReaderObject *self,
1204 const char *method, Py_ssize_t sizehint)
1205{
1206 MultibyteDecodeBuffer buf;
1207 PyObject *cres;
1208 Py_ssize_t rsize, finalsize = 0;
1209
1210 if (sizehint == 0)
1211 return PyUnicode_FromUnicode(NULL, 0);
1212
1213 buf.outobj = buf.excobj = NULL;
1214 cres = NULL;
1215
1216 for (;;) {
1217 if (sizehint < 0)
1218 cres = PyObject_CallMethod(self->stream,
1219 (char *)method, NULL);
1220 else
1221 cres = PyObject_CallMethod(self->stream,
1222 (char *)method, "i", sizehint);
1223 if (cres == NULL)
1224 goto errorexit;
1225
1226 if (!PyString_Check(cres)) {
1227 PyErr_SetString(PyExc_TypeError,
1228 "stream function returned a "
1229 "non-string object");
1230 goto errorexit;
1231 }
1232
1233 if (self->pendingsize > 0) {
1234 PyObject *ctr;
1235 char *ctrdata;
1236
1237 rsize = PyString_GET_SIZE(cres) + self->pendingsize;
1238 ctr = PyString_FromStringAndSize(NULL, rsize);
1239 if (ctr == NULL)
1240 goto errorexit;
1241 ctrdata = PyString_AS_STRING(ctr);
1242 memcpy(ctrdata, self->pending, self->pendingsize);
1243 memcpy(ctrdata + self->pendingsize,
1244 PyString_AS_STRING(cres),
1245 PyString_GET_SIZE(cres));
1246 Py_DECREF(cres);
1247 cres = ctr;
1248 self->pendingsize = 0;
1249 }
1250
1251 rsize = PyString_GET_SIZE(cres);
1252 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres),
1253 rsize) != 0)
1254 goto errorexit;
1255
1256 if (rsize > 0 && decoder_feed_buffer(
1257 (MultibyteStatefulDecoderContext *)self, &buf))
1258 goto errorexit;
1259
1260 if (rsize == 0 || sizehint < 0) { /* end of file */
1261 if (buf.inbuf < buf.inbuf_end &&
1262 multibytecodec_decerror(self->codec, &self->state,
1263 &buf, self->errors, MBERR_TOOFEW))
1264 goto errorexit;
1265 }
1266
1267 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1268 if (decoder_append_pending(STATEFUL_DCTX(self),
1269 &buf) != 0)
1270 goto errorexit;
1271 }
1272
1273 finalsize = (Py_ssize_t)(buf.outbuf -
1274 PyUnicode_AS_UNICODE(buf.outobj));
1275 Py_DECREF(cres);
1276 cres = NULL;
1277
1278 if (sizehint < 0 || finalsize != 0 || rsize == 0)
1279 break;
1280
1281 sizehint = 1; /* read 1 more byte and retry */
1282 }
1283
1284 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1285 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1286 goto errorexit;
1287
1288 Py_XDECREF(cres);
1289 Py_XDECREF(buf.excobj);
1290 return buf.outobj;
1291
1292errorexit:
1293 Py_XDECREF(cres);
1294 Py_XDECREF(buf.excobj);
1295 Py_XDECREF(buf.outobj);
1296 return NULL;
1297}
1298
1299static PyObject *
1300mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)
1301{
1302 PyObject *sizeobj = NULL;
1303 Py_ssize_t size;
1304
1305 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))
1306 return NULL;
1307
1308 if (sizeobj == Py_None || sizeobj == NULL)
1309 size = -1;
1310 else if (PyInt_Check(sizeobj))
1311 size = PyInt_AsSsize_t(sizeobj);
1312 else {
1313 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1314 return NULL;
1315 }
1316
1317 return mbstreamreader_iread(self, "read", size);
1318}
1319
1320static PyObject *
1321mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)
1322{
1323 PyObject *sizeobj = NULL;
1324 Py_ssize_t size;
1325
1326 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))
1327 return NULL;
1328
1329 if (sizeobj == Py_None || sizeobj == NULL)
1330 size = -1;
1331 else if (PyInt_Check(sizeobj))
1332 size = PyInt_AsSsize_t(sizeobj);
1333 else {
1334 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1335 return NULL;
1336 }
1337
1338 return mbstreamreader_iread(self, "readline", size);
1339}
1340
1341static PyObject *
1342mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)
1343{
1344 PyObject *sizehintobj = NULL, *r, *sr;
1345 Py_ssize_t sizehint;
1346
1347 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))
1348 return NULL;
1349
1350 if (sizehintobj == Py_None || sizehintobj == NULL)
1351 sizehint = -1;
1352 else if (PyInt_Check(sizehintobj))
1353 sizehint = PyInt_AsSsize_t(sizehintobj);
1354 else {
1355 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1356 return NULL;
1357 }
1358
1359 r = mbstreamreader_iread(self, "read", sizehint);
1360 if (r == NULL)
1361 return NULL;
1362
1363 sr = PyUnicode_Splitlines(r, 1);
1364 Py_DECREF(r);
1365 return sr;
1366}
1367
1368static PyObject *
1369mbstreamreader_reset(MultibyteStreamReaderObject *self)
1370{
1371 if (self->codec->decreset != NULL &&
1372 self->codec->decreset(&self->state, self->codec->config) != 0)
1373 return NULL;
1374 self->pendingsize = 0;
1375
1376 Py_RETURN_NONE;
1377}
1378
1379static struct PyMethodDef mbstreamreader_methods[] = {
1380 {"read", (PyCFunction)mbstreamreader_read,
1381 METH_VARARGS, NULL},
1382 {"readline", (PyCFunction)mbstreamreader_readline,
1383 METH_VARARGS, NULL},
1384 {"readlines", (PyCFunction)mbstreamreader_readlines,
1385 METH_VARARGS, NULL},
1386 {"reset", (PyCFunction)mbstreamreader_reset,
1387 METH_NOARGS, NULL},
1388 {NULL, NULL},
1389};
1390
1391static PyMemberDef mbstreamreader_members[] = {
1392 {"stream", T_OBJECT,
1393 offsetof(MultibyteStreamReaderObject, stream),
1394 READONLY, NULL},
1395 {NULL,}
1396};
1397
1398static PyObject *
1399mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1400{
1401 MultibyteStreamReaderObject *self;
1402 PyObject *stream, *codec = NULL;
1403 char *errors = NULL;
1404
1405 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1406 streamkwarglist, &stream, &errors))
1407 return NULL;
1408
1409 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1410 if (self == NULL)
1411 return NULL;
1412
1413 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1414 if (codec == NULL)
1415 goto errorexit;
1416 if (!MultibyteCodec_Check(codec)) {
1417 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1418 goto errorexit;
1419 }
1420
1421 self->codec = ((MultibyteCodecObject *)codec)->codec;
1422 self->stream = stream;
1423 Py_INCREF(stream);
1424 self->pendingsize = 0;
1425 self->errors = internal_error_callback(errors);
1426 if (self->errors == NULL)
1427 goto errorexit;
1428 if (self->codec->decinit != NULL &&
1429 self->codec->decinit(&self->state, self->codec->config) != 0)
1430 goto errorexit;
1431
1432 Py_DECREF(codec);
1433 return (PyObject *)self;
1434
1435errorexit:
1436 Py_XDECREF(self);
1437 Py_XDECREF(codec);
1438 return NULL;
1439}
1440
1441static int
1442mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1443{
1444 return 0;
1445}
1446
1447static int
1448mbstreamreader_traverse(MultibyteStreamReaderObject *self,
1449 visitproc visit, void *arg)
1450{
1451 if (ERROR_ISCUSTOM(self->errors))
1452 Py_VISIT(self->errors);
1453 Py_VISIT(self->stream);
1454 return 0;
1455}
1456
1457static void
1458mbstreamreader_dealloc(MultibyteStreamReaderObject *self)
1459{
1460 PyObject_GC_UnTrack(self);
1461 ERROR_DECREF(self->errors);
1462 Py_DECREF(self->stream);
1463 self->ob_type->tp_free(self);
1464}
1465
1466static PyTypeObject MultibyteStreamReader_Type = {
1467 PyObject_HEAD_INIT(NULL)
1468 0, /* ob_size */
1469 "MultibyteStreamReader", /* tp_name */
1470 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */
1471 0, /* tp_itemsize */
1472 /* methods */
1473 (destructor)mbstreamreader_dealloc, /* tp_dealloc */
1474 0, /* tp_print */
1475 0, /* tp_getattr */
1476 0, /* tp_setattr */
1477 0, /* tp_compare */
1478 0, /* tp_repr */
1479 0, /* tp_as_number */
1480 0, /* tp_as_sequence */
1481 0, /* tp_as_mapping */
1482 0, /* tp_hash */
1483 0, /* tp_call */
1484 0, /* tp_str */
1485 PyObject_GenericGetAttr, /* tp_getattro */
1486 0, /* tp_setattro */
1487 0, /* tp_as_buffer */
1488 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1489 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1490 0, /* tp_doc */
1491 (traverseproc)mbstreamreader_traverse, /* tp_traverse */
1492 0, /* tp_clear */
1493 0, /* tp_richcompare */
1494 0, /* tp_weaklistoffset */
1495 0, /* tp_iter */
1496 0, /* tp_iterext */
1497 mbstreamreader_methods, /* tp_methods */
1498 mbstreamreader_members, /* tp_members */
1499 codecctx_getsets, /* tp_getset */
1500 0, /* tp_base */
1501 0, /* tp_dict */
1502 0, /* tp_descr_get */
1503 0, /* tp_descr_set */
1504 0, /* tp_dictoffset */
1505 mbstreamreader_init, /* tp_init */
1506 0, /* tp_alloc */
1507 mbstreamreader_new, /* tp_new */
1508};
1509
1510
1511/**
1512 * MultibyteStreamWriter object
1513 */
1514
1515static int
1516mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1517 PyObject *unistr)
1518{
1519 PyObject *str, *wr;
1520
1521 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1522 if (str == NULL)
1523 return -1;
1524
1525 wr = PyObject_CallMethod(self->stream, "write", "O", str);
1526 Py_DECREF(str);
1527 if (wr == NULL)
1528 return -1;
1529
1530 Py_DECREF(wr);
1531 return 0;
1532}
1533
1534static PyObject *
1535mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)
1536{
1537 if (mbstreamwriter_iwrite(self, strobj))
1538 return NULL;
1539 else
1540 Py_RETURN_NONE;
1541}
1542
1543static PyObject *
1544mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)
1545{
1546 PyObject *strobj;
1547 int i, r;
1548
1549 if (!PySequence_Check(lines)) {
1550 PyErr_SetString(PyExc_TypeError,
1551 "arg must be a sequence object");
1552 return NULL;
1553 }
1554
1555 for (i = 0; i < PySequence_Length(lines); i++) {
1556 /* length can be changed even within this loop */
1557 strobj = PySequence_GetItem(lines, i);
1558 if (strobj == NULL)
1559 return NULL;
1560
1561 r = mbstreamwriter_iwrite(self, strobj);
1562 Py_DECREF(strobj);
1563 if (r == -1)
1564 return NULL;
1565 }
1566
1567 Py_RETURN_NONE;
1568}
1569
1570static PyObject *
1571mbstreamwriter_reset(MultibyteStreamWriterObject *self)
1572{
1573 const Py_UNICODE *pending;
1574 PyObject *pwrt;
1575
1576 pending = self->pending;
1577 pwrt = multibytecodec_encode(self->codec, &self->state,
1578 &pending, self->pendingsize, self->errors,
1579 MBENC_FLUSH | MBENC_RESET);
1580 /* some pending buffer can be truncated when UnicodeEncodeError is
1581 * raised on 'strict' mode. but, 'reset' method is designed to
1582 * reset the pending buffer or states so failed string sequence
1583 * ought to be missed */
1584 self->pendingsize = 0;
1585 if (pwrt == NULL)
1586 return NULL;
1587
1588 if (PyString_Size(pwrt) > 0) {
1589 PyObject *wr;
1590 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt);
1591 if (wr == NULL) {
1592 Py_DECREF(pwrt);
1593 return NULL;
1594 }
1595 }
1596 Py_DECREF(pwrt);
1597
1598 Py_RETURN_NONE;
1599}
1600
1601static PyObject *
1602mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1603{
1604 MultibyteStreamWriterObject *self;
1605 PyObject *stream, *codec = NULL;
1606 char *errors = NULL;
1607
1608 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1609 streamkwarglist, &stream, &errors))
1610 return NULL;
1611
1612 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1613 if (self == NULL)
1614 return NULL;
1615
1616 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1617 if (codec == NULL)
1618 goto errorexit;
1619 if (!MultibyteCodec_Check(codec)) {
1620 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1621 goto errorexit;
1622 }
1623
1624 self->codec = ((MultibyteCodecObject *)codec)->codec;
1625 self->stream = stream;
1626 Py_INCREF(stream);
1627 self->pendingsize = 0;
1628 self->errors = internal_error_callback(errors);
1629 if (self->errors == NULL)
1630 goto errorexit;
1631 if (self->codec->encinit != NULL &&
1632 self->codec->encinit(&self->state, self->codec->config) != 0)
1633 goto errorexit;
1634
1635 Py_DECREF(codec);
1636 return (PyObject *)self;
1637
1638errorexit:
1639 Py_XDECREF(self);
1640 Py_XDECREF(codec);
1641 return NULL;
1642}
1643
1644static int
1645mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1646{
1647 return 0;
1648}
1649
1650static int
1651mbstreamwriter_traverse(MultibyteStreamWriterObject *self,
1652 visitproc visit, void *arg)
1653{
1654 if (ERROR_ISCUSTOM(self->errors))
1655 Py_VISIT(self->errors);
1656 Py_VISIT(self->stream);
1657 return 0;
1658}
1659
1660static void
1661mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)
1662{
1663 PyObject_GC_UnTrack(self);
1664 ERROR_DECREF(self->errors);
1665 Py_DECREF(self->stream);
1666 self->ob_type->tp_free(self);
1667}
1668
1669static struct PyMethodDef mbstreamwriter_methods[] = {
1670 {"write", (PyCFunction)mbstreamwriter_write,
1671 METH_O, NULL},
1672 {"writelines", (PyCFunction)mbstreamwriter_writelines,
1673 METH_O, NULL},
1674 {"reset", (PyCFunction)mbstreamwriter_reset,
1675 METH_NOARGS, NULL},
1676 {NULL, NULL},
1677};
1678
1679static PyMemberDef mbstreamwriter_members[] = {
1680 {"stream", T_OBJECT,
1681 offsetof(MultibyteStreamWriterObject, stream),
1682 READONLY, NULL},
1683 {NULL,}
1684};
1685
1686static PyTypeObject MultibyteStreamWriter_Type = {
1687 PyObject_HEAD_INIT(NULL)
1688 0, /* ob_size */
1689 "MultibyteStreamWriter", /* tp_name */
1690 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */
1691 0, /* tp_itemsize */
1692 /* methods */
1693 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */
1694 0, /* tp_print */
1695 0, /* tp_getattr */
1696 0, /* tp_setattr */
1697 0, /* tp_compare */
1698 0, /* tp_repr */
1699 0, /* tp_as_number */
1700 0, /* tp_as_sequence */
1701 0, /* tp_as_mapping */
1702 0, /* tp_hash */
1703 0, /* tp_call */
1704 0, /* tp_str */
1705 PyObject_GenericGetAttr, /* tp_getattro */
1706 0, /* tp_setattro */
1707 0, /* tp_as_buffer */
1708 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1709 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1710 0, /* tp_doc */
1711 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */
1712 0, /* tp_clear */
1713 0, /* tp_richcompare */
1714 0, /* tp_weaklistoffset */
1715 0, /* tp_iter */
1716 0, /* tp_iterext */
1717 mbstreamwriter_methods, /* tp_methods */
1718 mbstreamwriter_members, /* tp_members */
1719 codecctx_getsets, /* tp_getset */
1720 0, /* tp_base */
1721 0, /* tp_dict */
1722 0, /* tp_descr_get */
1723 0, /* tp_descr_set */
1724 0, /* tp_dictoffset */
1725 mbstreamwriter_init, /* tp_init */
1726 0, /* tp_alloc */
1727 mbstreamwriter_new, /* tp_new */
1728};
1729
1730
1731/**
1732 * Exposed factory function
1733 */
1734
1735static PyObject *
1736__create_codec(PyObject *ignore, PyObject *arg)
1737{
1738 MultibyteCodecObject *self;
1739 MultibyteCodec *codec;
1740
1741 if (!PyCObject_Check(arg)) {
1742 PyErr_SetString(PyExc_ValueError, "argument type invalid");
1743 return NULL;
1744 }
1745
1746 codec = PyCObject_AsVoidPtr(arg);
1747 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
1748 return NULL;
1749
1750 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);
1751 if (self == NULL)
1752 return NULL;
1753 self->codec = codec;
1754
1755 return (PyObject *)self;
1756}
1757
1758static struct PyMethodDef __methods[] = {
1759 {"__create_codec", (PyCFunction)__create_codec, METH_O},
1760 {NULL, NULL},
1761};
1762
1763PyMODINIT_FUNC
1764init_multibytecodec(void)
1765{
1766 int i;
1767 PyObject *m;
1768 PyTypeObject *typelist[] = {
1769 &MultibyteIncrementalEncoder_Type,
1770 &MultibyteIncrementalDecoder_Type,
1771 &MultibyteStreamReader_Type,
1772 &MultibyteStreamWriter_Type,
1773 NULL
1774 };
1775
1776 if (PyType_Ready(&MultibyteCodec_Type) < 0)
1777 return;
1778
1779 m = Py_InitModule("_multibytecodec", __methods);
1780 if (m == NULL)
1781 return;
1782
1783 for (i = 0; typelist[i] != NULL; i++) {
1784 if (PyType_Ready(typelist[i]) < 0)
1785 return;
1786 Py_INCREF(typelist[i]);
1787 PyModule_AddObject(m, typelist[i]->tp_name,
1788 (PyObject *)typelist[i]);
1789 }
1790
1791 if (PyErr_Occurred())
1792 Py_FatalError("can't initialize the _multibytecodec module");
1793}
Note: See TracBrowser for help on using the repository browser.