Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Blame
Revision Log

cPickle.c@ 3408

Visit:

Last change on this file since 3408 was 3225, checked in by bird, 19 years ago
Python 2.5
File size: 117.2 KB

Line
1	#include "Python.h"
2	#include "cStringIO.h"
3	#include "structmember.h"
4
5	PyDoc_STRVAR(cPickle_module_documentation,
6	"C implementation and optimization of the Python pickle module.");
7
8	#ifndef Py_eval_input
9	#include <graminit.h>
10	#define Py_eval_input eval_input
11	#endif /* Py_eval_input */
12
13	#define DEL_LIST_SLICE(list, from, to) (PyList_SetSlice(list, from, to, NULL))
14
15	#define WRITE_BUF_SIZE 256
16
17	/* Bump this when new opcodes are added to the pickle protocol. */
18	#define HIGHEST_PROTOCOL 2
19
20	/*
21	* Pickle opcodes. These must be kept in synch with pickle.py. Extensive
22	* docs are in pickletools.py.
23	*/
24	#define MARK '('
25	#define STOP '.'
26	#define POP '0'
27	#define POP_MARK '1'
28	#define DUP '2'
29	#define FLOAT 'F'
30	#define BINFLOAT 'G'
31	#define INT 'I'
32	#define BININT 'J'
33	#define BININT1 'K'
34	#define LONG 'L'
35	#define BININT2 'M'
36	#define NONE 'N'
37	#define PERSID 'P'
38	#define BINPERSID 'Q'
39	#define REDUCE 'R'
40	#define STRING 'S'
41	#define BINSTRING 'T'
42	#define SHORT_BINSTRING 'U'
43	#define UNICODE 'V'
44	#define BINUNICODE 'X'
45	#define APPEND 'a'
46	#define BUILD 'b'
47	#define GLOBAL 'c'
48	#define DICT 'd'
49	#define EMPTY_DICT '}'
50	#define APPENDS 'e'
51	#define GET 'g'
52	#define BINGET 'h'
53	#define INST 'i'
54	#define LONG_BINGET 'j'
55	#define LIST 'l'
56	#define EMPTY_LIST ']'
57	#define OBJ 'o'
58	#define PUT 'p'
59	#define BINPUT 'q'
60	#define LONG_BINPUT 'r'
61	#define SETITEM 's'
62	#define TUPLE 't'
63	#define EMPTY_TUPLE ')'
64	#define SETITEMS 'u'
65
66	/* Protocol 2. */
67	#define PROTO '\x80' /* identify pickle protocol */
68	#define NEWOBJ '\x81' /* build object by applying cls.__new__ to argtuple */
69	#define EXT1 '\x82' /* push object from extension registry; 1-byte index */
70	#define EXT2 '\x83' /* ditto, but 2-byte index */
71	#define EXT4 '\x84' /* ditto, but 4-byte index */
72	#define TUPLE1 '\x85' /* build 1-tuple from stack top */
73	#define TUPLE2 '\x86' /* build 2-tuple from two topmost stack items */
74	#define TUPLE3 '\x87' /* build 3-tuple from three topmost stack items */
75	#define NEWTRUE '\x88' /* push True */
76	#define NEWFALSE '\x89' /* push False */
77	#define LONG1 '\x8a' /* push long from < 256 bytes */
78	#define LONG4 '\x8b' /* push really big long */
79
80	/* There aren't opcodes -- they're ways to pickle bools before protocol 2,
81	* so that unpicklers written before bools were introduced unpickle them
82	* as ints, but unpicklers after can recognize that bools were intended.
83	* Note that protocol 2 added direct ways to pickle bools.
84	*/
85	#undef TRUE
86	#define TRUE "I01\n"
87	#undef FALSE
88	#define FALSE "I00\n"
89
90	/* Keep in synch with pickle.Pickler._BATCHSIZE. This is how many elements
91	* batch_list/dict() pumps out before doing APPENDS/SETITEMS. Nothing will
92	* break if this gets out of synch with pickle.py, but it's unclear that
93	* would help anything either.
94	*/
95	#define BATCHSIZE 1000
96
97	static char MARKv = MARK;
98
99	static PyObject *PickleError;
100	static PyObject *PicklingError;
101	static PyObject *UnpickleableError;
102	static PyObject *UnpicklingError;
103	static PyObject *BadPickleGet;
104
105	/* As the name says, an empty tuple. */
106	static PyObject *empty_tuple;
107
108	/* copy_reg.dispatch_table, {type_object: pickling_function} */
109	static PyObject *dispatch_table;
110
111	/* For EXT[124] opcodes. */
112	/* copy_reg._extension_registry, {(module_name, function_name): code} */
113	static PyObject *extension_registry;
114	/* copy_reg._inverted_registry, {code: (module_name, function_name)} */
115	static PyObject *inverted_registry;
116	/* copy_reg._extension_cache, {code: object} */
117	static PyObject *extension_cache;
118
119	/* For looking up name pairs in copy_reg._extension_registry. */
120	static PyObject *two_tuple;
121
122	static PyObject __class___str, __getinitargs___str, *__dict___str,
123	__getstate___str, __setstate___str, __name___str, __reduce___str,
124	*__reduce_ex___str,
125	write_str, append_str,
126	read_str, readline_str, *__main___str,
127	copy_reg_str, dispatch_table_str;
128
129	/*************************************************************************
130	Internal Data type for pickle data. */
131
132	typedef struct {
133	PyObject_HEAD
134	int length; /* number of initial slots in data currently used */
135	int size; /* number of slots in data allocated */
136	PyObject **data;
137	} Pdata;
138
139	static void
140	Pdata_dealloc(Pdata *self)
141	{
142	int i;
143	PyObject **p;
144
145	for (i = self->length, p = self->data; --i >= 0; p++) {
146	Py_DECREF(*p);
147	}
148	if (self->data)
149	free(self->data);
150	PyObject_Del(self);
151	}
152
153	static PyTypeObject PdataType = {
154	PyObject_HEAD_INIT(NULL) 0, "cPickle.Pdata", sizeof(Pdata), 0,
155	(destructor)Pdata_dealloc,
156	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0L,0L,0L,0L, ""
157	};
158
159	#define Pdata_Check(O) ((O)->ob_type == &PdataType)
160
161	static PyObject *
162	Pdata_New(void)
163	{
164	Pdata *self;
165
166	if (!(self = PyObject_New(Pdata, &PdataType)))
167	return NULL;
168	self->size = 8;
169	self->length = 0;
170	self->data = malloc(self->size * sizeof(PyObject*));
171	if (self->data)
172	return (PyObject*)self;
173	Py_DECREF(self);
174	return PyErr_NoMemory();
175	}
176
177	static int
178	stackUnderflow(void)
179	{
180	PyErr_SetString(UnpicklingError, "unpickling stack underflow");
181	return -1;
182	}
183
184	/* Retain only the initial clearto items. If clearto >= the current
185	* number of items, this is a (non-erroneous) NOP.
186	*/
187	static int
188	Pdata_clear(Pdata *self, int clearto)
189	{
190	int i;
191	PyObject **p;
192
193	if (clearto < 0) return stackUnderflow();
194	if (clearto >= self->length) return 0;
195
196	for (i = self->length, p = self->data + clearto;
197	--i >= clearto;
198	p++) {
199	Py_CLEAR(*p);
200	}
201	self->length = clearto;
202
203	return 0;
204	}
205
206	static int
207	Pdata_grow(Pdata *self)
208	{
209	int bigger;
210	size_t nbytes;
211	PyObject **tmp;
212
213	bigger = self->size << 1;
214	if (bigger <= 0) /* was 0, or new value overflows */
215	goto nomemory;
216	if ((int)(size_t)bigger != bigger)
217	goto nomemory;
218	nbytes = (size_t)bigger * sizeof(PyObject *);
219	if (nbytes / sizeof(PyObject *) != (size_t)bigger)
220	goto nomemory;
221	tmp = realloc(self->data, nbytes);
222	if (tmp == NULL)
223	goto nomemory;
224	self->data = tmp;
225	self->size = bigger;
226	return 0;
227
228	nomemory:
229	PyErr_NoMemory();
230	return -1;
231	}
232
233	/* D is a Pdata*. Pop the topmost element and store it into V, which
234	* must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
235	* is raised and V is set to NULL. D and V may be evaluated several times.
236	*/
237	#define PDATA_POP(D, V) { \
238	if ((D)->length) \
239	(V) = (D)->data[--((D)->length)]; \
240	else { \
241	PyErr_SetString(UnpicklingError, "bad pickle data"); \
242	(V) = NULL; \
243	} \
244	}
245
246	/* PDATA_PUSH and PDATA_APPEND both push rvalue PyObject* O on to Pdata*
247	* D. If the Pdata stack can't be grown to hold the new value, both
248	* raise MemoryError and execute "return ER". The difference is in ownership
249	* of O after: _PUSH transfers ownership of O from the caller to the stack
250	* (no incref of O is done, and in case of error O is decrefed), while
251	* _APPEND pushes a new reference.
252	*/
253
254	/* Push O on stack D, giving ownership of O to the stack. */
255	#define PDATA_PUSH(D, O, ER) { \
256	if (((Pdata)(D))->length == ((Pdata)(D))->size && \
257	Pdata_grow((Pdata*)(D)) < 0) { \
258	Py_DECREF(O); \
259	return ER; \
260	} \
261	((Pdata)(D))->data[((Pdata)(D))->length++] = (O); \
262	}
263
264	/* Push O on stack D, pushing a new reference. */
265	#define PDATA_APPEND(D, O, ER) { \
266	if (((Pdata)(D))->length == ((Pdata)(D))->size && \
267	Pdata_grow((Pdata*)(D)) < 0) \
268	return ER; \
269	Py_INCREF(O); \
270	((Pdata)(D))->data[((Pdata)(D))->length++] = (O); \
271	}
272
273
274	static PyObject *
275	Pdata_popTuple(Pdata *self, int start)
276	{
277	PyObject *r;
278	int i, j, l;
279
280	l = self->length-start;
281	r = PyTuple_New(l);
282	if (r == NULL)
283	return NULL;
284	for (i = start, j = 0 ; j < l; i++, j++)
285	PyTuple_SET_ITEM(r, j, self->data[i]);
286
287	self->length = start;
288	return r;
289	}
290
291	static PyObject *
292	Pdata_popList(Pdata *self, int start)
293	{
294	PyObject *r;
295	int i, j, l;
296
297	l=self->length-start;
298	if (!( r=PyList_New(l))) return NULL;
299	for (i=start, j=0 ; j < l; i++, j++)
300	PyList_SET_ITEM(r, j, self->data[i]);
301
302	self->length=start;
303	return r;
304	}
305
306	/*************************************************************************/
307
308	#define ARG_TUP(self, o) { \
309	if (self->arg \|\| (self->arg=PyTuple_New(1))) { \
310	Py_XDECREF(PyTuple_GET_ITEM(self->arg,0)); \
311	PyTuple_SET_ITEM(self->arg,0,o); \
312	} \
313	else { \
314	Py_DECREF(o); \
315	} \
316	}
317
318	#define FREE_ARG_TUP(self) { \
319	if (self->arg->ob_refcnt > 1) { \
320	Py_DECREF(self->arg); \
321	self->arg=NULL; \
322	} \
323	}
324
325	typedef struct Picklerobject {
326	PyObject_HEAD
327	FILE *fp;
328	PyObject *write;
329	PyObject *file;
330	PyObject *memo;
331	PyObject *arg;
332	PyObject *pers_func;
333	PyObject *inst_pers_func;
334
335	/* pickle protocol number, >= 0 */
336	int proto;
337
338	/* bool, true if proto > 0 */
339	int bin;
340
341	int fast; /* Fast mode doesn't save in memo, don't use if circ ref */
342	int nesting;
343	int (write_func)(struct Picklerobject , const char *, Py_ssize_t);
344	char *write_buf;
345	int buf_size;
346	PyObject *dispatch_table;
347	int fast_container; /* count nested container dumps */
348	PyObject *fast_memo;
349	} Picklerobject;
350
351	#ifndef PY_CPICKLE_FAST_LIMIT
352	#define PY_CPICKLE_FAST_LIMIT 50
353	#endif
354
355	static PyTypeObject Picklertype;
356
357	typedef struct Unpicklerobject {
358	PyObject_HEAD
359	FILE *fp;
360	PyObject *file;
361	PyObject *readline;
362	PyObject *read;
363	PyObject *memo;
364	PyObject *arg;
365	Pdata *stack;
366	PyObject *mark;
367	PyObject *pers_func;
368	PyObject *last_string;
369	int *marks;
370	int num_marks;
371	int marks_size;
372	Py_ssize_t (read_func)(struct Unpicklerobject , char **, Py_ssize_t);
373	Py_ssize_t (readline_func)(struct Unpicklerobject , char **);
374	int buf_size;
375	char *buf;
376	PyObject *find_class;
377	} Unpicklerobject;
378
379	static PyTypeObject Unpicklertype;
380
381	/* Forward decls that need the above structs */
382	static int save(Picklerobject , PyObject , int);
383	static int put2(Picklerobject , PyObject );
384
385	static
386	PyObject *
387	cPickle_ErrFormat(PyObject ErrType, char stringformat, char *format, ...)
388	{
389	va_list va;
390	PyObject args=0, retval=0;
391	va_start(va, format);
392
393	if (format) args = Py_VaBuildValue(format, va);
394	va_end(va);
395	if (format && ! args) return NULL;
396	if (stringformat && !(retval=PyString_FromString(stringformat)))
397	return NULL;
398
399	if (retval) {
400	if (args) {
401	PyObject *v;
402	v=PyString_Format(retval, args);
403	Py_DECREF(retval);
404	Py_DECREF(args);
405	if (! v) return NULL;
406	retval=v;
407	}
408	}
409	else
410	if (args) retval=args;
411	else {
412	PyErr_SetObject(ErrType,Py_None);
413	return NULL;
414	}
415	PyErr_SetObject(ErrType,retval);
416	Py_DECREF(retval);
417	return NULL;
418	}
419
420	static int
421	write_file(Picklerobject self, const char s, Py_ssize_t n)
422	{
423	size_t nbyteswritten;
424
425	if (s == NULL) {
426	return 0;
427	}
428
429	if (n > INT_MAX) {
430	/* String too large */
431	return -1;
432	}
433
434	Py_BEGIN_ALLOW_THREADS
435	nbyteswritten = fwrite(s, sizeof(char), n, self->fp);
436	Py_END_ALLOW_THREADS
437	if (nbyteswritten != (size_t)n) {
438	PyErr_SetFromErrno(PyExc_IOError);
439	return -1;
440	}
441
442	return (int)n;
443	}
444
445	static int
446	write_cStringIO(Picklerobject self, const char s, Py_ssize_t n)
447	{
448	if (s == NULL) {
449	return 0;
450	}
451
452	if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) {
453	return -1;
454	}
455
456	return (int)n;
457	}
458
459	static int
460	write_none(Picklerobject self, const char s, Py_ssize_t n)
461	{
462	if (s == NULL) return 0;
463	if (n > INT_MAX) return -1;
464	return (int)n;
465	}
466
467	static int
468	write_other(Picklerobject self, const char s, Py_ssize_t _n)
469	{
470	PyObject py_str = 0, junk = 0;
471	int n;
472
473	if (_n > INT_MAX)
474	return -1;
475	n = (int)_n;
476	if (s == NULL) {
477	if (!( self->buf_size )) return 0;
478	py_str = PyString_FromStringAndSize(self->write_buf,
479	self->buf_size);
480	if (!py_str)
481	return -1;
482	}
483	else {
484	if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) {
485	if (write_other(self, NULL, 0) < 0)
486	return -1;
487	}
488
489	if (n > WRITE_BUF_SIZE) {
490	if (!( py_str =
491	PyString_FromStringAndSize(s, n)))
492	return -1;
493	}
494	else {
495	memcpy(self->write_buf + self->buf_size, s, n);
496	self->buf_size += n;
497	return n;
498	}
499	}
500
501	if (self->write) {
502	/* object with write method */
503	ARG_TUP(self, py_str);
504	if (self->arg) {
505	junk = PyObject_Call(self->write, self->arg, NULL);
506	FREE_ARG_TUP(self);
507	}
508	if (junk) Py_DECREF(junk);
509	else return -1;
510	}
511	else
512	PDATA_PUSH(self->file, py_str, -1);
513
514	self->buf_size = 0;
515	return n;
516	}
517
518
519	static Py_ssize_t
520	read_file(Unpicklerobject self, char *s, Py_ssize_t n)
521	{
522	size_t nbytesread;
523
524	if (self->buf_size == 0) {
525	int size;
526
527	size = ((n < 32) ? 32 : n);
528	if (!( self->buf = (char *)malloc(size))) {
529	PyErr_NoMemory();
530	return -1;
531	}
532
533	self->buf_size = size;
534	}
535	else if (n > self->buf_size) {
536	self->buf = (char *)realloc(self->buf, n);
537	if (!self->buf) {
538	PyErr_NoMemory();
539	return -1;
540	}
541	self->buf_size = n;
542	}
543
544	Py_BEGIN_ALLOW_THREADS
545	nbytesread = fread(self->buf, sizeof(char), n, self->fp);
546	Py_END_ALLOW_THREADS
547	if (nbytesread != (size_t)n) {
548	if (feof(self->fp)) {
549	PyErr_SetNone(PyExc_EOFError);
550	return -1;
551	}
552
553	PyErr_SetFromErrno(PyExc_IOError);
554	return -1;
555	}
556
557	*s = self->buf;
558
559	return n;
560	}
561
562
563	static Py_ssize_t
564	readline_file(Unpicklerobject self, char *s)
565	{
566	int i;
567
568	if (self->buf_size == 0) {
569	if (!( self->buf = (char *)malloc(40))) {
570	PyErr_NoMemory();
571	return -1;
572	}
573	self->buf_size = 40;
574	}
575
576	i = 0;
577	while (1) {
578	int bigger;
579	for (; i < (self->buf_size - 1); i++) {
580	if (feof(self->fp) \|\|
581	(self->buf[i] = getc(self->fp)) == '\n') {
582	self->buf[i + 1] = '\0';
583	*s = self->buf;
584	return i + 1;
585	}
586	}
587	bigger = self->buf_size << 1;
588	if (bigger <= 0) { /* overflow */
589	PyErr_NoMemory();
590	return -1;
591	}
592	self->buf = (char *)realloc(self->buf, bigger);
593	if (!self->buf) {
594	PyErr_NoMemory();
595	return -1;
596	}
597	self->buf_size = bigger;
598	}
599	}
600
601
602	static Py_ssize_t
603	read_cStringIO(Unpicklerobject self, char *s, Py_ssize_t n)
604	{
605	char *ptr;
606
607	if (PycStringIO->cread((PyObject *)self->file, &ptr, n) != n) {
608	PyErr_SetNone(PyExc_EOFError);
609	return -1;
610	}
611
612	*s = ptr;
613
614	return n;
615	}
616
617
618	static Py_ssize_t
619	readline_cStringIO(Unpicklerobject self, char *s)
620	{
621	Py_ssize_t n;
622	char *ptr;
623
624	if ((n = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {
625	return -1;
626	}
627
628	*s = ptr;
629
630	return n;
631	}
632
633
634	static Py_ssize_t
635	read_other(Unpicklerobject self, char *s, Py_ssize_t n)
636	{
637	PyObject bytes, str=0;
638
639	if (!( bytes = PyInt_FromSsize_t(n))) return -1;
640
641	ARG_TUP(self, bytes);
642	if (self->arg) {
643	str = PyObject_Call(self->read, self->arg, NULL);
644	FREE_ARG_TUP(self);
645	}
646	if (! str) return -1;
647
648	Py_XDECREF(self->last_string);
649	self->last_string = str;
650
651	if (! (*s = PyString_AsString(str))) return -1;
652	return n;
653	}
654
655
656	static Py_ssize_t
657	readline_other(Unpicklerobject self, char *s)
658	{
659	PyObject *str;
660	Py_ssize_t str_size;
661
662	if (!( str = PyObject_CallObject(self->readline, empty_tuple))) {
663	return -1;
664	}
665
666	if ((str_size = PyString_Size(str)) < 0)
667	return -1;
668
669	Py_XDECREF(self->last_string);
670	self->last_string = str;
671
672	if (! (*s = PyString_AsString(str)))
673	return -1;
674
675	return str_size;
676	}
677
678	/* Copy the first n bytes from s into newly malloc'ed memory, plus a
679	* trailing 0 byte. Return a pointer to that, or NULL if out of memory.
680	* The caller is responsible for free()'ing the return value.
681	*/
682	static char *
683	pystrndup(const char *s, int n)
684	{
685	char r = (char )malloc(n+1);
686	if (r == NULL)
687	return (char*)PyErr_NoMemory();
688	memcpy(r, s, n);
689	r[n] = 0;
690	return r;
691	}
692
693
694	static int
695	get(Picklerobject self, PyObject id)
696	{
697	PyObject value, mv;
698	long c_value;
699	char s[30];
700	size_t len;
701
702	if (!( mv = PyDict_GetItem(self->memo, id))) {
703	PyErr_SetObject(PyExc_KeyError, id);
704	return -1;
705	}
706
707	if (!( value = PyTuple_GetItem(mv, 0)))
708	return -1;
709
710	if (!( PyInt_Check(value))) {
711	PyErr_SetString(PicklingError, "no int where int expected in memo");
712	return -1;
713	}
714	c_value = PyInt_AS_LONG((PyIntObject*)value);
715
716	if (!self->bin) {
717	s[0] = GET;
718	PyOS_snprintf(s + 1, sizeof(s) - 1, "%ld\n", c_value);
719	len = strlen(s);
720	}
721	else if (Pdata_Check(self->file)) {
722	if (write_other(self, NULL, 0) < 0) return -1;
723	PDATA_APPEND(self->file, mv, -1);
724	return 0;
725	}
726	else {
727	if (c_value < 256) {
728	s[0] = BINGET;
729	s[1] = (int)(c_value & 0xff);
730	len = 2;
731	}
732	else {
733	s[0] = LONG_BINGET;
734	s[1] = (int)(c_value & 0xff);
735	s[2] = (int)((c_value >> 8) & 0xff);
736	s[3] = (int)((c_value >> 16) & 0xff);
737	s[4] = (int)((c_value >> 24) & 0xff);
738	len = 5;
739	}
740	}
741
742	if (self->write_func(self, s, len) < 0)
743	return -1;
744
745	return 0;
746	}
747
748
749	static int
750	put(Picklerobject self, PyObject ob)
751	{
752	if (ob->ob_refcnt < 2 \|\| self->fast)
753	return 0;
754
755	return put2(self, ob);
756	}
757
758
759	static int
760	put2(Picklerobject self, PyObject ob)
761	{
762	char c_str[30];
763	int p;
764	size_t len;
765	int res = -1;
766	PyObject py_ob_id = 0, memo_len = 0, *t = 0;
767
768	if (self->fast)
769	return 0;
770
771	if ((p = PyDict_Size(self->memo)) < 0)
772	goto finally;
773
774	/* Make sure memo keys are positive! */
775	/* XXX Why?
776	* XXX And does "positive" really mean non-negative?
777	* XXX pickle.py starts with PUT index 0, not 1. This makes for
778	* XXX gratuitous differences between the pickling modules.
779	*/
780	p++;
781
782	if (!( py_ob_id = PyLong_FromVoidPtr(ob)))
783	goto finally;
784
785	if (!( memo_len = PyInt_FromLong(p)))
786	goto finally;
787
788	if (!( t = PyTuple_New(2)))
789	goto finally;
790
791	PyTuple_SET_ITEM(t, 0, memo_len);
792	Py_INCREF(memo_len);
793	PyTuple_SET_ITEM(t, 1, ob);
794	Py_INCREF(ob);
795
796	if (PyDict_SetItem(self->memo, py_ob_id, t) < 0)
797	goto finally;
798
799	if (!self->bin) {
800	c_str[0] = PUT;
801	PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%d\n", p);
802	len = strlen(c_str);
803	}
804	else if (Pdata_Check(self->file)) {
805	if (write_other(self, NULL, 0) < 0) return -1;
806	PDATA_APPEND(self->file, memo_len, -1);
807	res=0; /* Job well done ;) */
808	goto finally;
809	}
810	else {
811	if (p >= 256) {
812	c_str[0] = LONG_BINPUT;
813	c_str[1] = (int)(p & 0xff);
814	c_str[2] = (int)((p >> 8) & 0xff);
815	c_str[3] = (int)((p >> 16) & 0xff);
816	c_str[4] = (int)((p >> 24) & 0xff);
817	len = 5;
818	}
819	else {
820	c_str[0] = BINPUT;
821	c_str[1] = p;
822	len = 2;
823	}
824	}
825
826	if (self->write_func(self, c_str, len) < 0)
827	goto finally;
828
829	res = 0;
830
831	finally:
832	Py_XDECREF(py_ob_id);
833	Py_XDECREF(memo_len);
834	Py_XDECREF(t);
835
836	return res;
837	}
838
839	static PyObject *
840	whichmodule(PyObject global, PyObject global_name)
841	{
842	Py_ssize_t i, j;
843	PyObject module = 0, modules_dict = 0,
844	global_name_attr = 0, name = 0;
845
846	module = PyObject_GetAttrString(global, "__module__");
847	if (module)
848	return module;
849	if (PyErr_ExceptionMatches(PyExc_AttributeError))
850	PyErr_Clear();
851	else
852	return NULL;
853
854	if (!( modules_dict = PySys_GetObject("modules")))
855	return NULL;
856
857	i = 0;
858	while ((j = PyDict_Next(modules_dict, &i, &name, &module))) {
859
860	if (PyObject_Compare(name, __main___str)==0) continue;
861
862	global_name_attr = PyObject_GetAttr(module, global_name);
863	if (!global_name_attr) {
864	if (PyErr_ExceptionMatches(PyExc_AttributeError))
865	PyErr_Clear();
866	else
867	return NULL;
868	continue;
869	}
870
871	if (global_name_attr != global) {
872	Py_DECREF(global_name_attr);
873	continue;
874	}
875
876	Py_DECREF(global_name_attr);
877
878	break;
879	}
880
881	/* The following implements the rule in pickle.py added in 1.5
882	that used __main__ if no module is found. I don't actually
883	like this rule. jlf
884	*/
885	if (!j) {
886	j=1;
887	name=__main___str;
888	}
889
890	Py_INCREF(name);
891	return name;
892	}
893
894
895	static int
896	fast_save_enter(Picklerobject self, PyObject obj)
897	{
898	/* if fast_container < 0, we're doing an error exit. */
899	if (++self->fast_container >= PY_CPICKLE_FAST_LIMIT) {
900	PyObject *key = NULL;
901	if (self->fast_memo == NULL) {
902	self->fast_memo = PyDict_New();
903	if (self->fast_memo == NULL) {
904	self->fast_container = -1;
905	return 0;
906	}
907	}
908	key = PyLong_FromVoidPtr(obj);
909	if (key == NULL)
910	return 0;
911	if (PyDict_GetItem(self->fast_memo, key)) {
912	Py_DECREF(key);
913	PyErr_Format(PyExc_ValueError,
914	"fast mode: can't pickle cyclic objects "
915	"including object type %s at %p",
916	obj->ob_type->tp_name, obj);
917	self->fast_container = -1;
918	return 0;
919	}
920	if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {
921	Py_DECREF(key);
922	self->fast_container = -1;
923	return 0;
924	}
925	Py_DECREF(key);
926	}
927	return 1;
928	}
929
930	int
931	fast_save_leave(Picklerobject self, PyObject obj)
932	{
933	if (self->fast_container-- >= PY_CPICKLE_FAST_LIMIT) {
934	PyObject *key = PyLong_FromVoidPtr(obj);
935	if (key == NULL)
936	return 0;
937	if (PyDict_DelItem(self->fast_memo, key) < 0) {
938	Py_DECREF(key);
939	return 0;
940	}
941	Py_DECREF(key);
942	}
943	return 1;
944	}
945
946	static int
947	save_none(Picklerobject self, PyObject args)
948	{
949	static char none = NONE;
950	if (self->write_func(self, &none, 1) < 0)
951	return -1;
952
953	return 0;
954	}
955
956	static int
957	save_bool(Picklerobject self, PyObject args)
958	{
959	static const char *buf[2] = {FALSE, TRUE};
960	static char len[2] = {sizeof(FALSE)-1, sizeof(TRUE)-1};
961	long l = PyInt_AS_LONG((PyIntObject *)args);
962
963	if (self->proto >= 2) {
964	char opcode = l ? NEWTRUE : NEWFALSE;
965	if (self->write_func(self, &opcode, 1) < 0)
966	return -1;
967	}
968	else if (self->write_func(self, buf[l], len[l]) < 0)
969	return -1;
970	return 0;
971	}
972
973	static int
974	save_int(Picklerobject self, PyObject args)
975	{
976	char c_str[32];
977	long l = PyInt_AS_LONG((PyIntObject *)args);
978	int len = 0;
979
980	if (!self->bin
981	#if SIZEOF_LONG > 4
982	\|\| l > 0x7fffffffL
983	\|\| l < -0x80000000L
984	#endif
985	) {
986	/* Text-mode pickle, or long too big to fit in the 4-byte
987	* signed BININT format: store as a string.
988	*/
989	c_str[0] = INT;
990	PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%ld\n", l);
991	if (self->write_func(self, c_str, strlen(c_str)) < 0)
992	return -1;
993	}
994	else {
995	/* Binary pickle and l fits in a signed 4-byte int. */
996	c_str[1] = (int)( l & 0xff);
997	c_str[2] = (int)((l >> 8) & 0xff);
998	c_str[3] = (int)((l >> 16) & 0xff);
999	c_str[4] = (int)((l >> 24) & 0xff);
1000
1001	if ((c_str[4] == 0) && (c_str[3] == 0)) {
1002	if (c_str[2] == 0) {
1003	c_str[0] = BININT1;
1004	len = 2;
1005	}
1006	else {
1007	c_str[0] = BININT2;
1008	len = 3;
1009	}
1010	}
1011	else {
1012	c_str[0] = BININT;
1013	len = 5;
1014	}
1015
1016	if (self->write_func(self, c_str, len) < 0)
1017	return -1;
1018	}
1019
1020	return 0;
1021	}
1022
1023
1024	static int
1025	save_long(Picklerobject self, PyObject args)
1026	{
1027	int size;
1028	int res = -1;
1029	PyObject *repr = NULL;
1030
1031	static char l = LONG;
1032
1033	if (self->proto >= 2) {
1034	/* Linear-time pickling. */
1035	size_t nbits;
1036	size_t nbytes;
1037	unsigned char *pdata;
1038	char c_str[5];
1039	int i;
1040	int sign = _PyLong_Sign(args);
1041
1042	if (sign == 0) {
1043	/* It's 0 -- an empty bytestring. */
1044	c_str[0] = LONG1;
1045	c_str[1] = 0;
1046	i = self->write_func(self, c_str, 2);
1047	if (i < 0) goto finally;
1048	res = 0;
1049	goto finally;
1050	}
1051	nbits = _PyLong_NumBits(args);
1052	if (nbits == (size_t)-1 && PyErr_Occurred())
1053	goto finally;
1054	/* How many bytes do we need? There are nbits >> 3 full
1055	* bytes of data, and nbits & 7 leftover bits. If there
1056	* are any leftover bits, then we clearly need another
1057	* byte. Wnat's not so obvious is that we probably
1058	* need another byte even if there aren't any leftovers:
1059	* the most-significant bit of the most-significant byte
1060	* acts like a sign bit, and it's usually got a sense
1061	* opposite of the one we need. The exception is longs
1062	* of the form -(2*(8j-1)) for j > 0. Such a long is
1063	* its own 256's-complement, so has the right sign bit
1064	* even without the extra byte. That's a pain to check
1065	* for in advance, though, so we always grab an extra
1066	* byte at the start, and cut it back later if possible.
1067	*/
1068	nbytes = (nbits >> 3) + 1;
1069	if ((int)nbytes < 0 \|\| (size_t)(int)nbytes != nbytes) {
1070	PyErr_SetString(PyExc_OverflowError, "long too large "
1071	"to pickle");
1072	goto finally;
1073	}
1074	repr = PyString_FromStringAndSize(NULL, (int)nbytes);
1075	if (repr == NULL) goto finally;
1076	pdata = (unsigned char *)PyString_AS_STRING(repr);
1077	i = _PyLong_AsByteArray((PyLongObject *)args,
1078	pdata, nbytes,
1079	1 /* little endian /, 1 / signed */);
1080	if (i < 0) goto finally;
1081	/* If the long is negative, this may be a byte more than
1082	* needed. This is so iff the MSB is all redundant sign
1083	* bits.
1084	*/
1085	if (sign < 0 && nbytes > 1 && pdata[nbytes - 1] == 0xff &&
1086	(pdata[nbytes - 2] & 0x80) != 0)
1087	--nbytes;
1088
1089	if (nbytes < 256) {
1090	c_str[0] = LONG1;
1091	c_str[1] = (char)nbytes;
1092	size = 2;
1093	}
1094	else {
1095	c_str[0] = LONG4;
1096	size = (int)nbytes;
1097	for (i = 1; i < 5; i++) {
1098	c_str[i] = (char)(size & 0xff);
1099	size >>= 8;
1100	}
1101	size = 5;
1102	}
1103	i = self->write_func(self, c_str, size);
1104	if (i < 0) goto finally;
1105	i = self->write_func(self, (char *)pdata, (int)nbytes);
1106	if (i < 0) goto finally;
1107	res = 0;
1108	goto finally;
1109	}
1110
1111	/* proto < 2: write the repr and newline. This is quadratic-time
1112	* (in the number of digits), in both directions.
1113	*/
1114	if (!( repr = PyObject_Repr(args)))
1115	goto finally;
1116
1117	if ((size = PyString_Size(repr)) < 0)
1118	goto finally;
1119
1120	if (self->write_func(self, &l, 1) < 0)
1121	goto finally;
1122
1123	if (self->write_func(self,
1124	PyString_AS_STRING((PyStringObject *)repr),
1125	size) < 0)
1126	goto finally;
1127
1128	if (self->write_func(self, "\n", 1) < 0)
1129	goto finally;
1130
1131	res = 0;
1132
1133	finally:
1134	Py_XDECREF(repr);
1135	return res;
1136	}
1137
1138
1139	static int
1140	save_float(Picklerobject self, PyObject args)
1141	{
1142	double x = PyFloat_AS_DOUBLE((PyFloatObject *)args);
1143
1144	if (self->bin) {
1145	char str[9];
1146	str[0] = BINFLOAT;
1147	if (_PyFloat_Pack8(x, (unsigned char *)&str[1], 0) < 0)
1148	return -1;
1149	if (self->write_func(self, str, 9) < 0)
1150	return -1;
1151	}
1152	else {
1153	char c_str[250];
1154	c_str[0] = FLOAT;
1155	PyOS_ascii_formatd(c_str + 1, sizeof(c_str) - 2, "%.17g", x);
1156	/* Extend the formatted string with a newline character */
1157	strcat(c_str, "\n");
1158
1159	if (self->write_func(self, c_str, strlen(c_str)) < 0)
1160	return -1;
1161	}
1162
1163	return 0;
1164	}
1165
1166
1167	static int
1168	save_string(Picklerobject self, PyObject args, int doput)
1169	{
1170	int size, len;
1171	PyObject *repr=0;
1172
1173	if ((size = PyString_Size(args)) < 0)
1174	return -1;
1175
1176	if (!self->bin) {
1177	char *repr_str;
1178
1179	static char string = STRING;
1180
1181	if (!( repr = PyObject_Repr(args)))
1182	return -1;
1183
1184	if ((len = PyString_Size(repr)) < 0)
1185	goto err;
1186	repr_str = PyString_AS_STRING((PyStringObject *)repr);
1187
1188	if (self->write_func(self, &string, 1) < 0)
1189	goto err;
1190
1191	if (self->write_func(self, repr_str, len) < 0)
1192	goto err;
1193
1194	if (self->write_func(self, "\n", 1) < 0)
1195	goto err;
1196
1197	Py_XDECREF(repr);
1198	}
1199	else {
1200	int i;
1201	char c_str[5];
1202
1203	if ((size = PyString_Size(args)) < 0)
1204	return -1;
1205
1206	if (size < 256) {
1207	c_str[0] = SHORT_BINSTRING;
1208	c_str[1] = size;
1209	len = 2;
1210	}
1211	else {
1212	c_str[0] = BINSTRING;
1213	for (i = 1; i < 5; i++)
1214	c_str[i] = (int)(size >> ((i - 1) * 8));
1215	len = 5;
1216	}
1217
1218	if (self->write_func(self, c_str, len) < 0)
1219	return -1;
1220
1221	if (size > 128 && Pdata_Check(self->file)) {
1222	if (write_other(self, NULL, 0) < 0) return -1;
1223	PDATA_APPEND(self->file, args, -1);
1224	}
1225	else {
1226	if (self->write_func(self,
1227	PyString_AS_STRING(
1228	(PyStringObject *)args),
1229	size) < 0)
1230	return -1;
1231	}
1232	}
1233
1234	if (doput)
1235	if (put(self, args) < 0)
1236	return -1;
1237
1238	return 0;
1239
1240	err:
1241	Py_XDECREF(repr);
1242	return -1;
1243	}
1244
1245
1246	#ifdef Py_USING_UNICODE
1247	/* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
1248	backslash and newline characters to \uXXXX escapes. */
1249	static PyObject *
1250	modified_EncodeRawUnicodeEscape(const Py_UNICODE *s, int size)
1251	{
1252	PyObject *repr;
1253	char *p;
1254	char *q;
1255
1256	static const char *hexdigit = "0123456789ABCDEF";
1257
1258	repr = PyString_FromStringAndSize(NULL, 6 * size);
1259	if (repr == NULL)
1260	return NULL;
1261	if (size == 0)
1262	return repr;
1263
1264	p = q = PyString_AS_STRING(repr);
1265	while (size-- > 0) {
1266	Py_UNICODE ch = *s++;
1267	/* Map 16-bit characters to '\uxxxx' */
1268	if (ch >= 256 \|\| ch == '\\' \|\| ch == '\n') {
1269	*p++ = '\\';
1270	*p++ = 'u';
1271	*p++ = hexdigit[(ch >> 12) & 0xf];
1272	*p++ = hexdigit[(ch >> 8) & 0xf];
1273	*p++ = hexdigit[(ch >> 4) & 0xf];
1274	*p++ = hexdigit[ch & 15];
1275	}
1276	/* Copy everything else as-is */
1277	else
1278	*p++ = (char) ch;
1279	}
1280	*p = '\0';
1281	_PyString_Resize(&repr, p - q);
1282	return repr;
1283	}
1284
1285
1286	static int
1287	save_unicode(Picklerobject self, PyObject args, int doput)
1288	{
1289	int size, len;
1290	PyObject *repr=0;
1291
1292	if (!PyUnicode_Check(args))
1293	return -1;
1294
1295	if (!self->bin) {
1296	char *repr_str;
1297	static char string = UNICODE;
1298
1299	repr = modified_EncodeRawUnicodeEscape(
1300	PyUnicode_AS_UNICODE(args), PyUnicode_GET_SIZE(args));
1301	if (!repr)
1302	return -1;
1303
1304	if ((len = PyString_Size(repr)) < 0)
1305	goto err;
1306	repr_str = PyString_AS_STRING((PyStringObject *)repr);
1307
1308	if (self->write_func(self, &string, 1) < 0)
1309	goto err;
1310
1311	if (self->write_func(self, repr_str, len) < 0)
1312	goto err;
1313
1314	if (self->write_func(self, "\n", 1) < 0)
1315	goto err;
1316
1317	Py_XDECREF(repr);
1318	}
1319	else {
1320	int i;
1321	char c_str[5];
1322
1323	if (!( repr = PyUnicode_AsUTF8String(args)))
1324	return -1;
1325
1326	if ((size = PyString_Size(repr)) < 0)
1327	goto err;
1328
1329	c_str[0] = BINUNICODE;
1330	for (i = 1; i < 5; i++)
1331	c_str[i] = (int)(size >> ((i - 1) * 8));
1332	len = 5;
1333
1334	if (self->write_func(self, c_str, len) < 0)
1335	goto err;
1336
1337	if (size > 128 && Pdata_Check(self->file)) {
1338	if (write_other(self, NULL, 0) < 0)
1339	goto err;
1340	PDATA_APPEND(self->file, repr, -1);
1341	}
1342	else {
1343	if (self->write_func(self, PyString_AS_STRING(repr),
1344	size) < 0)
1345	goto err;
1346	}
1347
1348	Py_DECREF(repr);
1349	}
1350
1351	if (doput)
1352	if (put(self, args) < 0)
1353	return -1;
1354
1355	return 0;
1356
1357	err:
1358	Py_XDECREF(repr);
1359	return -1;
1360	}
1361	#endif
1362
1363	/* A helper for save_tuple. Push the len elements in tuple t on the stack. */
1364	static int
1365	store_tuple_elements(Picklerobject self, PyObject t, int len)
1366	{
1367	int i;
1368	int res = -1; /* guilty until proved innocent */
1369
1370	assert(PyTuple_Size(t) == len);
1371
1372	for (i = 0; i < len; i++) {
1373	PyObject *element = PyTuple_GET_ITEM(t, i);
1374
1375	if (element == NULL)
1376	goto finally;
1377	if (save(self, element, 0) < 0)
1378	goto finally;
1379	}
1380	res = 0;
1381
1382	finally:
1383	return res;
1384	}
1385
1386	/* Tuples are ubiquitous in the pickle protocols, so many techniques are
1387	* used across protocols to minimize the space needed to pickle them.
1388	* Tuples are also the only builtin immutable type that can be recursive
1389	* (a tuple can be reached from itself), and that requires some subtle
1390	* magic so that it works in all cases. IOW, this is a long routine.
1391	*/
1392	static int
1393	save_tuple(Picklerobject self, PyObject args)
1394	{
1395	PyObject *py_tuple_id = NULL;
1396	int len, i;
1397	int res = -1;
1398
1399	static char tuple = TUPLE;
1400	static char pop = POP;
1401	static char pop_mark = POP_MARK;
1402	static char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
1403
1404	if ((len = PyTuple_Size(args)) < 0)
1405	goto finally;
1406
1407	if (len == 0) {
1408	char c_str[2];
1409
1410	if (self->proto) {
1411	c_str[0] = EMPTY_TUPLE;
1412	len = 1;
1413	}
1414	else {
1415	c_str[0] = MARK;
1416	c_str[1] = TUPLE;
1417	len = 2;
1418	}
1419	if (self->write_func(self, c_str, len) >= 0)
1420	res = 0;
1421	/* Don't memoize an empty tuple. */
1422	goto finally;
1423	}
1424
1425	/* A non-empty tuple. */
1426
1427	/* id(tuple) isn't in the memo now. If it shows up there after
1428	* saving the tuple elements, the tuple must be recursive, in
1429	* which case we'll pop everything we put on the stack, and fetch
1430	* its value from the memo.
1431	*/
1432	py_tuple_id = PyLong_FromVoidPtr(args);
1433	if (py_tuple_id == NULL)
1434	goto finally;
1435
1436	if (len <= 3 && self->proto >= 2) {
1437	/* Use TUPLE{1,2,3} opcodes. */
1438	if (store_tuple_elements(self, args, len) < 0)
1439	goto finally;
1440	if (PyDict_GetItem(self->memo, py_tuple_id)) {
1441	/* pop the len elements */
1442	for (i = 0; i < len; ++i)
1443	if (self->write_func(self, &pop, 1) < 0)
1444	goto finally;
1445	/* fetch from memo */
1446	if (get(self, py_tuple_id) < 0)
1447	goto finally;
1448	res = 0;
1449	goto finally;
1450	}
1451	/* Not recursive. */
1452	if (self->write_func(self, len2opcode + len, 1) < 0)
1453	goto finally;
1454	goto memoize;
1455	}
1456
1457	/* proto < 2 and len > 0, or proto >= 2 and len > 3.
1458	* Generate MARK elt1 elt2 ... TUPLE
1459	*/
1460	if (self->write_func(self, &MARKv, 1) < 0)
1461	goto finally;
1462
1463	if (store_tuple_elements(self, args, len) < 0)
1464	goto finally;
1465
1466	if (PyDict_GetItem(self->memo, py_tuple_id)) {
1467	/* pop the stack stuff we pushed */
1468	if (self->bin) {
1469	if (self->write_func(self, &pop_mark, 1) < 0)
1470	goto finally;
1471	}
1472	else {
1473	/* Note that we pop one more than len, to remove
1474	* the MARK too.
1475	*/
1476	for (i = 0; i <= len; i++)
1477	if (self->write_func(self, &pop, 1) < 0)
1478	goto finally;
1479	}
1480	/* fetch from memo */
1481	if (get(self, py_tuple_id) >= 0)
1482	res = 0;
1483	goto finally;
1484	}
1485
1486	/* Not recursive. */
1487	if (self->write_func(self, &tuple, 1) < 0)
1488	goto finally;
1489
1490	memoize:
1491	if (put(self, args) >= 0)
1492	res = 0;
1493
1494	finally:
1495	Py_XDECREF(py_tuple_id);
1496	return res;
1497	}
1498
1499	/* iter is an iterator giving items, and we batch up chunks of
1500	* MARK item item ... item APPENDS
1501	* opcode sequences. Calling code should have arranged to first create an
1502	* empty list, or list-like object, for the APPENDS to operate on.
1503	* Returns 0 on success, <0 on error.
1504	*/
1505	static int
1506	batch_list(Picklerobject self, PyObject iter)
1507	{
1508	PyObject *obj;
1509	PyObject *slice[BATCHSIZE];
1510	int i, n;
1511
1512	static char append = APPEND;
1513	static char appends = APPENDS;
1514
1515	assert(iter != NULL);
1516
1517	if (self->proto == 0) {
1518	/* APPENDS isn't available; do one at a time. */
1519	for (;;) {
1520	obj = PyIter_Next(iter);
1521	if (obj == NULL) {
1522	if (PyErr_Occurred())
1523	return -1;
1524	break;
1525	}
1526	i = save(self, obj, 0);
1527	Py_DECREF(obj);
1528	if (i < 0)
1529	return -1;
1530	if (self->write_func(self, &append, 1) < 0)
1531	return -1;
1532	}
1533	return 0;
1534	}
1535
1536	/* proto > 0: write in batches of BATCHSIZE. */
1537	do {
1538	/* Get next group of (no more than) BATCHSIZE elements. */
1539	for (n = 0; n < BATCHSIZE; ++n) {
1540	obj = PyIter_Next(iter);
1541	if (obj == NULL) {
1542	if (PyErr_Occurred())
1543	goto BatchFailed;
1544	break;
1545	}
1546	slice[n] = obj;
1547	}
1548
1549	if (n > 1) {
1550	/* Pump out MARK, slice[0:n], APPENDS. */
1551	if (self->write_func(self, &MARKv, 1) < 0)
1552	goto BatchFailed;
1553	for (i = 0; i < n; ++i) {
1554	if (save(self, slice[i], 0) < 0)
1555	goto BatchFailed;
1556	}
1557	if (self->write_func(self, &appends, 1) < 0)
1558	goto BatchFailed;
1559	}
1560	else if (n == 1) {
1561	if (save(self, slice[0], 0) < 0)
1562	goto BatchFailed;
1563	if (self->write_func(self, &append, 1) < 0)
1564	goto BatchFailed;
1565	}
1566
1567	for (i = 0; i < n; ++i) {
1568	Py_DECREF(slice[i]);
1569	}
1570	} while (n == BATCHSIZE);
1571	return 0;
1572
1573	BatchFailed:
1574	while (--n >= 0) {
1575	Py_DECREF(slice[n]);
1576	}
1577	return -1;
1578	}
1579
1580	static int
1581	save_list(Picklerobject self, PyObject args)
1582	{
1583	int res = -1;
1584	char s[3];
1585	int len;
1586	PyObject *iter;
1587
1588	if (self->fast && !fast_save_enter(self, args))
1589	goto finally;
1590
1591	/* Create an empty list. */
1592	if (self->bin) {
1593	s[0] = EMPTY_LIST;
1594	len = 1;
1595	}
1596	else {
1597	s[0] = MARK;
1598	s[1] = LIST;
1599	len = 2;
1600	}
1601
1602	if (self->write_func(self, s, len) < 0)
1603	goto finally;
1604
1605	/* Get list length, and bow out early if empty. */
1606	if ((len = PyList_Size(args)) < 0)
1607	goto finally;
1608
1609	/* Memoize. */
1610	if (len == 0) {
1611	if (put(self, args) >= 0)
1612	res = 0;
1613	goto finally;
1614	}
1615	if (put2(self, args) < 0)
1616	goto finally;
1617
1618	/* Materialize the list elements. */
1619	iter = PyObject_GetIter(args);
1620	if (iter == NULL)
1621	goto finally;
1622	res = batch_list(self, iter);
1623	Py_DECREF(iter);
1624
1625	finally:
1626	if (self->fast && !fast_save_leave(self, args))
1627	res = -1;
1628
1629	return res;
1630	}
1631
1632
1633	/* iter is an iterator giving (key, value) pairs, and we batch up chunks of
1634	* MARK key value ... key value SETITEMS
1635	* opcode sequences. Calling code should have arranged to first create an
1636	* empty dict, or dict-like object, for the SETITEMS to operate on.
1637	* Returns 0 on success, <0 on error.
1638	*
1639	* This is very much like batch_list(). The difference between saving
1640	* elements directly, and picking apart two-tuples, is so long-winded at
1641	* the C level, though, that attempts to combine these routines were too
1642	* ugly to bear.
1643	*/
1644	static int
1645	batch_dict(Picklerobject self, PyObject iter)
1646	{
1647	PyObject *p;
1648	PyObject *slice[BATCHSIZE];
1649	int i, n;
1650
1651	static char setitem = SETITEM;
1652	static char setitems = SETITEMS;
1653
1654	assert(iter != NULL);
1655
1656	if (self->proto == 0) {
1657	/* SETITEMS isn't available; do one at a time. */
1658	for (;;) {
1659	p = PyIter_Next(iter);
1660	if (p == NULL) {
1661	if (PyErr_Occurred())
1662	return -1;
1663	break;
1664	}
1665	if (!PyTuple_Check(p) \|\| PyTuple_Size(p) != 2) {
1666	PyErr_SetString(PyExc_TypeError, "dict items "
1667	"iterator must return 2-tuples");
1668	return -1;
1669	}
1670	i = save(self, PyTuple_GET_ITEM(p, 0), 0);
1671	if (i >= 0)
1672	i = save(self, PyTuple_GET_ITEM(p, 1), 0);
1673	Py_DECREF(p);
1674	if (i < 0)
1675	return -1;
1676	if (self->write_func(self, &setitem, 1) < 0)
1677	return -1;
1678	}
1679	return 0;
1680	}
1681
1682	/* proto > 0: write in batches of BATCHSIZE. */
1683	do {
1684	/* Get next group of (no more than) BATCHSIZE elements. */
1685	for (n = 0; n < BATCHSIZE; ++n) {
1686	p = PyIter_Next(iter);
1687	if (p == NULL) {
1688	if (PyErr_Occurred())
1689	goto BatchFailed;
1690	break;
1691	}
1692	if (!PyTuple_Check(p) \|\| PyTuple_Size(p) != 2) {
1693	PyErr_SetString(PyExc_TypeError, "dict items "
1694	"iterator must return 2-tuples");
1695	goto BatchFailed;
1696	}
1697	slice[n] = p;
1698	}
1699
1700	if (n > 1) {
1701	/* Pump out MARK, slice[0:n], SETITEMS. */
1702	if (self->write_func(self, &MARKv, 1) < 0)
1703	goto BatchFailed;
1704	for (i = 0; i < n; ++i) {
1705	p = slice[i];
1706	if (save(self, PyTuple_GET_ITEM(p, 0), 0) < 0)
1707	goto BatchFailed;
1708	if (save(self, PyTuple_GET_ITEM(p, 1), 0) < 0)
1709	goto BatchFailed;
1710	}
1711	if (self->write_func(self, &setitems, 1) < 0)
1712	goto BatchFailed;
1713	}
1714	else if (n == 1) {
1715	p = slice[0];
1716	if (save(self, PyTuple_GET_ITEM(p, 0), 0) < 0)
1717	goto BatchFailed;
1718	if (save(self, PyTuple_GET_ITEM(p, 1), 0) < 0)
1719	goto BatchFailed;
1720	if (self->write_func(self, &setitem, 1) < 0)
1721	goto BatchFailed;
1722	}
1723
1724	for (i = 0; i < n; ++i) {
1725	Py_DECREF(slice[i]);
1726	}
1727	} while (n == BATCHSIZE);
1728	return 0;
1729
1730	BatchFailed:
1731	while (--n >= 0) {
1732	Py_DECREF(slice[n]);
1733	}
1734	return -1;
1735	}
1736
1737	static int
1738	save_dict(Picklerobject self, PyObject args)
1739	{
1740	int res = -1;
1741	char s[3];
1742	int len;
1743	PyObject *iter;
1744
1745	if (self->fast && !fast_save_enter(self, args))
1746	goto finally;
1747
1748	/* Create an empty dict. */
1749	if (self->bin) {
1750	s[0] = EMPTY_DICT;
1751	len = 1;
1752	}
1753	else {
1754	s[0] = MARK;
1755	s[1] = DICT;
1756	len = 2;
1757	}
1758
1759	if (self->write_func(self, s, len) < 0)
1760	goto finally;
1761
1762	/* Get dict size, and bow out early if empty. */
1763	if ((len = PyDict_Size(args)) < 0)
1764	goto finally;
1765
1766	if (len == 0) {
1767	if (put(self, args) >= 0)
1768	res = 0;
1769	goto finally;
1770	}
1771	if (put2(self, args) < 0)
1772	goto finally;
1773
1774	/* Materialize the dict items. */
1775	iter = PyObject_CallMethod(args, "iteritems", "()");
1776	if (iter == NULL)
1777	goto finally;
1778	res = batch_dict(self, iter);
1779	Py_DECREF(iter);
1780
1781	finally:
1782	if (self->fast && !fast_save_leave(self, args))
1783	res = -1;
1784
1785	return res;
1786	}
1787
1788
1789	static int
1790	save_inst(Picklerobject self, PyObject args)
1791	{
1792	PyObject class = 0, module = 0, name = 0, state = 0,
1793	getinitargs_func = 0, getstate_func = 0, *class_args = 0;
1794	char module_str, name_str;
1795	int module_size, name_size, res = -1;
1796
1797	static char inst = INST, obj = OBJ, build = BUILD;
1798
1799	if (self->fast && !fast_save_enter(self, args))
1800	goto finally;
1801
1802	if (self->write_func(self, &MARKv, 1) < 0)
1803	goto finally;
1804
1805	if (!( class = PyObject_GetAttr(args, __class___str)))
1806	goto finally;
1807
1808	if (self->bin) {
1809	if (save(self, class, 0) < 0)
1810	goto finally;
1811	}
1812
1813	if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) {
1814	PyObject *element = 0;
1815	int i, len;
1816
1817	if (!( class_args =
1818	PyObject_Call(getinitargs_func, empty_tuple, NULL)))
1819	goto finally;
1820
1821	if ((len = PyObject_Size(class_args)) < 0)
1822	goto finally;
1823
1824	for (i = 0; i < len; i++) {
1825	if (!( element = PySequence_GetItem(class_args, i)))
1826	goto finally;
1827
1828	if (save(self, element, 0) < 0) {
1829	Py_DECREF(element);
1830	goto finally;
1831	}
1832
1833	Py_DECREF(element);
1834	}
1835	}
1836	else {
1837	if (PyErr_ExceptionMatches(PyExc_AttributeError))
1838	PyErr_Clear();
1839	else
1840	goto finally;
1841	}
1842
1843	if (!self->bin) {
1844	if (!( name = ((PyClassObject *)class)->cl_name )) {
1845	PyErr_SetString(PicklingError, "class has no name");
1846	goto finally;
1847	}
1848
1849	if (!( module = whichmodule(class, name)))
1850	goto finally;
1851
1852
1853	if ((module_size = PyString_Size(module)) < 0 \|\|
1854	(name_size = PyString_Size(name)) < 0)
1855	goto finally;
1856
1857	module_str = PyString_AS_STRING((PyStringObject *)module);
1858	name_str = PyString_AS_STRING((PyStringObject *)name);
1859
1860	if (self->write_func(self, &inst, 1) < 0)
1861	goto finally;
1862
1863	if (self->write_func(self, module_str, module_size) < 0)
1864	goto finally;
1865
1866	if (self->write_func(self, "\n", 1) < 0)
1867	goto finally;
1868
1869	if (self->write_func(self, name_str, name_size) < 0)
1870	goto finally;
1871
1872	if (self->write_func(self, "\n", 1) < 0)
1873	goto finally;
1874	}
1875	else if (self->write_func(self, &obj, 1) < 0) {
1876	goto finally;
1877	}
1878
1879	if ((getstate_func = PyObject_GetAttr(args, __getstate___str))) {
1880	state = PyObject_Call(getstate_func, empty_tuple, NULL);
1881	if (!state)
1882	goto finally;
1883	}
1884	else {
1885	if (PyErr_ExceptionMatches(PyExc_AttributeError))
1886	PyErr_Clear();
1887	else
1888	goto finally;
1889
1890	if (!( state = PyObject_GetAttr(args, __dict___str))) {
1891	if (PyErr_ExceptionMatches(PyExc_AttributeError))
1892	PyErr_Clear();
1893	else
1894	goto finally;
1895	res = 0;
1896	goto finally;
1897	}
1898	}
1899
1900	if (!PyDict_Check(state)) {
1901	if (put2(self, args) < 0)
1902	goto finally;
1903	}
1904	else {
1905	if (put(self, args) < 0)
1906	goto finally;
1907	}
1908
1909	if (save(self, state, 0) < 0)
1910	goto finally;
1911
1912	if (self->write_func(self, &build, 1) < 0)
1913	goto finally;
1914
1915	res = 0;
1916
1917	finally:
1918	if (self->fast && !fast_save_leave(self, args))
1919	res = -1;
1920
1921	Py_XDECREF(module);
1922	Py_XDECREF(class);
1923	Py_XDECREF(state);
1924	Py_XDECREF(getinitargs_func);
1925	Py_XDECREF(getstate_func);
1926	Py_XDECREF(class_args);
1927
1928	return res;
1929	}
1930
1931
1932	static int
1933	save_global(Picklerobject self, PyObject args, PyObject *name)
1934	{
1935	PyObject global_name = 0, module = 0, mod = 0, klass = 0;
1936	char name_str, module_str;
1937	int module_size, name_size, res = -1;
1938
1939	static char global = GLOBAL;
1940
1941	if (name) {
1942	global_name = name;
1943	Py_INCREF(global_name);
1944	}
1945	else {
1946	if (!( global_name = PyObject_GetAttr(args, __name___str)))
1947	goto finally;
1948	}
1949
1950	if (!( module = whichmodule(args, global_name)))
1951	goto finally;
1952
1953	if ((module_size = PyString_Size(module)) < 0 \|\|
1954	(name_size = PyString_Size(global_name)) < 0)
1955	goto finally;
1956
1957	module_str = PyString_AS_STRING((PyStringObject *)module);
1958	name_str = PyString_AS_STRING((PyStringObject *)global_name);
1959
1960	/* XXX This can be doing a relative import. Clearly it shouldn't,
1961	but I don't know how to stop it. :-( */
1962	mod = PyImport_ImportModule(module_str);
1963	if (mod == NULL) {
1964	cPickle_ErrFormat(PicklingError,
1965	"Can't pickle %s: import of module %s "
1966	"failed",
1967	"OS", args, module);
1968	goto finally;
1969	}
1970	klass = PyObject_GetAttrString(mod, name_str);
1971	if (klass == NULL) {
1972	cPickle_ErrFormat(PicklingError,
1973	"Can't pickle %s: attribute lookup %s.%s "
1974	"failed",
1975	"OSS", args, module, global_name);
1976	goto finally;
1977	}
1978	if (klass != args) {
1979	Py_DECREF(klass);
1980	cPickle_ErrFormat(PicklingError,
1981	"Can't pickle %s: it's not the same object "
1982	"as %s.%s",
1983	"OSS", args, module, global_name);
1984	goto finally;
1985	}
1986	Py_DECREF(klass);
1987
1988	if (self->proto >= 2) {
1989	/* See whether this is in the extension registry, and if
1990	* so generate an EXT opcode.
1991	*/
1992	PyObject py_code; / extension code as Python object */
1993	long code; /* extension code as C value */
1994	char c_str[5];
1995	int n;
1996
1997	PyTuple_SET_ITEM(two_tuple, 0, module);
1998	PyTuple_SET_ITEM(two_tuple, 1, global_name);
1999	py_code = PyDict_GetItem(extension_registry, two_tuple);
2000	if (py_code == NULL)
2001	goto gen_global; /* not registered */
2002
2003	/* Verify py_code has the right type and value. */
2004	if (!PyInt_Check(py_code)) {
2005	cPickle_ErrFormat(PicklingError, "Can't pickle %s: "
2006	"extension code %s isn't an integer",
2007	"OO", args, py_code);
2008	goto finally;
2009	}
2010	code = PyInt_AS_LONG(py_code);
2011	if (code <= 0 \|\| code > 0x7fffffffL) {
2012	cPickle_ErrFormat(PicklingError, "Can't pickle %s: "
2013	"extension code %ld is out of range",
2014	"Ol", args, code);
2015	goto finally;
2016	}
2017
2018	/* Generate an EXT opcode. */
2019	if (code <= 0xff) {
2020	c_str[0] = EXT1;
2021	c_str[1] = (char)code;
2022	n = 2;
2023	}
2024	else if (code <= 0xffff) {
2025	c_str[0] = EXT2;
2026	c_str[1] = (char)(code & 0xff);
2027	c_str[2] = (char)((code >> 8) & 0xff);
2028	n = 3;
2029	}
2030	else {
2031	c_str[0] = EXT4;
2032	c_str[1] = (char)(code & 0xff);
2033	c_str[2] = (char)((code >> 8) & 0xff);
2034	c_str[3] = (char)((code >> 16) & 0xff);
2035	c_str[4] = (char)((code >> 24) & 0xff);
2036	n = 5;
2037	}
2038
2039	if (self->write_func(self, c_str, n) >= 0)
2040	res = 0;
2041	goto finally; /* and don't memoize */
2042	}
2043
2044	gen_global:
2045	if (self->write_func(self, &global, 1) < 0)
2046	goto finally;
2047
2048	if (self->write_func(self, module_str, module_size) < 0)
2049	goto finally;
2050
2051	if (self->write_func(self, "\n", 1) < 0)
2052	goto finally;
2053
2054	if (self->write_func(self, name_str, name_size) < 0)
2055	goto finally;
2056
2057	if (self->write_func(self, "\n", 1) < 0)
2058	goto finally;
2059
2060	if (put(self, args) < 0)
2061	goto finally;
2062
2063	res = 0;
2064
2065	finally:
2066	Py_XDECREF(module);
2067	Py_XDECREF(global_name);
2068	Py_XDECREF(mod);
2069
2070	return res;
2071	}
2072
2073	static int
2074	save_pers(Picklerobject self, PyObject args, PyObject *f)
2075	{
2076	PyObject *pid = 0;
2077	int size, res = -1;
2078
2079	static char persid = PERSID, binpersid = BINPERSID;
2080
2081	Py_INCREF(args);
2082	ARG_TUP(self, args);
2083	if (self->arg) {
2084	pid = PyObject_Call(f, self->arg, NULL);
2085	FREE_ARG_TUP(self);
2086	}
2087	if (! pid) return -1;
2088
2089	if (pid != Py_None) {
2090	if (!self->bin) {
2091	if (!PyString_Check(pid)) {
2092	PyErr_SetString(PicklingError,
2093	"persistent id must be string");
2094	goto finally;
2095	}
2096
2097	if (self->write_func(self, &persid, 1) < 0)
2098	goto finally;
2099
2100	if ((size = PyString_Size(pid)) < 0)
2101	goto finally;
2102
2103	if (self->write_func(self,
2104	PyString_AS_STRING(
2105	(PyStringObject *)pid),
2106	size) < 0)
2107	goto finally;
2108
2109	if (self->write_func(self, "\n", 1) < 0)
2110	goto finally;
2111
2112	res = 1;
2113	goto finally;
2114	}
2115	else if (save(self, pid, 1) >= 0) {
2116	if (self->write_func(self, &binpersid, 1) < 0)
2117	res = -1;
2118	else
2119	res = 1;
2120	}
2121
2122	goto finally;
2123	}
2124
2125	res = 0;
2126
2127	finally:
2128	Py_XDECREF(pid);
2129
2130	return res;
2131	}
2132
2133	/* We're saving ob, and args is the 2-thru-5 tuple returned by the
2134	* appropriate __reduce__ method for ob.
2135	*/
2136	static int
2137	save_reduce(Picklerobject self, PyObject args, PyObject *ob)
2138	{
2139	PyObject *callable;
2140	PyObject *argtup;
2141	PyObject *state = NULL;
2142	PyObject *listitems = NULL;
2143	PyObject *dictitems = NULL;
2144
2145	int use_newobj = self->proto >= 2;
2146
2147	static char reduce = REDUCE;
2148	static char build = BUILD;
2149	static char newobj = NEWOBJ;
2150
2151	if (! PyArg_UnpackTuple(args, "save_reduce", 2, 5,
2152	&callable,
2153	&argtup,
2154	&state,
2155	&listitems,
2156	&dictitems))
2157	return -1;
2158
2159	if (!PyTuple_Check(argtup)) {
2160	PyErr_SetString(PicklingError,
2161	"args from reduce() should be a tuple");
2162	return -1;
2163	}
2164
2165	if (state == Py_None)
2166	state = NULL;
2167	if (listitems == Py_None)
2168	listitems = NULL;
2169	if (dictitems == Py_None)
2170	dictitems = NULL;
2171
2172	/* Protocol 2 special case: if callable's name is __newobj__, use
2173	* NEWOBJ. This consumes a lot of code.
2174	*/
2175	if (use_newobj) {
2176	PyObject *temp = PyObject_GetAttr(callable, __name___str);
2177
2178	if (temp == NULL) {
2179	if (PyErr_ExceptionMatches(PyExc_AttributeError))
2180	PyErr_Clear();
2181	else
2182	return -1;
2183	use_newobj = 0;
2184	}
2185	else {
2186	use_newobj = PyString_Check(temp) &&
2187	strcmp(PyString_AS_STRING(temp),
2188	"__newobj__") == 0;
2189	Py_DECREF(temp);
2190	}
2191	}
2192	if (use_newobj) {
2193	PyObject *cls;
2194	PyObject *newargtup;
2195	int n, i;
2196
2197	/* Sanity checks. */
2198	n = PyTuple_Size(argtup);
2199	if (n < 1) {
2200	PyErr_SetString(PicklingError, "__newobj__ arglist "
2201	"is empty");
2202	return -1;
2203	}
2204
2205	cls = PyTuple_GET_ITEM(argtup, 0);
2206	if (! PyObject_HasAttrString(cls, "__new__")) {
2207	PyErr_SetString(PicklingError, "args[0] from "
2208	"__newobj__ args has no __new__");
2209	return -1;
2210	}
2211
2212	/* XXX How could ob be NULL? */
2213	if (ob != NULL) {
2214	PyObject *ob_dot_class;
2215
2216	ob_dot_class = PyObject_GetAttr(ob, __class___str);
2217	if (ob_dot_class == NULL) {
2218	if (PyErr_ExceptionMatches(
2219	PyExc_AttributeError))
2220	PyErr_Clear();
2221	else
2222	return -1;
2223	}
2224	i = ob_dot_class != cls; /* true iff a problem */
2225	Py_XDECREF(ob_dot_class);
2226	if (i) {
2227	PyErr_SetString(PicklingError, "args[0] from "
2228	"__newobj__ args has the wrong class");
2229	return -1;
2230	}
2231	}
2232
2233	/* Save the class and its __new__ arguments. */
2234	if (save(self, cls, 0) < 0)
2235	return -1;
2236
2237	newargtup = PyTuple_New(n-1); /* argtup[1:] */
2238	if (newargtup == NULL)
2239	return -1;
2240	for (i = 1; i < n; ++i) {
2241	PyObject *temp = PyTuple_GET_ITEM(argtup, i);
2242	Py_INCREF(temp);
2243	PyTuple_SET_ITEM(newargtup, i-1, temp);
2244	}
2245	i = save(self, newargtup, 0) < 0;
2246	Py_DECREF(newargtup);
2247	if (i < 0)
2248	return -1;
2249
2250	/* Add NEWOBJ opcode. */
2251	if (self->write_func(self, &newobj, 1) < 0)
2252	return -1;
2253	}
2254	else {
2255	/* Not using NEWOBJ. */
2256	if (save(self, callable, 0) < 0 \|\|
2257	save(self, argtup, 0) < 0 \|\|
2258	self->write_func(self, &reduce, 1) < 0)
2259	return -1;
2260	}
2261
2262	/* Memoize. */
2263	/* XXX How can ob be NULL? */
2264	if (ob != NULL) {
2265	if (state && !PyDict_Check(state)) {
2266	if (put2(self, ob) < 0)
2267	return -1;
2268	}
2269	else if (put(self, ob) < 0)
2270	return -1;
2271	}
2272
2273
2274	if (listitems && batch_list(self, listitems) < 0)
2275	return -1;
2276
2277	if (dictitems && batch_dict(self, dictitems) < 0)
2278	return -1;
2279
2280	if (state) {
2281	if (save(self, state, 0) < 0 \|\|
2282	self->write_func(self, &build, 1) < 0)
2283	return -1;
2284	}
2285
2286	return 0;
2287	}
2288
2289	static int
2290	save(Picklerobject self, PyObject args, int pers_save)
2291	{
2292	PyTypeObject *type;
2293	PyObject py_ob_id = 0, __reduce__ = 0, *t = 0;
2294	PyObject *arg_tup;
2295	int res = -1;
2296	int tmp, size;
2297
2298	if (self->nesting++ > Py_GetRecursionLimit()){
2299	PyErr_SetString(PyExc_RuntimeError,
2300	"maximum recursion depth exceeded");
2301	goto finally;
2302	}
2303
2304	if (!pers_save && self->pers_func) {
2305	if ((tmp = save_pers(self, args, self->pers_func)) != 0) {
2306	res = tmp;
2307	goto finally;
2308	}
2309	}
2310
2311	if (args == Py_None) {
2312	res = save_none(self, args);
2313	goto finally;
2314	}
2315
2316	type = args->ob_type;
2317
2318	switch (type->tp_name[0]) {
2319	case 'b':
2320	if (args == Py_False \|\| args == Py_True) {
2321	res = save_bool(self, args);