Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Blame
Revision Log

xmltok_impl.c@ 3506

Visit:

Last change on this file since 3506 was 3225, checked in by bird, 19 years ago
Python 2.5
File size: 43.4 KB

Line
1	/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd
2	See the file COPYING for copying permission.
3	*/
4
5	#ifndef IS_INVALID_CHAR
6	#define IS_INVALID_CHAR(enc, ptr, n) (0)
7	#endif
8
9	#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
10	case BT_LEAD ## n: \
11	if (end - ptr < n) \
12	return XML_TOK_PARTIAL_CHAR; \
13	if (IS_INVALID_CHAR(enc, ptr, n)) { \
14	*(nextTokPtr) = (ptr); \
15	return XML_TOK_INVALID; \
16	} \
17	ptr += n; \
18	break;
19
20	#define INVALID_CASES(ptr, nextTokPtr) \
21	INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
22	INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
23	INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
24	case BT_NONXML: \
25	case BT_MALFORM: \
26	case BT_TRAIL: \
27	*(nextTokPtr) = (ptr); \
28	return XML_TOK_INVALID;
29
30	#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
31	case BT_LEAD ## n: \
32	if (end - ptr < n) \
33	return XML_TOK_PARTIAL_CHAR; \
34	if (!IS_NAME_CHAR(enc, ptr, n)) { \
35	*nextTokPtr = ptr; \
36	return XML_TOK_INVALID; \
37	} \
38	ptr += n; \
39	break;
40
41	#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
42	case BT_NONASCII: \
43	if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
44	*nextTokPtr = ptr; \
45	return XML_TOK_INVALID; \
46	} \
47	case BT_NMSTRT: \
48	case BT_HEX: \
49	case BT_DIGIT: \
50	case BT_NAME: \
51	case BT_MINUS: \
52	ptr += MINBPC(enc); \
53	break; \
54	CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
55	CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
56	CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
57
58	#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
59	case BT_LEAD ## n: \
60	if (end - ptr < n) \
61	return XML_TOK_PARTIAL_CHAR; \
62	if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
63	*nextTokPtr = ptr; \
64	return XML_TOK_INVALID; \
65	} \
66	ptr += n; \
67	break;
68
69	#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
70	case BT_NONASCII: \
71	if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
72	*nextTokPtr = ptr; \
73	return XML_TOK_INVALID; \
74	} \
75	case BT_NMSTRT: \
76	case BT_HEX: \
77	ptr += MINBPC(enc); \
78	break; \
79	CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
80	CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
81	CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
82
83	#ifndef PREFIX
84	#define PREFIX(ident) ident
85	#endif
86
87	/* ptr points to character following "<!-" */
88
89	static int PTRCALL
90	PREFIX(scanComment)(const ENCODING enc, const char ptr,
91	const char end, const char *nextTokPtr)
92	{
93	if (ptr != end) {
94	if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
95	*nextTokPtr = ptr;
96	return XML_TOK_INVALID;
97	}
98	ptr += MINBPC(enc);
99	while (ptr != end) {
100	switch (BYTE_TYPE(enc, ptr)) {
101	INVALID_CASES(ptr, nextTokPtr)
102	case BT_MINUS:
103	if ((ptr += MINBPC(enc)) == end)
104	return XML_TOK_PARTIAL;
105	if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
106	if ((ptr += MINBPC(enc)) == end)
107	return XML_TOK_PARTIAL;
108	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
109	*nextTokPtr = ptr;
110	return XML_TOK_INVALID;
111	}
112	*nextTokPtr = ptr + MINBPC(enc);
113	return XML_TOK_COMMENT;
114	}
115	break;
116	default:
117	ptr += MINBPC(enc);
118	break;
119	}
120	}
121	}
122	return XML_TOK_PARTIAL;
123	}
124
125	/* ptr points to character following "<!" */
126
127	static int PTRCALL
128	PREFIX(scanDecl)(const ENCODING enc, const char ptr,
129	const char end, const char *nextTokPtr)
130	{
131	if (ptr == end)
132	return XML_TOK_PARTIAL;
133	switch (BYTE_TYPE(enc, ptr)) {
134	case BT_MINUS:
135	return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
136	case BT_LSQB:
137	*nextTokPtr = ptr + MINBPC(enc);
138	return XML_TOK_COND_SECT_OPEN;
139	case BT_NMSTRT:
140	case BT_HEX:
141	ptr += MINBPC(enc);
142	break;
143	default:
144	*nextTokPtr = ptr;
145	return XML_TOK_INVALID;
146	}
147	while (ptr != end) {
148	switch (BYTE_TYPE(enc, ptr)) {
149	case BT_PERCNT:
150	if (ptr + MINBPC(enc) == end)
151	return XML_TOK_PARTIAL;
152	/* don't allow <!ENTITY% foo "whatever"> */
153	switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
154	case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
155	*nextTokPtr = ptr;
156	return XML_TOK_INVALID;
157	}
158	/* fall through */
159	case BT_S: case BT_CR: case BT_LF:
160	*nextTokPtr = ptr;
161	return XML_TOK_DECL_OPEN;
162	case BT_NMSTRT:
163	case BT_HEX:
164	ptr += MINBPC(enc);
165	break;
166	default:
167	*nextTokPtr = ptr;
168	return XML_TOK_INVALID;
169	}
170	}
171	return XML_TOK_PARTIAL;
172	}
173
174	static int PTRCALL
175	PREFIX(checkPiTarget)(const ENCODING enc, const char ptr,
176	const char end, int tokPtr)
177	{
178	int upper = 0;
179	*tokPtr = XML_TOK_PI;
180	if (end - ptr != MINBPC(enc)*3)
181	return 1;
182	switch (BYTE_TO_ASCII(enc, ptr)) {
183	case ASCII_x:
184	break;
185	case ASCII_X:
186	upper = 1;
187	break;
188	default:
189	return 1;
190	}
191	ptr += MINBPC(enc);
192	switch (BYTE_TO_ASCII(enc, ptr)) {
193	case ASCII_m:
194	break;
195	case ASCII_M:
196	upper = 1;
197	break;
198	default:
199	return 1;
200	}
201	ptr += MINBPC(enc);
202	switch (BYTE_TO_ASCII(enc, ptr)) {
203	case ASCII_l:
204	break;
205	case ASCII_L:
206	upper = 1;
207	break;
208	default:
209	return 1;
210	}
211	if (upper)
212	return 0;
213	*tokPtr = XML_TOK_XML_DECL;
214	return 1;
215	}
216
217	/* ptr points to character following "<?" */
218
219	static int PTRCALL
220	PREFIX(scanPi)(const ENCODING enc, const char ptr,
221	const char end, const char *nextTokPtr)
222	{
223	int tok;
224	const char *target = ptr;
225	if (ptr == end)
226	return XML_TOK_PARTIAL;
227	switch (BYTE_TYPE(enc, ptr)) {
228	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
229	default:
230	*nextTokPtr = ptr;
231	return XML_TOK_INVALID;
232	}
233	while (ptr != end) {
234	switch (BYTE_TYPE(enc, ptr)) {
235	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
236	case BT_S: case BT_CR: case BT_LF:
237	if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
238	*nextTokPtr = ptr;
239	return XML_TOK_INVALID;
240	}
241	ptr += MINBPC(enc);
242	while (ptr != end) {
243	switch (BYTE_TYPE(enc, ptr)) {
244	INVALID_CASES(ptr, nextTokPtr)
245	case BT_QUEST:
246	ptr += MINBPC(enc);
247	if (ptr == end)
248	return XML_TOK_PARTIAL;
249	if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {
250	*nextTokPtr = ptr + MINBPC(enc);
251	return tok;
252	}
253	break;
254	default:
255	ptr += MINBPC(enc);
256	break;
257	}
258	}
259	return XML_TOK_PARTIAL;
260	case BT_QUEST:
261	if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
262	*nextTokPtr = ptr;
263	return XML_TOK_INVALID;
264	}
265	ptr += MINBPC(enc);
266	if (ptr == end)
267	return XML_TOK_PARTIAL;
268	if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {
269	*nextTokPtr = ptr + MINBPC(enc);
270	return tok;
271	}
272	/* fall through */
273	default:
274	*nextTokPtr = ptr;
275	return XML_TOK_INVALID;
276	}
277	}
278	return XML_TOK_PARTIAL;
279	}
280
281	static int PTRCALL
282	PREFIX(scanCdataSection)(const ENCODING enc, const char ptr,
283	const char end, const char *nextTokPtr)
284	{
285	static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
286	ASCII_T, ASCII_A, ASCII_LSQB };
287	int i;
288	/* CDATA[ */
289	if (end - ptr < 6 * MINBPC(enc))
290	return XML_TOK_PARTIAL;
291	for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
292	if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
293	*nextTokPtr = ptr;
294	return XML_TOK_INVALID;
295	}
296	}
297	*nextTokPtr = ptr;
298	return XML_TOK_CDATA_SECT_OPEN;
299	}
300
301	static int PTRCALL
302	PREFIX(cdataSectionTok)(const ENCODING enc, const char ptr,
303	const char end, const char *nextTokPtr)
304	{
305	if (ptr == end)
306	return XML_TOK_NONE;
307	if (MINBPC(enc) > 1) {
308	size_t n = end - ptr;
309	if (n & (MINBPC(enc) - 1)) {
310	n &= ~(MINBPC(enc) - 1);
311	if (n == 0)
312	return XML_TOK_PARTIAL;
313	end = ptr + n;
314	}
315	}
316	switch (BYTE_TYPE(enc, ptr)) {
317	case BT_RSQB:
318	ptr += MINBPC(enc);
319	if (ptr == end)
320	return XML_TOK_PARTIAL;
321	if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
322	break;
323	ptr += MINBPC(enc);
324	if (ptr == end)
325	return XML_TOK_PARTIAL;
326	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
327	ptr -= MINBPC(enc);
328	break;
329	}
330	*nextTokPtr = ptr + MINBPC(enc);
331	return XML_TOK_CDATA_SECT_CLOSE;
332	case BT_CR:
333	ptr += MINBPC(enc);
334	if (ptr == end)
335	return XML_TOK_PARTIAL;
336	if (BYTE_TYPE(enc, ptr) == BT_LF)
337	ptr += MINBPC(enc);
338	*nextTokPtr = ptr;
339	return XML_TOK_DATA_NEWLINE;
340	case BT_LF:
341	*nextTokPtr = ptr + MINBPC(enc);
342	return XML_TOK_DATA_NEWLINE;
343	INVALID_CASES(ptr, nextTokPtr)
344	default:
345	ptr += MINBPC(enc);
346	break;
347	}
348	while (ptr != end) {
349	switch (BYTE_TYPE(enc, ptr)) {
350	#define LEAD_CASE(n) \
351	case BT_LEAD ## n: \
352	if (end - ptr < n \|\| IS_INVALID_CHAR(enc, ptr, n)) { \
353	*nextTokPtr = ptr; \
354	return XML_TOK_DATA_CHARS; \
355	} \
356	ptr += n; \
357	break;
358	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
359	#undef LEAD_CASE
360	case BT_NONXML:
361	case BT_MALFORM:
362	case BT_TRAIL:
363	case BT_CR:
364	case BT_LF:
365	case BT_RSQB:
366	*nextTokPtr = ptr;
367	return XML_TOK_DATA_CHARS;
368	default:
369	ptr += MINBPC(enc);
370	break;
371	}
372	}
373	*nextTokPtr = ptr;
374	return XML_TOK_DATA_CHARS;
375	}
376
377	/* ptr points to character following "</" */
378
379	static int PTRCALL
380	PREFIX(scanEndTag)(const ENCODING enc, const char ptr,
381	const char end, const char *nextTokPtr)
382	{
383	if (ptr == end)
384	return XML_TOK_PARTIAL;
385	switch (BYTE_TYPE(enc, ptr)) {
386	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
387	default:
388	*nextTokPtr = ptr;
389	return XML_TOK_INVALID;
390	}
391	while (ptr != end) {
392	switch (BYTE_TYPE(enc, ptr)) {
393	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
394	case BT_S: case BT_CR: case BT_LF:
395	for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {
396	switch (BYTE_TYPE(enc, ptr)) {
397	case BT_S: case BT_CR: case BT_LF:
398	break;
399	case BT_GT:
400	*nextTokPtr = ptr + MINBPC(enc);
401	return XML_TOK_END_TAG;
402	default:
403	*nextTokPtr = ptr;
404	return XML_TOK_INVALID;
405	}
406	}
407	return XML_TOK_PARTIAL;
408	#ifdef XML_NS
409	case BT_COLON:
410	/* no need to check qname syntax here,
411	since end-tag must match exactly */
412	ptr += MINBPC(enc);
413	break;
414	#endif
415	case BT_GT:
416	*nextTokPtr = ptr + MINBPC(enc);
417	return XML_TOK_END_TAG;
418	default:
419	*nextTokPtr = ptr;
420	return XML_TOK_INVALID;
421	}
422	}
423	return XML_TOK_PARTIAL;
424	}
425
426	/* ptr points to character following "&#X" */
427
428	static int PTRCALL
429	PREFIX(scanHexCharRef)(const ENCODING enc, const char ptr,
430	const char end, const char *nextTokPtr)
431	{
432	if (ptr != end) {
433	switch (BYTE_TYPE(enc, ptr)) {
434	case BT_DIGIT:
435	case BT_HEX:
436	break;
437	default:
438	*nextTokPtr = ptr;
439	return XML_TOK_INVALID;
440	}
441	for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {
442	switch (BYTE_TYPE(enc, ptr)) {
443	case BT_DIGIT:
444	case BT_HEX:
445	break;
446	case BT_SEMI:
447	*nextTokPtr = ptr + MINBPC(enc);
448	return XML_TOK_CHAR_REF;
449	default:
450	*nextTokPtr = ptr;
451	return XML_TOK_INVALID;
452	}
453	}
454	}
455	return XML_TOK_PARTIAL;
456	}
457
458	/* ptr points to character following "&#" */
459
460	static int PTRCALL
461	PREFIX(scanCharRef)(const ENCODING enc, const char ptr,
462	const char end, const char *nextTokPtr)
463	{
464	if (ptr != end) {
465	if (CHAR_MATCHES(enc, ptr, ASCII_x))
466	return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
467	switch (BYTE_TYPE(enc, ptr)) {
468	case BT_DIGIT:
469	break;
470	default:
471	*nextTokPtr = ptr;
472	return XML_TOK_INVALID;
473	}
474	for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {
475	switch (BYTE_TYPE(enc, ptr)) {
476	case BT_DIGIT:
477	break;
478	case BT_SEMI:
479	*nextTokPtr = ptr + MINBPC(enc);
480	return XML_TOK_CHAR_REF;
481	default:
482	*nextTokPtr = ptr;
483	return XML_TOK_INVALID;
484	}
485	}
486	}
487	return XML_TOK_PARTIAL;
488	}
489
490	/* ptr points to character following "&" */
491
492	static int PTRCALL
493	PREFIX(scanRef)(const ENCODING enc, const char ptr, const char *end,
494	const char **nextTokPtr)
495	{
496	if (ptr == end)
497	return XML_TOK_PARTIAL;
498	switch (BYTE_TYPE(enc, ptr)) {
499	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
500	case BT_NUM:
501	return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
502	default:
503	*nextTokPtr = ptr;
504	return XML_TOK_INVALID;
505	}
506	while (ptr != end) {
507	switch (BYTE_TYPE(enc, ptr)) {
508	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
509	case BT_SEMI:
510	*nextTokPtr = ptr + MINBPC(enc);
511	return XML_TOK_ENTITY_REF;
512	default:
513	*nextTokPtr = ptr;
514	return XML_TOK_INVALID;
515	}
516	}
517	return XML_TOK_PARTIAL;
518	}
519
520	/* ptr points to character following first character of attribute name */
521
522	static int PTRCALL
523	PREFIX(scanAtts)(const ENCODING enc, const char ptr, const char *end,
524	const char **nextTokPtr)
525	{
526	#ifdef XML_NS
527	int hadColon = 0;
528	#endif
529	while (ptr != end) {
530	switch (BYTE_TYPE(enc, ptr)) {
531	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
532	#ifdef XML_NS
533	case BT_COLON:
534	if (hadColon) {
535	*nextTokPtr = ptr;
536	return XML_TOK_INVALID;
537	}
538	hadColon = 1;
539	ptr += MINBPC(enc);
540	if (ptr == end)
541	return XML_TOK_PARTIAL;
542	switch (BYTE_TYPE(enc, ptr)) {
543	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
544	default:
545	*nextTokPtr = ptr;
546	return XML_TOK_INVALID;
547	}
548	break;
549	#endif
550	case BT_S: case BT_CR: case BT_LF:
551	for (;;) {
552	int t;
553
554	ptr += MINBPC(enc);
555	if (ptr == end)
556	return XML_TOK_PARTIAL;
557	t = BYTE_TYPE(enc, ptr);
558	if (t == BT_EQUALS)
559	break;
560	switch (t) {
561	case BT_S:
562	case BT_LF:
563	case BT_CR:
564	break;
565	default:
566	*nextTokPtr = ptr;
567	return XML_TOK_INVALID;
568	}
569	}
570	/* fall through */
571	case BT_EQUALS:
572	{
573	int open;
574	#ifdef XML_NS
575	hadColon = 0;
576	#endif
577	for (;;) {
578	ptr += MINBPC(enc);
579	if (ptr == end)
580	return XML_TOK_PARTIAL;
581	open = BYTE_TYPE(enc, ptr);
582	if (open == BT_QUOT \|\| open == BT_APOS)
583	break;
584	switch (open) {
585	case BT_S:
586	case BT_LF:
587	case BT_CR:
588	break;
589	default:
590	*nextTokPtr = ptr;
591	return XML_TOK_INVALID;
592	}
593	}
594	ptr += MINBPC(enc);
595	/* in attribute value */
596	for (;;) {
597	int t;
598	if (ptr == end)
599	return XML_TOK_PARTIAL;
600	t = BYTE_TYPE(enc, ptr);
601	if (t == open)
602	break;
603	switch (t) {
604	INVALID_CASES(ptr, nextTokPtr)
605	case BT_AMP:
606	{
607	int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
608	if (tok <= 0) {
609	if (tok == XML_TOK_INVALID)
610	*nextTokPtr = ptr;
611	return tok;
612	}
613	break;
614	}
615	case BT_LT:
616	*nextTokPtr = ptr;
617	return XML_TOK_INVALID;
618	default:
619	ptr += MINBPC(enc);
620	break;
621	}
622	}
623	ptr += MINBPC(enc);
624	if (ptr == end)
625	return XML_TOK_PARTIAL;
626	switch (BYTE_TYPE(enc, ptr)) {
627	case BT_S:
628	case BT_CR:
629	case BT_LF:
630	break;
631	case BT_SOL:
632	goto sol;
633	case BT_GT:
634	goto gt;
635	default:
636	*nextTokPtr = ptr;
637	return XML_TOK_INVALID;
638	}
639	/* ptr points to closing quote */
640	for (;;) {
641	ptr += MINBPC(enc);
642	if (ptr == end)
643	return XML_TOK_PARTIAL;
644	switch (BYTE_TYPE(enc, ptr)) {
645	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
646	case BT_S: case BT_CR: case BT_LF:
647	continue;
648	case BT_GT:
649	gt:
650	*nextTokPtr = ptr + MINBPC(enc);
651	return XML_TOK_START_TAG_WITH_ATTS;
652	case BT_SOL:
653	sol:
654	ptr += MINBPC(enc);
655	if (ptr == end)
656	return XML_TOK_PARTIAL;
657	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
658	*nextTokPtr = ptr;
659	return XML_TOK_INVALID;
660	}
661	*nextTokPtr = ptr + MINBPC(enc);
662	return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
663	default:
664	*nextTokPtr = ptr;
665	return XML_TOK_INVALID;
666	}
667	break;
668	}
669	break;
670	}
671	default:
672	*nextTokPtr = ptr;
673	return XML_TOK_INVALID;
674	}
675	}
676	return XML_TOK_PARTIAL;
677	}
678
679	/* ptr points to character following "<" */
680
681	static int PTRCALL
682	PREFIX(scanLt)(const ENCODING enc, const char ptr, const char *end,
683	const char **nextTokPtr)
684	{
685	#ifdef XML_NS
686	int hadColon;
687	#endif
688	if (ptr == end)
689	return XML_TOK_PARTIAL;
690	switch (BYTE_TYPE(enc, ptr)) {
691	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
692	case BT_EXCL:
693	if ((ptr += MINBPC(enc)) == end)
694	return XML_TOK_PARTIAL;
695	switch (BYTE_TYPE(enc, ptr)) {
696	case BT_MINUS:
697	return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
698	case BT_LSQB:
699	return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
700	end, nextTokPtr);
701	}
702	*nextTokPtr = ptr;
703	return XML_TOK_INVALID;
704	case BT_QUEST:
705	return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
706	case BT_SOL:
707	return PREFIX(scanEndTag)(enc, ptr + MINBPC(enc), end, nextTokPtr);
708	default:
709	*nextTokPtr = ptr;
710	return XML_TOK_INVALID;
711	}
712	#ifdef XML_NS
713	hadColon = 0;
714	#endif
715	/* we have a start-tag */
716	while (ptr != end) {
717	switch (BYTE_TYPE(enc, ptr)) {
718	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
719	#ifdef XML_NS
720	case BT_COLON:
721	if (hadColon) {
722	*nextTokPtr = ptr;
723	return XML_TOK_INVALID;
724	}
725	hadColon = 1;
726	ptr += MINBPC(enc);
727	if (ptr == end)
728	return XML_TOK_PARTIAL;
729	switch (BYTE_TYPE(enc, ptr)) {
730	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
731	default:
732	*nextTokPtr = ptr;
733	return XML_TOK_INVALID;
734	}
735	break;
736	#endif
737	case BT_S: case BT_CR: case BT_LF:
738	{
739	ptr += MINBPC(enc);
740	while (ptr != end) {
741	switch (BYTE_TYPE(enc, ptr)) {
742	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
743	case BT_GT:
744	goto gt;
745	case BT_SOL:
746	goto sol;
747	case BT_S: case BT_CR: case BT_LF:
748	ptr += MINBPC(enc);
749	continue;
750	default:
751	*nextTokPtr = ptr;
752	return XML_TOK_INVALID;
753	}
754	return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
755	}
756	return XML_TOK_PARTIAL;
757	}
758	case BT_GT:
759	gt:
760	*nextTokPtr = ptr + MINBPC(enc);
761	return XML_TOK_START_TAG_NO_ATTS;
762	case BT_SOL:
763	sol:
764	ptr += MINBPC(enc);
765	if (ptr == end)
766	return XML_TOK_PARTIAL;
767	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
768	*nextTokPtr = ptr;
769	return XML_TOK_INVALID;
770	}
771	*nextTokPtr = ptr + MINBPC(enc);
772	return XML_TOK_EMPTY_ELEMENT_NO_ATTS;
773	default:
774	*nextTokPtr = ptr;
775	return XML_TOK_INVALID;
776	}
777	}
778	return XML_TOK_PARTIAL;
779	}
780
781	static int PTRCALL
782	PREFIX(contentTok)(const ENCODING enc, const char ptr, const char *end,
783	const char **nextTokPtr)
784	{
785	if (ptr == end)
786	return XML_TOK_NONE;
787	if (MINBPC(enc) > 1) {
788	size_t n = end - ptr;
789	if (n & (MINBPC(enc) - 1)) {
790	n &= ~(MINBPC(enc) - 1);
791	if (n == 0)
792	return XML_TOK_PARTIAL;
793	end = ptr + n;
794	}
795	}
796	switch (BYTE_TYPE(enc, ptr)) {
797	case BT_LT:
798	return PREFIX(scanLt)(enc, ptr + MINBPC(enc), end, nextTokPtr);
799	case BT_AMP:
800	return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
801	case BT_CR:
802	ptr += MINBPC(enc);
803	if (ptr == end)
804	return XML_TOK_TRAILING_CR;
805	if (BYTE_TYPE(enc, ptr) == BT_LF)
806	ptr += MINBPC(enc);
807	*nextTokPtr = ptr;
808	return XML_TOK_DATA_NEWLINE;
809	case BT_LF:
810	*nextTokPtr = ptr + MINBPC(enc);
811	return XML_TOK_DATA_NEWLINE;
812	case BT_RSQB:
813	ptr += MINBPC(enc);
814	if (ptr == end)
815	return XML_TOK_TRAILING_RSQB;
816	if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
817	break;
818	ptr += MINBPC(enc);
819	if (ptr == end)
820	return XML_TOK_TRAILING_RSQB;
821	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
822	ptr -= MINBPC(enc);
823	break;
824	}
825	*nextTokPtr = ptr;
826	return XML_TOK_INVALID;
827	INVALID_CASES(ptr, nextTokPtr)
828	default:
829	ptr += MINBPC(enc);
830	break;
831	}
832	while (ptr != end) {
833	switch (BYTE_TYPE(enc, ptr)) {
834	#define LEAD_CASE(n) \
835	case BT_LEAD ## n: \
836	if (end - ptr < n \|\| IS_INVALID_CHAR(enc, ptr, n)) { \
837	*nextTokPtr = ptr; \
838	return XML_TOK_DATA_CHARS; \
839	} \
840	ptr += n; \
841	break;
842	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
843	#undef LEAD_CASE
844	case BT_RSQB:
845	if (ptr + MINBPC(enc) != end) {
846	if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
847	ptr += MINBPC(enc);
848	break;
849	}
850	if (ptr + 2*MINBPC(enc) != end) {
851	if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
852	ptr += MINBPC(enc);
853	break;
854	}
855	nextTokPtr = ptr + 2MINBPC(enc);
856	return XML_TOK_INVALID;
857	}
858	}
859	/* fall through */
860	case BT_AMP:
861	case BT_LT:
862	case BT_NONXML:
863	case BT_MALFORM:
864	case BT_TRAIL:
865	case BT_CR:
866	case BT_LF:
867	*nextTokPtr = ptr;
868	return XML_TOK_DATA_CHARS;
869	default:
870	ptr += MINBPC(enc);
871	break;
872	}
873	}
874	*nextTokPtr = ptr;
875	return XML_TOK_DATA_CHARS;
876	}
877
878	/* ptr points to character following "%" */
879
880	static int PTRCALL
881	PREFIX(scanPercent)(const ENCODING enc, const char ptr, const char *end,
882	const char **nextTokPtr)
883	{
884	if (ptr == end)
885	return -XML_TOK_PERCENT;
886	switch (BYTE_TYPE(enc, ptr)) {
887	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
888	case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
889	*nextTokPtr = ptr;
890	return XML_TOK_PERCENT;
891	default:
892	*nextTokPtr = ptr;
893	return XML_TOK_INVALID;
894	}
895	while (ptr != end) {
896	switch (BYTE_TYPE(enc, ptr)) {
897	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
898	case BT_SEMI:
899	*nextTokPtr = ptr + MINBPC(enc);
900	return XML_TOK_PARAM_ENTITY_REF;
901	default:
902	*nextTokPtr = ptr;
903	return XML_TOK_INVALID;
904	}
905	}
906	return XML_TOK_PARTIAL;
907	}
908
909	static int PTRCALL
910	PREFIX(scanPoundName)(const ENCODING enc, const char ptr, const char *end,
911	const char **nextTokPtr)
912	{
913	if (ptr == end)
914	return XML_TOK_PARTIAL;
915	switch (BYTE_TYPE(enc, ptr)) {
916	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
917	default:
918	*nextTokPtr = ptr;
919	return XML_TOK_INVALID;
920	}
921	while (ptr != end) {
922	switch (BYTE_TYPE(enc, ptr)) {
923	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
924	case BT_CR: case BT_LF: case BT_S:
925	case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
926	*nextTokPtr = ptr;
927	return XML_TOK_POUND_NAME;
928	default:
929	*nextTokPtr = ptr;
930	return XML_TOK_INVALID;
931	}
932	}
933	return -XML_TOK_POUND_NAME;
934	}
935
936	static int PTRCALL
937	PREFIX(scanLit)(int open, const ENCODING *enc,
938	const char ptr, const char end,
939	const char **nextTokPtr)
940	{
941	while (ptr != end) {
942	int t = BYTE_TYPE(enc, ptr);
943	switch (t) {
944	INVALID_CASES(ptr, nextTokPtr)
945	case BT_QUOT:
946	case BT_APOS:
947	ptr += MINBPC(enc);
948	if (t != open)
949	break;
950	if (ptr == end)
951	return -XML_TOK_LITERAL;
952	*nextTokPtr = ptr;
953	switch (BYTE_TYPE(enc, ptr)) {
954	case BT_S: case BT_CR: case BT_LF:
955	case BT_GT: case BT_PERCNT: case BT_LSQB:
956	return XML_TOK_LITERAL;
957	default:
958	return XML_TOK_INVALID;
959	}
960	default:
961	ptr += MINBPC(enc);
962	break;
963	}
964	}
965	return XML_TOK_PARTIAL;
966	}
967
968	static int PTRCALL
969	PREFIX(prologTok)(const ENCODING enc, const char ptr, const char *end,
970	const char **nextTokPtr)
971	{
972	int tok;
973	if (ptr == end)
974	return XML_TOK_NONE;
975	if (MINBPC(enc) > 1) {
976	size_t n = end - ptr;
977	if (n & (MINBPC(enc) - 1)) {
978	n &= ~(MINBPC(enc) - 1);
979	if (n == 0)
980	return XML_TOK_PARTIAL;
981	end = ptr + n;
982	}
983	}
984	switch (BYTE_TYPE(enc, ptr)) {
985	case BT_QUOT:
986	return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
987	case BT_APOS:
988	return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
989	case BT_LT:
990	{
991	ptr += MINBPC(enc);
992	if (ptr == end)
993	return XML_TOK_PARTIAL;
994	switch (BYTE_TYPE(enc, ptr)) {
995	case BT_EXCL:
996	return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
997	case BT_QUEST:
998	return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
999	case BT_NMSTRT:
1000	case BT_HEX:
1001	case BT_NONASCII:
1002	case BT_LEAD2:
1003	case BT_LEAD3:
1004	case BT_LEAD4:
1005	*nextTokPtr = ptr - MINBPC(enc);
1006	return XML_TOK_INSTANCE_START;
1007	}
1008	*nextTokPtr = ptr;
1009	return XML_TOK_INVALID;
1010	}
1011	case BT_CR:
1012	if (ptr + MINBPC(enc) == end) {
1013	*nextTokPtr = end;
1014	/* indicate that this might be part of a CR/LF pair */
1015	return -XML_TOK_PROLOG_S;
1016	}
1017	/* fall through */
1018	case BT_S: case BT_LF:
1019	for (;;) {
1020	ptr += MINBPC(enc);
1021	if (ptr == end)
1022	break;
1023	switch (BYTE_TYPE(enc, ptr)) {
1024	case BT_S: case BT_LF:
1025	break;
1026	case BT_CR:
1027	/* don't split CR/LF pair */
1028	if (ptr + MINBPC(enc) != end)
1029	break;
1030	/* fall through */
1031	default:
1032	*nextTokPtr = ptr;
1033	return XML_TOK_PROLOG_S;
1034	}
1035	}
1036	*nextTokPtr = ptr;
1037	return XML_TOK_PROLOG_S;
1038	case BT_PERCNT:
1039	return PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1040	case BT_COMMA:
1041	*nextTokPtr = ptr + MINBPC(enc);
1042	return XML_TOK_COMMA;
1043	case BT_LSQB:
1044	*nextTokPtr = ptr + MINBPC(enc);
1045	return XML_TOK_OPEN_BRACKET;
1046	case BT_RSQB:
1047	ptr += MINBPC(enc);
1048	if (ptr == end)
1049	return -XML_TOK_CLOSE_BRACKET;
1050	if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
1051	if (ptr + MINBPC(enc) == end)
1052	return XML_TOK_PARTIAL;
1053	if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
1054	nextTokPtr = ptr + 2MINBPC(enc);
1055	return XML_TOK_COND_SECT_CLOSE;
1056	}
1057	}
1058	*nextTokPtr = ptr;
1059	return XML_TOK_CLOSE_BRACKET;
1060	case BT_LPAR:
1061	*nextTokPtr = ptr + MINBPC(enc);
1062	return XML_TOK_OPEN_PAREN;
1063	case BT_RPAR:
1064	ptr += MINBPC(enc);
1065	if (ptr == end)
1066	return -XML_TOK_CLOSE_PAREN;
1067	switch (BYTE_TYPE(enc, ptr)) {
1068	case BT_AST:
1069	*nextTokPtr = ptr + MINBPC(enc);
1070	return XML_TOK_CLOSE_PAREN_ASTERISK;
1071	case BT_QUEST:
1072	*nextTokPtr = ptr + MINBPC(enc);
1073	return XML_TOK_CLOSE_PAREN_QUESTION;
1074	case BT_PLUS:
1075	*nextTokPtr = ptr + MINBPC(enc);
1076	return XML_TOK_CLOSE_PAREN_PLUS;
1077	case BT_CR: case BT_LF: case BT_S:
1078	case BT_GT: case BT_COMMA: case BT_VERBAR:
1079	case BT_RPAR:
1080	*nextTokPtr = ptr;
1081	return XML_TOK_CLOSE_PAREN;
1082	}
1083	*nextTokPtr = ptr;
1084	return XML_TOK_INVALID;
1085	case BT_VERBAR:
1086	*nextTokPtr = ptr + MINBPC(enc);
1087	return XML_TOK_OR;
1088	case BT_GT:
1089	*nextTokPtr = ptr + MINBPC(enc);
1090	return XML_TOK_DECL_CLOSE;
1091	case BT_NUM:
1092	return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1093	#define LEAD_CASE(n) \
1094	case BT_LEAD ## n: \
1095	if (end - ptr < n) \
1096	return XML_TOK_PARTIAL_CHAR; \
1097	if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
1098	ptr += n; \
1099	tok = XML_TOK_NAME; \
1100	break; \
1101	} \
1102	if (IS_NAME_CHAR(enc, ptr, n)) { \
1103	ptr += n; \
1104	tok = XML_TOK_NMTOKEN; \
1105	break; \
1106	} \
1107	*nextTokPtr = ptr; \
1108	return XML_TOK_INVALID;
1109	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1110	#undef LEAD_CASE
1111	case BT_NMSTRT:
1112	case BT_HEX:
1113	tok = XML_TOK_NAME;
1114	ptr += MINBPC(enc);
1115	break;
1116	case BT_DIGIT:
1117	case BT_NAME:
1118	case BT_MINUS:
1119	#ifdef XML_NS
1120	case BT_COLON:
1121	#endif
1122	tok = XML_TOK_NMTOKEN;
1123	ptr += MINBPC(enc);
1124	break;
1125	case BT_NONASCII:
1126	if (IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {
1127	ptr += MINBPC(enc);
1128	tok = XML_TOK_NAME;
1129	break;
1130	}
1131	if (IS_NAME_CHAR_MINBPC(enc, ptr)) {
1132	ptr += MINBPC(enc);
1133	tok = XML_TOK_NMTOKEN;
1134	break;
1135	}
1136	/* fall through */
1137	default:
1138	*nextTokPtr = ptr;
1139	return XML_TOK_INVALID;
1140	}
1141	while (ptr != end) {
1142	switch (BYTE_TYPE(enc, ptr)) {
1143	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
1144	case BT_GT: case BT_RPAR: case BT_COMMA:
1145	case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
1146	case BT_S: case BT_CR: case BT_LF:
1147	*nextTokPtr = ptr;
1148	return tok;
1149	#ifdef XML_NS
1150	case BT_COLON:
1151	ptr += MINBPC(enc);
1152	switch (tok) {
1153	case XML_TOK_NAME:
1154	if (ptr == end)
1155	return XML_TOK_PARTIAL;
1156	tok = XML_TOK_PREFIXED_NAME;
1157	switch (BYTE_TYPE(enc, ptr)) {
1158	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
1159	default:
1160	tok = XML_TOK_NMTOKEN;
1161	break;
1162	}
1163	break;
1164	case XML_TOK_PREFIXED_NAME:
1165	tok = XML_TOK_NMTOKEN;
1166	break;
1167	}
1168	break;
1169	#endif
1170	case BT_PLUS:
1171	if (tok == XML_TOK_NMTOKEN) {
1172	*nextTokPtr = ptr;
1173	return XML_TOK_INVALID;
1174	}
1175	*nextTokPtr = ptr + MINBPC(enc);
1176	return XML_TOK_NAME_PLUS;
1177	case BT_AST:
1178	if (tok == XML_TOK_NMTOKEN) {
1179	*nextTokPtr = ptr;
1180	return XML_TOK_INVALID;
1181	}
1182	*nextTokPtr = ptr + MINBPC(enc);
1183	return XML_TOK_NAME_ASTERISK;
1184	case BT_QUEST:
1185	if (tok == XML_TOK_NMTOKEN) {
1186	*nextTokPtr = ptr;
1187	return XML_TOK_INVALID;
1188	}
1189	*nextTokPtr = ptr + MINBPC(enc);
1190	return XML_TOK_NAME_QUESTION;
1191	default:
1192	*nextTokPtr = ptr;
1193	return XML_TOK_INVALID;
1194	}
1195	}
1196	return -tok;
1197	}
1198
1199	static int PTRCALL
1200	PREFIX(attributeValueTok)(const ENCODING enc, const char ptr,
1201	const char end, const char *nextTokPtr)
1202	{
1203	const char *start;
1204	if (ptr == end)
1205	return XML_TOK_NONE;
1206	start = ptr;
1207	while (ptr != end) {
1208	switch (BYTE_TYPE(enc, ptr)) {
1209	#define LEAD_CASE(n) \
1210	case BT_LEAD ## n: ptr += n; break;
1211	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1212	#undef LEAD_CASE
1213	case BT_AMP:
1214	if (ptr == start)
1215	return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1216	*nextTokPtr = ptr;
1217	return XML_TOK_DATA_CHARS;
1218	case BT_LT:
1219	/* this is for inside entity references */
1220	*nextTokPtr = ptr;
1221	return XML_TOK_INVALID;
1222	case BT_LF:
1223	if (ptr == start) {
1224	*nextTokPtr = ptr + MINBPC(enc);
1225	return XML_TOK_DATA_NEWLINE;
1226	}
1227	*nextTokPtr = ptr;
1228	return XML_TOK_DATA_CHARS;
1229	case BT_CR:
1230	if (ptr == start) {
1231	ptr += MINBPC(enc);
1232	if (ptr == end)
1233	return XML_TOK_TRAILING_CR;
1234	if (BYTE_TYPE(enc, ptr) == BT_LF)
1235	ptr += MINBPC(enc);
1236	*nextTokPtr = ptr;
1237	return XML_TOK_DATA_NEWLINE;
1238	}
1239	*nextTokPtr = ptr;
1240	return XML_TOK_DATA_CHARS;
1241	case BT_S:
1242	if (ptr == start) {
1243	*nextTokPtr = ptr + MINBPC(enc);
1244	return XML_TOK_ATTRIBUTE_VALUE_S;
1245	}
1246	*nextTokPtr = ptr;
1247	return XML_TOK_DATA_CHARS;
1248	default:
1249	ptr += MINBPC(enc);
1250	break;
1251	}
1252	}
1253	*nextTokPtr = ptr;
1254	return XML_TOK_DATA_CHARS;
1255	}
1256
1257	static int PTRCALL
1258	PREFIX(entityValueTok)(const ENCODING enc, const char ptr,
1259	const char end, const char *nextTokPtr)
1260	{
1261	const char *start;
1262	if (ptr == end)
1263	return XML_TOK_NONE;
1264	start = ptr;
1265	while (ptr != end) {
1266	switch (BYTE_TYPE(enc, ptr)) {
1267	#define LEAD_CASE(n) \
1268	case BT_LEAD ## n: ptr += n; break;
1269	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1270	#undef LEAD_CASE
1271	case BT_AMP:
1272	if (ptr == start)
1273	return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1274	*nextTokPtr = ptr;
1275	return XML_TOK_DATA_CHARS;
1276	case BT_PERCNT:
1277	if (ptr == start) {
1278	int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
1279	end, nextTokPtr);
1280	return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
1281	}
1282	*nextTokPtr = ptr;
1283	return XML_TOK_DATA_CHARS;
1284	case BT_LF:
1285	if (ptr == start) {
1286	*nextTokPtr = ptr + MINBPC(enc);
1287	return XML_TOK_DATA_NEWLINE;
1288	}
1289	*nextTokPtr = ptr;
1290	return XML_TOK_DATA_CHARS;
1291	case BT_CR:
1292	if (ptr == start) {
1293	ptr += MINBPC(enc);
1294	if (ptr == end)
1295	return XML_TOK_TRAILING_CR;
1296	if (BYTE_TYPE(enc, ptr) == BT_LF)
1297	ptr += MINBPC(enc);
1298	*nextTokPtr = ptr;
1299	return XML_TOK_DATA_NEWLINE;
1300	}
1301	*nextTokPtr = ptr;
1302	return XML_TOK_DATA_CHARS;
1303	default:
1304	ptr += MINBPC(enc);
1305	break;
1306	}
1307	}
1308	*nextTokPtr = ptr;
1309	return XML_TOK_DATA_CHARS;
1310	}
1311
1312	#ifdef XML_DTD
1313
1314	static int PTRCALL
1315	PREFIX(ignoreSectionTok)(const ENCODING enc, const char ptr,
1316	const char end, const char *nextTokPtr)
1317	{
1318	int level = 0;
1319	if (MINBPC(enc) > 1) {
1320	size_t n = end - ptr;
1321	if (n & (MINBPC(enc) - 1)) {
1322	n &= ~(MINBPC(enc) - 1);
1323	end = ptr + n;
1324	}
1325	}
1326	while (ptr != end) {
1327	switch (BYTE_TYPE(enc, ptr)) {
1328	INVALID_CASES(ptr, nextTokPtr)
1329	case BT_LT:
1330	if ((ptr += MINBPC(enc)) == end)
1331	return XML_TOK_PARTIAL;
1332	if (CHAR_MATCHES(enc, ptr, ASCII_EXCL)) {
1333	if ((ptr += MINBPC(enc)) == end)
1334	return XML_TOK_PARTIAL;
1335	if (CHAR_MATCHES(enc, ptr, ASCII_LSQB)) {
1336	++level;
1337	ptr += MINBPC(enc);
1338	}
1339	}
1340	break;
1341	case BT_RSQB:
1342	if ((ptr += MINBPC(enc)) == end)
1343	return XML_TOK_PARTIAL;
1344	if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
1345	if ((ptr += MINBPC(enc)) == end)
1346	return XML_TOK_PARTIAL;
1347	if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {
1348	ptr += MINBPC(enc);
1349	if (level == 0) {
1350	*nextTokPtr = ptr;
1351	return XML_TOK_IGNORE_SECT;
1352	}
1353	--level;
1354	}
1355	}
1356	break;
1357	default:
1358	ptr += MINBPC(enc);
1359	break;
1360	}
1361	}
1362	return XML_TOK_PARTIAL;
1363	}
1364
1365	#endif /* XML_DTD */
1366
1367	static int PTRCALL
1368	PREFIX(isPublicId)(const ENCODING enc, const char ptr, const char *end,
1369	const char **badPtr)
1370	{
1371	ptr += MINBPC(enc);
1372	end -= MINBPC(enc);
1373	for (; ptr != end; ptr += MINBPC(enc)) {
1374	switch (BYTE_TYPE(enc, ptr)) {
1375	case BT_DIGIT:
1376	case BT_HEX:
1377	case BT_MINUS:
1378	case BT_APOS:
1379	case BT_LPAR:
1380	case BT_RPAR:
1381	case BT_PLUS:
1382	case BT_COMMA:
1383	case BT_SOL:
1384	case BT_EQUALS:
1385	case BT_QUEST:
1386	case BT_CR:
1387	case BT_LF:
1388	case BT_SEMI:
1389	case BT_EXCL:
1390	case BT_AST:
1391	case BT_PERCNT:
1392	case BT_NUM:
1393	#ifdef XML_NS
1394	case BT_COLON:
1395	#endif
1396	break;
1397	case BT_S:
1398	if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
1399	*badPtr = ptr;
1400	return 0;
1401	}
1402	break;
1403	case BT_NAME:
1404	case BT_NMSTRT:
1405	if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
1406	break;
1407	default:
1408	switch (BYTE_TO_ASCII(enc, ptr)) {
1409	case 0x24: /* $ */
1410	case 0x40: /* @ */
1411	break;
1412	default:
1413	*badPtr = ptr;
1414	return 0;
1415	}
1416	break;
1417	}
1418	}
1419	return 1;
1420	}
1421
1422	/* This must only be called for a well-formed start-tag or empty
1423	element tag. Returns the number of attributes. Pointers to the
1424	first attsMax attributes are stored in atts.
1425	*/
1426
1427	static int PTRCALL
1428	PREFIX(getAtts)(const ENCODING enc, const char ptr,
1429	int attsMax, ATTRIBUTE *atts)
1430	{
1431	enum { other, inName, inValue } state = inName;
1432	int nAtts = 0;
1433	int open = 0; /* defined when state == inValue;
1434	initialization just to shut up compilers */
1435
1436	for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
1437	switch (BYTE_TYPE(enc, ptr)) {
1438	#define START_NAME \
1439	if (state == other) { \
1440	if (nAtts < attsMax) { \
1441	atts[nAtts].name = ptr; \
1442	atts[nAtts].normalized = 1; \
1443	} \
1444	state = inName; \
1445	}
1446	#define LEAD_CASE(n) \
1447	case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
1448	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1449	#undef LEAD_CASE
1450	case BT_NONASCII:
1451	case BT_NMSTRT:
1452	case BT_HEX:
1453	START_NAME
1454	break;
1455	#undef START_NAME
1456	case BT_QUOT:
1457	if (state != inValue) {
1458	if (nAtts < attsMax)
1459	atts[nAtts].valuePtr = ptr + MINBPC(enc);
1460	state = inValue;
1461	open = BT_QUOT;
1462	}
1463	else if (open == BT_QUOT) {
1464	state = other;
1465	if (nAtts < attsMax)
1466	atts[nAtts].valueEnd = ptr;
1467	nAtts++;
1468	}
1469	break;
1470	case BT_APOS:
1471	if (state != inValue) {
1472	if (nAtts < attsMax)
1473	atts[nAtts].valuePtr = ptr + MINBPC(enc);
1474	state = inValue;
1475	open = BT_APOS;
1476	}
1477	else if (open == BT_APOS) {
1478	state = other;
1479	if (nAtts < attsMax)
1480	atts[nAtts].valueEnd = ptr;
1481	nAtts++;
1482	}
1483	break;
1484	case BT_AMP:
1485	if (nAtts < attsMax)
1486	atts[nAtts].normalized = 0;
1487	break;
1488	case BT_S:
1489	if (state == inName)
1490	state = other;
1491	else if (state == inValue
1492	&& nAtts < attsMax
1493	&& atts[nAtts].normalized
1494	&& (ptr == atts[nAtts].valuePtr
1495	\|\| BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
1496	\|\| BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
1497	\|\| BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
1498	atts[nAtts].normalized = 0;
1499	break;
1500	case BT_CR: case BT_LF:
1501	/* This case ensures that the first attribute name is counted
1502	Apart from that we could just change state on the quote. */
1503	if (state == inName)
1504	state = other;
1505	else if (state == inValue && nAtts < attsMax)
1506	atts[nAtts].normalized = 0;
1507	break;
1508	case BT_GT:
1509	case BT_SOL:
1510	if (state != inValue)
1511	return nAtts;
1512	break;
1513	default:
1514	break;
1515	}
1516	}
1517	/* not reached */
1518	}
1519
1520	static int PTRFASTCALL
1521	PREFIX(charRefNumber)(const ENCODING enc, const char ptr)
1522	{
1523	int result = 0;
1524	/* skip &# */
1525	ptr += 2*MINBPC(enc);
1526	if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
1527	for (ptr += MINBPC(enc);
1528	!CHAR_MATCHES(enc, ptr, ASCII_SEMI);
1529	ptr += MINBPC(enc)) {
1530	int c = BYTE_TO_ASCII(enc, ptr);
1531	switch (c) {
1532	case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
1533	case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
1534	result <<= 4;
1535	result \|= (c - ASCII_0);
1536	break;
1537	case ASCII_A: case ASCII_B: case ASCII_C:
1538	case ASCII_D: case ASCII_E: case ASCII_F:
1539	result <<= 4;
1540	result += 10 + (c - ASCII_A);
1541	break;
1542	case ASCII_a: case ASCII_b: case ASCII_c:
1543	case ASCII_d: case ASCII_e: case ASCII_f:
1544	result <<= 4;
1545	result += 10 + (c - ASCII_a);
1546	break;
1547	}
1548	if (result >= 0x110000)
1549	return -1;
1550	}
1551	}
1552	else {
1553	for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
1554	int c = BYTE_TO_ASCII(enc, ptr);
1555	result *= 10;
1556	result += (c - ASCII_0);
1557	if (result >= 0x110000)
1558	return -1;
1559	}
1560	}
1561	return checkCharRefNumber(result);
1562	}
1563
1564	static int PTRCALL
1565	PREFIX(predefinedEntityName)(const ENCODING enc, const char ptr,
1566	const char *end)
1567	{
1568	switch ((end - ptr)/MINBPC(enc)) {
1569	case 2:
1570	if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
1571	switch (BYTE_TO_ASCII(enc, ptr)) {
1572	case ASCII_l:
1573	return ASCII_LT;
1574	case ASCII_g:
1575	return ASCII_GT;
1576	}
1577	}
1578	break;
1579	case 3:
1580	if (CHAR_MATCHES(enc, ptr, ASCII_a)) {
1581	ptr += MINBPC(enc);
1582	if (CHAR_MATCHES(enc, ptr, ASCII_m)) {
1583	ptr += MINBPC(enc);
1584	if (CHAR_MATCHES(enc, ptr, ASCII_p))
1585	return ASCII_AMP;
1586	}
1587	}
1588	break;
1589	case 4:
1590	switch (BYTE_TO_ASCII(enc, ptr)) {
1591	case ASCII_q:
1592	ptr += MINBPC(enc);
1593	if (CHAR_MATCHES(enc, ptr, ASCII_u)) {
1594	ptr += MINBPC(enc);
1595	if (CHAR_MATCHES(enc, ptr, ASCII_o)) {
1596	ptr += MINBPC(enc);
1597	if (CHAR_MATCHES(enc, ptr, ASCII_t))
1598	return ASCII_QUOT;
1599	}
1600	}
1601	break;
1602	case ASCII_a:
1603	ptr += MINBPC(enc);
1604	if (CHAR_MATCHES(enc, ptr, ASCII_p)) {
1605	ptr += MINBPC(enc);
1606	if (CHAR_MATCHES(enc, ptr, ASCII_o)) {
1607	ptr += MINBPC(enc);
1608	if (CHAR_MATCHES(enc, ptr, ASCII_s))
1609	return ASCII_APOS;
1610	}
1611	}
1612	break;
1613	}
1614	}
1615	return 0;
1616	}
1617
1618	static int PTRCALL
1619	PREFIX(sameName)(const ENCODING enc, const char ptr1, const char *ptr2)
1620	{
1621	for (;;) {
1622	switch (BYTE_TYPE(enc, ptr1)) {
1623	#define LEAD_CASE(n) \
1624	case BT_LEAD ## n: \
1625	if (ptr1++ != ptr2++) \
1626	return 0;
1627	LEAD_CASE(4) LEAD_CASE(3) LEAD_CASE(2)
1628	#undef LEAD_CASE
1629	/* fall through */
1630	if (ptr1++ != ptr2++)
1631	return 0;
1632	break;
1633	case BT_NONASCII:
1634	case BT_NMSTRT:
1635	#ifdef XML_NS
1636	case BT_COLON:
1637	#endif
1638	case BT_HEX:
1639	case BT_DIGIT:
1640	case BT_NAME:
1641	case BT_MINUS:
1642	if (ptr2++ != ptr1++)
1643	return 0;
1644	if (MINBPC(enc) > 1) {
1645	if (ptr2++ != ptr1++)
1646	return 0;
1647	if (MINBPC(enc) > 2) {
1648	if (ptr2++ != ptr1++)
1649	return 0;
1650	if (MINBPC(enc) > 3) {
1651	if (ptr2++ != ptr1++)
1652	return 0;
1653	}
1654	}
1655	}
1656	break;
1657	default:
1658	if (MINBPC(enc) == 1 && ptr1 == ptr2)
1659	return 1;
1660	switch (BYTE_TYPE(enc, ptr2)) {
1661	case BT_LEAD2:
1662	case BT_LEAD3:
1663	case BT_LEAD4:
1664	case BT_NONASCII:
1665	case BT_NMSTRT:
1666	#ifdef XML_NS
1667	case BT_COLON:
1668	#endif
1669	case BT_HEX:
1670	case BT_DIGIT:
1671	case BT_NAME:
1672	case BT_MINUS:
1673	return 0;
1674	default:
1675	return 1;
1676	}
1677	}
1678	}
1679	/* not reached */
1680	}
1681
1682	static int PTRCALL
1683	PREFIX(nameMatchesAscii)(const ENCODING enc, const char ptr1,
1684	const char end1, const char ptr2)
1685	{
1686	for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
1687	if (ptr1 == end1)
1688	return 0;
1689	if (!CHAR_MATCHES(enc, ptr1, *ptr2))
1690	return 0;
1691	}
1692	return ptr1 == end1;
1693	}
1694
1695	static int PTRFASTCALL
1696	PREFIX(nameLength)(const ENCODING enc, const char ptr)
1697	{
1698	const char *start = ptr;
1699	for (;;) {
1700	switch (BYTE_TYPE(enc, ptr)) {
1701	#define LEAD_CASE(n) \
1702	case BT_LEAD ## n: ptr += n; break;
1703	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1704	#undef LEAD_CASE
1705	case BT_NONASCII:
1706	case BT_NMSTRT:
1707	#ifdef XML_NS
1708	case BT_COLON:
1709	#endif
1710	case BT_HEX:
1711	case BT_DIGIT:
1712	case BT_NAME:
1713	case BT_MINUS:
1714	ptr += MINBPC(enc);
1715	break;
1716	default:
1717	return (int)(ptr - start);
1718	}
1719	}
1720	}
1721
1722	static const char * PTRFASTCALL
1723	PREFIX(skipS)(const ENCODING enc, const char ptr)
1724	{
1725	for (;;) {
1726	switch (BYTE_TYPE(enc, ptr)) {
1727	case BT_LF:
1728	case BT_CR:
1729	case BT_S:
1730	ptr += MINBPC(enc);
1731	break;
1732	default:
1733	return ptr;
1734	}
1735	}
1736	}
1737
1738	static void PTRCALL
1739	PREFIX(updatePosition)(const ENCODING *enc,
1740	const char *ptr,
1741	const char *end,
1742	POSITION *pos)
1743	{
1744	while (ptr != end) {
1745	switch (BYTE_TYPE(enc, ptr)) {
1746	#define LEAD_CASE(n) \
1747	case BT_LEAD ## n: \
1748	ptr += n; \
1749	break;
1750	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1751	#undef LEAD_CASE
1752	case BT_LF:
1753	pos->columnNumber = (XML_Size)-1;
1754	pos->lineNumber++;
1755	ptr += MINBPC(enc);
1756	break;
1757	case BT_CR:
1758	pos->lineNumber++;
1759	ptr += MINBPC(enc);
1760	if (ptr != end && BYTE_TYPE(enc, ptr) == BT_LF)
1761	ptr += MINBPC(enc);
1762	pos->columnNumber = (XML_Size)-1;
1763	break;
1764	default:
1765	ptr += MINBPC(enc);
1766	break;
1767	}
1768	pos->columnNumber++;
1769	}
1770	}
1771
1772	#undef DO_LEAD_CASE
1773	#undef MULTIBYTE_CASES
1774	#undef INVALID_CASES
1775	#undef CHECK_NAME_CASE
1776	#undef CHECK_NAME_CASES
1777	#undef CHECK_NMSTRT_CASE
1778	#undef CHECK_NMSTRT_CASES
1779

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: trunk/essentials/dev-lang/python/Modules/expat/xmltok_impl.c@ 3506

Download in other formats: