source: trunk/essentials/dev-lang/python/Parser/tokenizer.h@ 3951

Last change on this file since 3951 was 3225, checked in by bird, 19 years ago

Python 2.5
File size: 2.3 KB
Line 
1#ifndef Py_TOKENIZER_H
2#define Py_TOKENIZER_H
3#ifdef __cplusplus
4extern "C" {
5#endif
6
7#include "object.h"
8
9/* Tokenizer interface */
10
11#include "token.h" /* For token types */
12
13#define MAXINDENT 100 /* Max indentation level */
14
15/* Tokenizer state */
16struct tok_state {
17 /* Input state; buf <= cur <= inp <= end */
18 /* NB an entire line is held in the buffer */
19 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
20 char *cur; /* Next character in buffer */
21 char *inp; /* End of data in buffer */
22 char *end; /* End of input buffer if buf != NULL */
23 char *start; /* Start of current token if not NULL */
24 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
25 /* NB If done != E_OK, cur must be == inp!!! */
26 FILE *fp; /* Rest of input; NULL if tokenizing a string */
27 int tabsize; /* Tab spacing */
28 int indent; /* Current indentation index */
29 int indstack[MAXINDENT]; /* Stack of indents */
30 int atbol; /* Nonzero if at begin of new line */
31 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
32 char *prompt, *nextprompt; /* For interactive prompting */
33 int lineno; /* Current line number */
34 int level; /* () [] {} Parentheses nesting level */
35 /* Used to allow free continuations inside them */
36 /* Stuff for checking on different tab sizes */
37 const char *filename; /* For error messages */
38 int altwarning; /* Issue warning if alternate tabs don't match */
39 int alterror; /* Issue error if alternate tabs don't match */
40 int alttabsize; /* Alternate tab spacing */