source: trunk/tools/qdoc3/tokenizer.h@ 1030

Last change on this file since 1030 was 846, checked in by Dmitry A. Kuminov, 14 years ago

trunk: Merged in qt 4.7.2 sources from branches/vendor/nokia/qt.
File size: 6.1 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
4** All rights reserved.
5** Contact: Nokia Corporation ([email protected])
6**
7** This file is part of the tools applications of the Qt Toolkit.
8**
9** $QT_BEGIN_LICENSE:LGPL$
10** Commercial Usage
11** Licensees holding valid Qt Commercial licenses may use this file in
12** accordance with the Qt Commercial License Agreement provided with the
13** Software or, alternatively, in accordance with the terms contained in
14** a written agreement between you and Nokia.
15**
16** GNU Lesser General Public License Usage
17** Alternatively, this file may be used under the terms of the GNU Lesser
18** General Public License version 2.1 as published by the Free Software
19** Foundation and appearing in the file LICENSE.LGPL included in the
20** packaging of this file. Please review the following information to
21** ensure the GNU Lesser General Public License version 2.1 requirements
22** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
23**
24** In addition, as a special exception, Nokia gives you certain additional
25** rights. These rights are described in the Nokia Qt LGPL Exception
26** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you have questions regarding the use of this file, please contact
37** Nokia at [email protected].
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42/*
43 tokenizer.h
44*/
45
46#ifndef TOKENIZER_H
47#define TOKENIZER_H
48
49#include <qfile.h>
50#include <qstack.h>
51#include <qstring.h>
52
53#include "location.h"
54
55QT_BEGIN_NAMESPACE
56
57/*
58 Here come the C++ tokens we support. The first part contains
59 all-purpose tokens; then come keywords.
60
61 If you add a keyword, make sure to modify the keyword array in
62 tokenizer.cpp as well, and possibly adjust Tok_FirstKeyword and
63 Tok_LastKeyword.
64*/
65enum { Tok_Eoi, Tok_Ampersand, Tok_Aster, Tok_Caret, Tok_LeftParen,
66 Tok_RightParen, Tok_LeftParenAster, Tok_Equal, Tok_LeftBrace,
67 Tok_RightBrace, Tok_Semicolon, Tok_Colon, Tok_LeftAngle,
68 Tok_RightAngle, Tok_Comma, Tok_Ellipsis, Tok_Gulbrandsen,
69 Tok_LeftBracket, Tok_RightBracket, Tok_Tilde, Tok_SomeOperator,
70 Tok_Number, Tok_String, Tok_Doc, Tok_Comment, Tok_Ident, Tok_At,
71 Tok_char, Tok_class, Tok_const, Tok_double, Tok_enum,
72 Tok_explicit, Tok_friend, Tok_inline, Tok_int, Tok_long,
73 Tok_namespace, Tok_operator, Tok_private, Tok_protected,
74 Tok_public, Tok_short, Tok_signals, Tok_signed, Tok_slots,
75 Tok_static, Tok_struct, Tok_template, Tok_typedef,
76 Tok_typename, Tok_union, Tok_unsigned, Tok_using, Tok_virtual,
77 Tok_void, Tok_volatile, Tok_int64, Tok_Q_OBJECT, Tok_Q_OVERRIDE,
78 Tok_Q_PROPERTY, Tok_Q_PRIVATE_PROPERTY, Tok_Q_DECLARE_SEQUENTIAL_ITERATOR,
79 Tok_Q_DECLARE_MUTABLE_SEQUENTIAL_ITERATOR,
80 Tok_Q_DECLARE_ASSOCIATIVE_ITERATOR,
81 Tok_Q_DECLARE_MUTABLE_ASSOCIATIVE_ITERATOR,
82 Tok_Q_DECLARE_FLAGS, Tok_Q_SIGNALS, Tok_Q_SLOTS, Tok_QT_COMPAT,
83 Tok_QT_COMPAT_CONSTRUCTOR, Tok_QT_DEPRECATED, Tok_QT_MOC_COMPAT,
84 Tok_QT_MODULE, Tok_QT3_SUPPORT, Tok_QT3_SUPPORT_CONSTRUCTOR,
85 Tok_QT3_MOC_SUPPORT, Tok_QDOC_PROPERTY,
86 Tok_FirstKeyword = Tok_char, Tok_LastKeyword = Tok_QDOC_PROPERTY };
87
88/*
89 The Tokenizer class implements lexical analysis of C++ source
90 files.
91
92 Not every operator or keyword of C++ is recognized; only those
93 that are interesting to us. Some Qt keywords or macros are also
94 recognized.
95*/
96
97class Tokenizer
98{
99 public:
100 Tokenizer(const Location& loc, const QByteArray &in);
101 Tokenizer(const Location& loc, QFile &file);
102
103 ~Tokenizer();
104
105 int getToken();
106 void setParsingFnOrMacro(bool macro) { parsingMacro = macro; }
107 bool parsingFnOrMacro() const { return parsingMacro; }
108
109 const Location &location() const { return yyTokLoc; }
110 QString previousLexeme() const;
111 QString lexeme() const;
112 QString version() const { return yyVersion; }
113 int braceDepth() const { return yyBraceDepth; }
114 int parenDepth() const { return yyParenDepth; }
115 int bracketDepth() const { return yyBracketDepth; }
116
117 static void initialize(const Config &config);
118 static void terminate();
119 static bool isTrue(const QString &condition);
120
121 private:
122 void init();
123 void start(const Location& loc);
124 /*
125 This limit on the length of a lexeme seems fairly high, but a
126 doc comment can be arbitrarily long. The previous 65,536 limit
127 was reached by Mark Summerfield.
128 */
129 enum { yyLexBufSize = 524288 };
130
131 int getch()
132 {
133 return yyPos == yyIn.size() ? EOF : yyIn[yyPos++];
134 }
135
136 inline int getChar()
137 {
138 if (yyCh == EOF)
139 return EOF;
140 if (yyLexLen < yyLexBufSize - 1) {
141 yyLex[yyLexLen++] = (char) yyCh;
142 yyLex[yyLexLen] = '\0';
143 }
144 yyCurLoc.advance(yyCh);
145 int ch = getch();
146 if (ch == EOF)
147 return EOF;
148 // cast explicitly to make sure the value of ch
149 // is in range [0..255] to avoid assert messages
150 // when using debug CRT that checks its input.
151 return int(uint(uchar(ch)));
152 }
153
154 int getTokenAfterPreprocessor();
155 void pushSkipping(bool skip);
156 bool popSkipping();
157
158 Location yyTokLoc;
159 Location yyCurLoc;
160 char *yyLexBuf1;
161 char *yyLexBuf2;
162 char *yyPrevLex;
163 char *yyLex;
164 size_t yyLexLen;
165 QStack<bool> yyPreprocessorSkipping;
166 int yyNumPreprocessorSkipping;
167 int yyBraceDepth;
168 int yyParenDepth;
169 int yyBracketDepth;
170 int yyCh;
171
172 QString yyVersion;
173 bool parsingMacro;
174
175 protected:
176 QByteArray yyIn;
177 int yyPos;
178};
179
180QT_END_NAMESPACE
181
182#endif
Note: See TracBrowser for help on using the repository browser.