source: trunk/tools/qdoc3/tokenizer.cpp@ 441

Last change on this file since 441 was 2, checked in by Dmitry A. Kuminov, 16 years ago

Initially imported qt-all-opensource-src-4.5.1 from Trolltech.
File size: 23.8 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
4** Contact: Qt Software Information ([email protected])
5**
6** This file is part of the tools applications of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial Usage
10** Licensees holding valid Qt Commercial licenses may use this file in
11** accordance with the Qt Commercial License Agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and Nokia.
14**
15** GNU Lesser General Public License Usage
16** Alternatively, this file may be used under the terms of the GNU Lesser
17** General Public License version 2.1 as published by the Free Software
18** Foundation and appearing in the file LICENSE.LGPL included in the
19** packaging of this file. Please review the following information to
20** ensure the GNU Lesser General Public License version 2.1 requirements
21** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
22**
23** In addition, as a special exception, Nokia gives you certain
24** additional rights. These rights are described in the Nokia Qt LGPL
25** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
26** package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you are unsure which license is appropriate for your use, please
37** contact the sales department at [email protected].
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42#include "config.h"
43#include "tokenizer.h"
44
45#include <qdebug.h>
46#include <qfile.h>
47#include <qhash.h>
48#include <qregexp.h>
49#include <qstring.h>
50
51#include <ctype.h>
52#include <string.h>
53
54QT_BEGIN_NAMESPACE
55
56#define LANGUAGE_CPP "Cpp"
57
58/* qmake ignore Q_OBJECT */
59
60/*
61 Keep in sync with tokenizer.h.
62*/
63static const char *kwords[] = {
64 "char", "class", "const", "double", "enum", "explicit",
65 "friend", "inline", "int", "long", "namespace", "operator",
66 "private", "protected", "public", "short", "signals", "signed",
67 "slots", "static", "struct", "template", "typedef", "typename",
68 "union", "unsigned", "using", "virtual", "void", "volatile",
69 "__int64", "Q_OBJECT", "Q_OVERRIDE", "Q_PROPERTY",
70 "Q_DECLARE_SEQUENTIAL_ITERATOR",
71 "Q_DECLARE_MUTABLE_SEQUENTIAL_ITERATOR",
72 "Q_DECLARE_ASSOCIATIVE_ITERATOR",
73 "Q_DECLARE_MUTABLE_ASSOCIATIVE_ITERATOR",
74 "Q_DECLARE_FLAGS",
75 "Q_SIGNALS",
76 "Q_SLOTS",
77 "QT_COMPAT",
78 "QT_COMPAT_CONSTRUCTOR",
79 "QT_DEPRECATED",
80 "QT_MOC_COMPAT",
81 "QT_MODULE",
82 "QT3_SUPPORT",
83 "QT3_SUPPORT_CONSTRUCTOR",
84 "QT3_MOC_SUPPORT",
85 "QDOC_PROPERTY"
86};
87
88static const int KwordHashTableSize = 4096;
89static int kwordHashTable[KwordHashTableSize];
90
91static QHash<QByteArray, bool> *ignoredTokensAndDirectives = 0;
92
93static QRegExp *comment = 0;
94static QRegExp *versionX = 0;
95static QRegExp *definedX = 0;
96
97static QRegExp *defines = 0;
98static QRegExp *falsehoods = 0;
99
100/*
101 This function is a perfect hash function for the 37 keywords of C99
102 (with a hash table size of 512). It should perform well on our
103 Qt-enhanced C++ subset.
104*/
105static int hashKword(const char *s, int len)
106{
107 return (((uchar) s[0]) + (((uchar) s[2]) << 5) +
108 (((uchar) s[len - 1]) << 3)) % KwordHashTableSize;
109}
110
111static void insertKwordIntoHash(const char *s, int number)
112{
113 int k = hashKword(s, strlen(s));
114 while (kwordHashTable[k]) {
115 if (++k == KwordHashTableSize)
116 k = 0;
117 }
118 kwordHashTable[k] = number;
119}
120
121Tokenizer::Tokenizer(const Location& loc, FILE *in)
122{
123 init();
124 QFile file;
125 file.open(in, QIODevice::ReadOnly);
126 yyIn = file.readAll();
127 file.close();
128 yyPos = 0;
129 start(loc);
130}
131
132Tokenizer::Tokenizer(const Location& loc, const QByteArray &in)
133 : yyIn(in)
134{
135 init();
136 yyPos = 0;
137 start(loc);
138}
139
140Tokenizer::~Tokenizer()
141{
142 delete[] yyLexBuf1;
143 delete[] yyLexBuf2;
144}
145
146int Tokenizer::getToken()
147{
148 char *t = yyPrevLex;
149 yyPrevLex = yyLex;
150 yyLex = t;
151
152 while (yyCh != EOF) {
153 yyTokLoc = yyCurLoc;
154 yyLexLen = 0;
155
156 if (isspace(yyCh)) {
157 do {
158 yyCh = getChar();
159 } while (isspace(yyCh));
160 }
161 else if (isalpha(yyCh) || yyCh == '_') {
162 do {
163 yyCh = getChar();
164 } while (isalnum(yyCh) || yyCh == '_');
165
166 int k = hashKword(yyLex, yyLexLen);
167 for (;;) {
168 int i = kwordHashTable[k];
169 if (i == 0) {
170 return Tok_Ident;
171 }
172 else if (i == -1) {
173 if (!parsingMacro && ignoredTokensAndDirectives->contains(yyLex)) {
174 if (ignoredTokensAndDirectives->value(yyLex)) { // it's a directive
175 int parenDepth = 0;
176 while (yyCh != EOF && (yyCh != ')' || parenDepth > 1)) {
177 if (yyCh == '(')
178 ++parenDepth;
179 else if (yyCh == ')')
180 --parenDepth;
181 yyCh = getChar();
182 }
183 if (yyCh == ')')
184 yyCh = getChar();
185 }
186 break;
187 }
188 }
189 else if (strcmp(yyLex, kwords[i - 1]) == 0) {
190 int ret = (int) Tok_FirstKeyword + i - 1;
191 if (ret != Tok_explicit && ret != Tok_inline && ret != Tok_typename)
192 return ret;
193 break;
194 }
195
196 if (++k == KwordHashTableSize)
197 k = 0;
198 }
199 }
200 else if (isdigit(yyCh)) {
201 do {
202 yyCh = getChar();
203 } while (isalnum(yyCh) || yyCh == '.' || yyCh == '+' ||
204 yyCh == '-');
205 return Tok_Number;
206 }
207 else {
208 switch (yyCh) {
209 case '!':
210 case '%':
211 yyCh = getChar();
212 if (yyCh == '=')
213 yyCh = getChar();
214 return Tok_SomeOperator;
215 case '"':
216 yyCh = getChar();
217
218 while (yyCh != EOF && yyCh != '"') {
219 if (yyCh == '\\')
220 yyCh = getChar();
221 yyCh = getChar();
222 }
223 yyCh = getChar();
224
225 if (yyCh == EOF)
226 yyTokLoc.warning(tr("Unterminated C++ string literal"),
227 tr("Maybe you forgot '/*!' at the beginning of the file?"));
228 else
229 return Tok_String;
230 break;
231 case '#':
232 return getTokenAfterPreprocessor();
233 case '&':
234 yyCh = getChar();
235 if (yyCh == '&' || yyCh == '=') {
236 yyCh = getChar();
237 return Tok_SomeOperator;
238 }
239 else {
240 return Tok_Ampersand;
241 }
242 case '\'':
243 yyCh = getChar();
244 if (yyCh == '\\')
245 yyCh = getChar();
246 do {
247 yyCh = getChar();
248 } while (yyCh != EOF && yyCh != '\'');
249
250 if (yyCh == EOF) {
251 yyTokLoc.warning(tr("Unterminated C++ character"
252 " literal"));
253 }
254 else {
255 yyCh = getChar();
256 return Tok_Number;
257 }
258 break;
259 case '(':
260 yyCh = getChar();
261 if (yyNumPreprocessorSkipping == 0)
262 yyParenDepth++;
263 if (isspace(yyCh)) {
264 do {
265 yyCh = getChar();
266 } while (isspace(yyCh));
267 yyLexLen = 1;
268 yyLex[1] = '\0';
269 }