source: trunk/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp@ 109

Last change on this file since 109 was 2, checked in by Dmitry A. Kuminov, 16 years ago

Initially imported qt-all-opensource-src-4.5.1 from Trolltech.
File size: 25.5 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
4** Contact: Qt Software Information ([email protected])
5**
6** This file is part of the Qt Assistant of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial Usage
10** Licensees holding valid Qt Commercial licenses may use this file in
11** accordance with the Qt Commercial License Agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and Nokia.
14**
15** GNU Lesser General Public License Usage
16** Alternatively, this file may be used under the terms of the GNU Lesser
17** General Public License version 2.1 as published by the Free Software
18** Foundation and appearing in the file LICENSE.LGPL included in the
19** packaging of this file. Please review the following information to
20** ensure the GNU Lesser General Public License version 2.1 requirements
21** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
22**
23** In addition, as a special exception, Nokia gives you certain
24** additional rights. These rights are described in the Nokia Qt LGPL
25** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
26** package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you are unsure which license is appropriate for your use, please
37** contact the sales department at [email protected].
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42#include "qhelpenginecore.h"
43#include "qhelp_global.h"
44#include "fulltextsearch/qhits_p.h"
45#include "fulltextsearch/qquery_p.h"
46#include "fulltextsearch/qanalyzer_p.h"
47#include "fulltextsearch/qdocument_p.h"
48#include "fulltextsearch/qsearchable_p.h"
49#include "fulltextsearch/qindexreader_p.h"
50#include "fulltextsearch/qindexwriter_p.h"
51#include "qhelpsearchindexwriter_clucene_p.h"
52
53#include <QtCore/QDir>
54#include <QtCore/QString>
55#include <QtCore/QFileInfo>
56#include <QtCore/QTextCodec>
57#include <QtCore/QTextStream>
58
59#include <QtNetwork/QLocalSocket>
60#include <QtNetwork/QLocalServer>
61
62#include "private/qfunctions_p.h"
63
64QT_BEGIN_NAMESPACE
65
66namespace qt {
67 namespace fulltextsearch {
68 namespace clucene {
69
70// taken from qtexthtmlparser
71static const struct QTextHtmlEntity
72{
73 const char *name;
74 quint16 code;
75} entities[] = {
76 { "AElig", 0x00c6 },
77 { "AMP", 38 },
78 { "Aacute", 0x00c1 },
79 { "Acirc", 0x00c2 },
80 { "Agrave", 0x00c0 },
81 { "Alpha", 0x0391 },
82 { "Aring", 0x00c5 },
83 { "Atilde", 0x00c3 },
84 { "Auml", 0x00c4 },
85 { "Beta", 0x0392 },
86 { "Ccedil", 0x00c7 },
87 { "Chi", 0x03a7 },
88 { "Dagger", 0x2021 },
89 { "Delta", 0x0394 },
90 { "ETH", 0x00d0 },
91 { "Eacute", 0x00c9 },
92 { "Ecirc", 0x00ca },
93 { "Egrave", 0x00c8 },
94 { "Epsilon", 0x0395 },
95 { "Eta", 0x0397 },
96 { "Euml", 0x00cb },
97 { "GT", 62 },
98 { "Gamma", 0x0393 },
99 { "Iacute", 0x00cd },
100 { "Icirc", 0x00ce },
101 { "Igrave", 0x00cc },
102 { "Iota", 0x0399 },
103 { "Iuml", 0x00cf },
104 { "Kappa", 0x039a },
105 { "LT", 60 },
106 { "Lambda", 0x039b },
107 { "Mu", 0x039c },
108 { "Ntilde", 0x00d1 },
109 { "Nu", 0x039d },
110 { "OElig", 0x0152 },
111 { "Oacute", 0x00d3 },
112 { "Ocirc", 0x00d4 },
113 { "Ograve", 0x00d2 },
114 { "Omega", 0x03a9 },
115 { "Omicron", 0x039f },
116 { "Oslash", 0x00d8 },
117 { "Otilde", 0x00d5 },
118 { "Ouml", 0x00d6 },
119 { "Phi", 0x03a6 },
120 { "Pi", 0x03a0 },
121 { "Prime", 0x2033 },
122 { "Psi", 0x03a8 },
123 { "QUOT", 34 },
124 { "Rho", 0x03a1 },
125 { "Scaron", 0x0160 },
126 { "Sigma", 0x03a3 },
127 { "THORN", 0x00de },
128 { "Tau", 0x03a4 },
129 { "Theta", 0x0398 },
130 { "Uacute", 0x00da },
131 { "Ucirc", 0x00db },
132 { "Ugrave", 0x00d9 },
133 { "Upsilon", 0x03a5 },
134 { "Uuml", 0x00dc },
135 { "Xi", 0x039e },
136 { "Yacute", 0x00dd },
137 { "Yuml", 0x0178 },
138 { "Zeta", 0x0396 },
139 { "aacute", 0x00e1 },
140 { "acirc", 0x00e2 },
141 { "acute", 0x00b4 },
142 { "aelig", 0x00e6 },
143 { "agrave", 0x00e0 },
144 { "alefsym", 0x2135 },
145 { "alpha", 0x03b1 },
146 { "amp", 38 },
147 { "and", 0x22a5 },
148 { "ang", 0x2220 },
149 { "apos", 0x0027 },
150 { "aring", 0x00e5 },
151 { "asymp", 0x2248 },
152 { "atilde", 0x00e3 },
153 { "auml", 0x00e4 },
154 { "bdquo", 0x201e },
155 { "beta", 0x03b2 },
156 { "brvbar", 0x00a6 },
157 { "bull", 0x2022 },
158 { "cap", 0x2229 },
159 { "ccedil", 0x00e7 },
160 { "cedil", 0x00b8 },
161 { "cent", 0x00a2 },
162 { "chi", 0x03c7 },
163 { "circ", 0x02c6 },
164 { "clubs", 0x2663 },
165 { "cong", 0x2245 },
166 { "copy", 0x00a9 },
167 { "crarr", 0x21b5 },
168 { "cup", 0x222a },
169 { "curren", 0x00a4 },
170 { "dArr", 0x21d3 },
171 { "dagger", 0x2020 },
172 { "darr", 0x2193 },
173 { "deg", 0x00b0 },
174 { "delta", 0x03b4 },
175 { "diams", 0x2666 },
176 { "divide", 0x00f7 },
177 { "eacute", 0x00e9 },
178 { "ecirc", 0x00ea },
179 { "egrave", 0x00e8 },
180 { "empty", 0x2205 },
181 { "emsp", 0x2003 },
182 { "ensp", 0x2002 },
183 { "epsilon", 0x03b5 },
184 { "equiv", 0x2261 },
185 { "eta", 0x03b7 },
186 { "eth", 0x00f0 },
187 { "euml", 0x00eb },
188 { "euro", 0x20ac },
189 { "exist", 0x2203 },
190 { "fnof", 0x0192 },
191 { "forall", 0x2200 },
192 { "frac12", 0x00bd },
193 { "frac14", 0x00bc },
194 { "frac34", 0x00be },
195 { "frasl", 0x2044 },
196 { "gamma", 0x03b3 },
197 { "ge", 0x2265 },
198 { "gt", 62 },
199 { "hArr", 0x21d4 },
200 { "harr", 0x2194 },
201 { "hearts", 0x2665 },
202 { "hellip", 0x2026 },
203 { "iacute", 0x00ed },
204 { "icirc", 0x00ee },
205 { "iexcl", 0x00a1 },
206 { "igrave", 0x00ec },
207 { "image", 0x2111 },
208 { "infin", 0x221e },
209 { "int", 0x222b },
210 { "iota", 0x03b9 },
211 { "iquest", 0x00bf },
212 { "isin", 0x2208 },
213 { "iuml", 0x00ef },
214 { "kappa", 0x03ba },
215 { "lArr", 0x21d0 },
216 { "lambda", 0x03bb },
217 { "lang", 0x2329 },
218 { "laquo", 0x00ab },
219 { "larr", 0x2190 },
220 { "lceil", 0x2308 },
221 { "ldquo", 0x201c },
222 { "le", 0x2264 },
223 { "lfloor", 0x230a },
224 { "lowast", 0x2217 },
225 { "loz", 0x25ca },
226 { "lrm", 0x200e },
227 { "lsaquo", 0x2039 },
228 { "lsquo", 0x2018 },
229 { "lt", 60 },
230 { "macr", 0x00af },
231 { "mdash", 0x2014 },
232 { "micro", 0x00b5 },
233 { "middot", 0x00b7 },
234 { "minus", 0x2212 },
235 { "mu", 0x03bc },
236 { "nabla", 0x2207 },
237 { "nbsp", 0x00a0 },
238 { "ndash", 0x2013 },
239 { "ne", 0x2260 },
240 { "ni", 0x220b },
241 { "not", 0x00ac },
242 { "notin", 0x2209 },
243 { "nsub", 0x2284 },
244 { "ntilde", 0x00f1 },
245 { "nu", 0x03bd },
246 { "oacute", 0x00f3 },
247 { "ocirc", 0x00f4 },
248 { "oelig", 0x0153 },
249 { "ograve", 0x00f2 },
250 { "oline", 0x203e },
251 { "omega", 0x03c9 },
252 { "omicron", 0x03bf },
253 { "oplus", 0x2295 },
254 { "or", 0x22a6 },
255 { "ordf", 0x00aa },
256 { "ordm", 0x00ba },
257 { "oslash", 0x00f8 },
258 { "otilde", 0x00f5 },
259 { "otimes", 0x2297 },
260 { "ouml", 0x00f6 },
261 { "para", 0x00b6 },
262 { "part", 0x2202 },
263 { "percnt", 0x0025 },
264 { "permil", 0x2030 },
265 { "perp", 0x22a5 },
266 { "phi", 0x03c6 },
267 { "pi", 0x03c0 },
268 { "piv", 0x03d6 },
269 { "plusmn", 0x00b1 },
270 { "pound", 0x00a3 },
271 { "prime", 0x2032 },
272 { "prod", 0x220f },
273 { "prop", 0x221d },
274 { "psi", 0x03c8 },
275 { "quot", 34 },
276 { "rArr", 0x21d2 },
277 { "radic", 0x221a },
278 { "rang", 0x232a },
279 { "raquo", 0x00bb },
280 { "rarr", 0x2192 },
281 { "rceil", 0x2309 },
282 { "rdquo", 0x201d },
283 { "real", 0x211c },
284 { "reg", 0x00ae },
285 { "rfloor", 0x230b },
286 { "rho", 0x03c1 },
287 { "rlm", 0x200f },
288 { "rsaquo", 0x203a },
289 { "rsquo", 0x2019 },
290 { "sbquo", 0x201a },
291 { "scaron", 0x0161 },
292 { "sdot", 0x22c5 },
293 { "sect", 0x00a7 },
294 { "shy", 0x00ad },
295 { "sigma", 0x03c3 },
296 { "sigmaf", 0x03c2 },
297 { "sim", 0x223c },
298 { "spades", 0x2660 },
299 { "sub", 0x2282 },
300 { "sube", 0x2286 },
301 { "sum", 0x2211 },
302 { "sup", 0x2283 },
303 { "sup1", 0x00b9 },
304 { "sup2", 0x00b2 },
305 { "sup3", 0x00b3 },
306 { "supe", 0x2287 },
307 { "szlig", 0x00df },
308 { "tau", 0x03c4 },
309 { "there4", 0x2234 },
310 { "theta", 0x03b8 },
311 { "thetasym", 0x03d1 },
312 { "thinsp", 0x2009 },
313 { "thorn", 0x00fe },
314 { "tilde", 0x02dc },
315 { "times", 0x00d7 },
316 { "trade", 0x2122 },
317 { "uArr", 0x21d1 },
318 { "uacute", 0x00fa },
319 { "uarr", 0x2191 },
320 { "ucirc", 0x00fb },
321 { "ugrave", 0x00f9 },
322 { "uml", 0x00a8 },
323 { "upsih", 0x03d2 },
324 { "upsilon", 0x03c5 },
325 { "uuml", 0x00fc },
326 { "weierp", 0x2118 },
327 { "xi", 0x03be },
328 { "yacute", 0x00fd },
329 { "yen", 0x00a5 },
330 { "yuml", 0x00ff },
331 { "zeta", 0x03b6 },
332 { "zwj", 0x200d },
333 { "zwnj", 0x200c }
334};
335
336Q_STATIC_GLOBAL_OPERATOR bool operator<(const QString &entityStr, const QTextHtmlEntity &entity)
337{
338 return entityStr < QLatin1String(entity.name);
339}
340
341Q_STATIC_GLOBAL_OPERATOR bool operator<(const QTextHtmlEntity &entity, const QString &entityStr)
342{
343 return QLatin1String(entity.name) < entityStr;
344}
345
346static QChar resolveEntity(const QString &entity)
347{
348 const QTextHtmlEntity *start = &entities[0];
349 const QTextHtmlEntity *end = &entities[(sizeof(entities) / sizeof(entities[0]))];
350 const QTextHtmlEntity *e = qBinaryFind(start, end, entity);
351 if (e == end)
352 return QChar();
353 return e->code;
354}
355
356static const uint latin1Extended[0xA0 - 0x80] = {
357 0x20ac, // 0x80
358 0x0081, // 0x81 direct mapping
359 0x201a, // 0x82
360 0x0192, // 0x83
361 0x201e, // 0x84
362 0x2026, // 0x85
363 0x2020, // 0x86
364 0x2021, // 0x87
365 0x02C6, // 0x88
366 0x2030, // 0x89
367 0x0160, // 0x8A
368 0x2039, // 0x8B
369 0x0152, // 0x8C
370 0x008D, // 0x8D direct mapping
371 0x017D, // 0x8E
372 0x008F, // 0x8F directmapping
373 0x0090, // 0x90 directmapping
374 0x2018, // 0x91
375 0x2019, // 0x92
376 0x201C, // 0x93
377 0X201D, // 0x94
378 0x2022, // 0x95
379 0x2013, // 0x96
380 0x2014, // 0x97
381 0x02DC, // 0x98
382 0x2122, // 0x99
383 0x0161, // 0x9A
384 0x203A, // 0x9B
385 0x0153, // 0x9C
386 0x009D, // 0x9D direct mapping
387 0x017E, // 0x9E
388 0x0178 // 0x9F
389};
390// end taken from qtexthtmlparser
391
392class DocumentHelper
393{
394public:
395 DocumentHelper(const QString &fileName, const QByteArray &data)
396 : fileName(fileName) , data(readData(data)) {}
397 ~DocumentHelper() {}
398
399 bool addFieldsToDocument(QCLuceneDocument *document,
400 const QString &namespaceName, const QString &attributes = QString())
401 {
402 if (!document)
403 return false;
404
405 if(!data.isEmpty()) {
406 QString parsedData = parseData();
407 QString parsedTitle = QHelpGlobal::documentTitle(data);
408
409 if(!parsedData.isEmpty()) {
410 document->add(new QCLuceneField(QLatin1String("content"),
411 parsedData,QCLuceneField::INDEX_TOKENIZED));
412 document->add(new QCLuceneField(QLatin1String("path"), fileName,
413 QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));
414 document->add(new QCLuceneField(QLatin1String("title"), parsedTitle,
415 QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));
416 document->add(new QCLuceneField(QLatin1String("titleTokenized"), parsedTitle,
417 QCLuceneField::STORE_YES | QCLuceneField::INDEX_TOKENIZED));
418 document->add(new QCLuceneField(QLatin1String("namespace"), namespaceName,
419 QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));
420 document->add(new QCLuceneField(QLatin1String("attribute"), attributes,
421 QCLuceneField::STORE_YES | QCLuceneField::INDEX_TOKENIZED));
422 return true;
423 }
424 }
425
426 return false;
427 }
428
429private:
430 QString readData(const QByteArray &data)
431 {
432 QTextStream textStream(data);
433 QByteArray charSet = QHelpGlobal::charsetFromData(data).toLatin1();
434 textStream.setCodec(QTextCodec::codecForName(charSet.constData()));
435
436 QString stream = textStream.readAll();
437 if (stream.isNull() || stream.isEmpty())
438 return QString();
439
440 return stream;
441 }
442
443 QString parseData() const
444 {
445 const int length = data.length();
446 const QChar *buf = data.unicode();
447
448 QString parsedContent;
449 parsedContent.reserve(length);
450
451 bool valid = true;
452 int j = 0, count = 0;
453
454 QChar c;
455 while (j < length) {
456 c = buf[j++];
457 if (c == QLatin1Char('<') || c == QLatin1Char('&')) {
458 if (count > 1 && c != QLatin1Char('&'))
459 parsedContent.append(QLatin1Char(' '));
460 else if (c == QLatin1Char('&')) {
461 // Note: this will modify the counter j, in case we sucessful parsed the entity
462 // we will have modified the counter to stay 1 before the closing ';', so
463 // the following if condition will be met with if (c == QLatin1Char(';'))
464 parsedContent.append(parseEntity(length, buf, j));
465 }
466
467 count = 0;
468 valid = false;
469 continue;
470 }
471 if ((c == QLatin1Char('>') || c == QLatin1Char(';')) && !valid) {
472 valid = true;
473 continue;
474 }
475 if (!valid)
476 continue;
477
478 if (c.isLetterOrNumber() || c.isPrint()) {
479 ++count;
480 parsedContent.append(c.toLower());
481 } else {
482 if (count > 1)
483 parsedContent.append(QLatin1Char(' '));
484 count = 0;
485 }
486 }
487
488 return parsedContent;
489 }
490
491 // taken from qtexthtmlparser
492 // parses an entity after "&", and returns it
493 QString parseEntity(int len, const QChar *buf, int &pos) const
494 {
495 int recover = pos;
496 QString entity;
497 while (pos < len) {
498 QChar c = buf[pos++];
499 if (c.isSpace() || pos - recover > 9) {
500 goto error;
501 }
502 if (c == QLatin1Char(';')) {
503 pos--;
504 break;
505 }
506 entity += c;
507 }
508 {
509 QChar resolved = resolveEntity(entity);
510 if (!resolved.isNull())
511 return QString(resolved);
512 }
513 if (entity.length() > 1 && entity.at(0) == QLatin1Char('#')) {
514 entity.remove(0, 1); // removing leading #
515
516 int base = 10;
517 bool ok = false;
518
519 if (entity.at(0).toLower() == QLatin1Char('x')) { // hex entity?
520 entity.remove(0, 1);
521 base = 16;
522 }
523
524 uint uc = entity.toUInt(&ok, base);
525 if (ok) {
526 if (uc >= 0x80 && uc < 0x80 + (sizeof(latin1Extended) / sizeof(latin1Extended[0])))
527 uc = latin1Extended[uc - 0x80]; // windows latin 1 extended
528 QString str;
529 if (uc > 0xffff) {
530 // surrogate pair
531 uc -= 0x10000;
532 ushort high = uc/0x400 + 0xd800;
533 ushort low = uc%0x400 + 0xdc00;
534 str.append(QChar(high));
535 str.append(QChar(low));
536 } else {
537 str.append(QChar(uc));
538 }
539 return str;
540 }
541 }
542 error:
543 pos = recover;
544 return QLatin1String(" ");
545 }
546 // end taken from qtexthtmlparser
547
548private:
549 QString fileName;
550 QString data;
551};
552
553
554QHelpSearchIndexWriter::QHelpSearchIndexWriter()
555 : QThread(0)
556 , m_cancel(false)
557{
558 // nothing todo
559}
560
561QHelpSearchIndexWriter::~QHelpSearchIndexWriter()
562{
563 mutex.lock();
564 this->m_cancel = true;
565 waitCondition.wakeOne();
566 mutex.unlock();
567
568 wait();
569}
570
571void QHelpSearchIndexWriter::cancelIndexing()
572{
573 mutex.lock();
574 this->m_cancel = true;
575 mutex.unlock();
576}
577
578void QHelpSearchIndexWriter::updateIndex(const QString &collectionFile,
579 const QString &indexFilesFolder, bool reindex)
580{
581 mutex.lock();
582 this->m_cancel = false;
583 this->m_reindex = reindex;
584 this->m_collectionFile = collectionFile;
585 this->m_indexFilesFolder = indexFilesFolder;
586 mutex.unlock();
587
588 start(QThread::NormalPriority);
589}
590
591void QHelpSearchIndexWriter::optimizeIndex()
592{
593 if (QCLuceneIndexReader::indexExists(m_indexFilesFolder)) {
594 if (QCLuceneIndexReader::isLocked(m_indexFilesFolder))
595 return;
596
597 QCLuceneStandardAnalyzer analyzer;
598 QCLuceneIndexWriter writer(m_indexFilesFolder, analyzer, false);
599 writer.optimize();
600 writer.close();
601 }
602}
603
604void QHelpSearchIndexWriter::run()
605{
606 QMutexLocker mutexLocker(&mutex);
607
608 if (m_cancel)
609 return;
610
611 const bool reindex = this->m_reindex;
612 const QString collectionFile(this->m_collectionFile);
613
614 mutexLocker.unlock();
615
616 QHelpEngineCore engine(collectionFile, 0);
617 if (!engine.setupData())
618 return;
619
620 const QLatin1String key("CluceneIndexedNamespaces");
621 if (reindex)
622 engine.setCustomValue(key, QLatin1String(""));
623
624 QMap<QString, QDateTime> indexMap;
625 const QLatin1String oldKey("CluceneSearchNamespaces");
626 if (!engine.customValue(oldKey, QString()).isNull()) {
627 // old style qhc file < 4.4.2, need to convert...
628 const QStringList indexedNamespaces = engine.customValue(oldKey).
629 toString().split(QLatin1String("|"), QString::SkipEmptyParts);
630 foreach (const QString &nameSpace, indexedNamespaces)
631 indexMap.insert(nameSpace, QDateTime());
632 engine.removeCustomValue(oldKey);
633 } else {
634 QDataStream dataStream(engine.customValue(key).toByteArray());
635 dataStream >> indexMap;
636 }
637
638 QString indexPath = m_indexFilesFolder;
639
640 QFileInfo fInfo(indexPath);
641 if (fInfo.exists() && !fInfo.isWritable()) {
642 qWarning("Full Text Search, could not create index (missing permissions).");
643 return;
644 }
645
646 emit indexingStarted();
647
648 QCLuceneIndexWriter *writer = 0;
649 QCLuceneStandardAnalyzer analyzer;
650 const QStringList registeredDocs = engine.registeredDocumentations();
651
652 QLocalSocket localSocket;
653 localSocket.connectToServer(QString(QLatin1String("QtAssistant%1"))
654 .arg(QLatin1String(QT_VERSION_STR)));
655
656 QLocalServer localServer;
657 bool otherInstancesRunning = true;
658 if (!localSocket.waitForConnected()) {
659 otherInstancesRunning = false;
660 localServer.listen(QString(QLatin1String("QtAssistant%1"))
661 .arg(QLatin1String(QT_VERSION_STR)));
662 }
663
664#if !defined(QT_NO_EXCEPTIONS)
665 try {
666#endif
667 // check if it's locked, and if the other instance is running
668 if (!otherInstancesRunning && QCLuceneIndexReader::isLocked(indexPath))
669 QCLuceneIndexReader::unlock(indexPath);
670
671 if (QCLuceneIndexReader::isLocked(indexPath)) {
672 // poll unless indexing finished to fake progress
673 while (QCLuceneIndexReader::isLocked(indexPath)) {
674 mutexLocker.relock();
675 if (m_cancel)
676 break;
677 mutexLocker.unlock();
678 this->sleep(1);
679 }
680 emit indexingFinished();
681 return;
682 }
683
684 if (QCLuceneIndexReader::indexExists(indexPath) && !reindex) {
685 foreach(const QString &namespaceName, registeredDocs) {
686 mutexLocker.relock();
687 if (m_cancel) {
688 emit indexingFinished();
689 return;
690 }
691 mutexLocker.unlock();
692
693 if (!indexMap.contains(namespaceName)) {
694 // make sure we remove some partly indexed stuff
695 removeDocuments(indexPath, namespaceName);
696 } else {
697 QString path = engine.documentationFileName(namespaceName);
698 if (indexMap.value(namespaceName) < QFileInfo(path).lastModified()) {
699 // make sure we remove some outdated indexed stuff
700 indexMap.remove(namespaceName);
701 removeDocuments(indexPath, namespaceName);
702 }
703
704 if (indexMap.contains(namespaceName)) {
705 // make sure we really have content indexed for namespace
706 // NOTE: Extra variable just for GCC 3.3.5
707 QLatin1String key("namespace");
708 QCLuceneTermQuery query(QCLuceneTerm(key, namespaceName));
709 QCLuceneIndexSearcher indexSearcher(indexPath);
710 QCLuceneHits hits = indexSearcher.search(query);
711 if (hits.length() <= 0)
712 indexMap.remove(namespaceName);
713 }
714 }
715 }
716 writer = new QCLuceneIndexWriter(indexPath, analyzer, false);
717 } else {
718 indexMap.clear();
719 writer = new QCLuceneIndexWriter(indexPath, analyzer, true);
720 }
721#if !defined(QT_NO_EXCEPTIONS)
722 } catch (...) {
723 qWarning("Full Text Search, could not create index writer.");
724 return;
725 }
726#endif
727
728 writer->setMergeFactor(100);
729 writer->setMinMergeDocs(1000);
730 writer->setMaxFieldLength(QCLuceneIndexWriter::DEFAULT_MAX_FIELD_LENGTH);
731
732 QStringList namespaces;
733 foreach(const QString &namespaceName, registeredDocs) {
734 mutexLocker.relock();
735 if (m_cancel) {
736 writer->close();
737 delete writer;
738 emit indexingFinished();
739 return;
740 }
741 mutexLocker.unlock();
742
743 namespaces.append(namespaceName);
744 if (indexMap.contains(namespaceName))
745 continue;
746
747 const QList<QStringList> attributeSets =
748 engine.filterAttributeSets(namespaceName);
749
750 if (attributeSets.isEmpty()) {
751 const QList<QUrl> docFiles = indexableFiles(&engine, namespaceName,
752 QStringList());
753 if (!addDocuments(docFiles, engine, QStringList(), namespaceName,
754 writer, analyzer))
755 break;
756 } else {
757 bool bail = false;
758 foreach (const QStringList &attributes, attributeSets) {
759 const QList<QUrl> docFiles = indexableFiles(&engine,
760 namespaceName, attributes);
761 if (!addDocuments(docFiles, engine, attributes, namespaceName,
762 writer, analyzer)) {
763 bail = true;
764 break;
765 }
766 }
767 if (bail)
768 break;
769 }
770
771 mutexLocker.relock();
772 if (!m_cancel) {
773 QString path(engine.documentationFileName(namespaceName));
774 indexMap.insert(namespaceName, QFileInfo(path).lastModified());
775 writeIndexMap(engine, indexMap);
776 }
777 mutexLocker.unlock();
778 }
779
780 writer->close();
781 delete writer;
782
783 mutexLocker.relock();
784 if (!m_cancel) {
785 mutexLocker.unlock();
786
787 QStringList indexedNamespaces = indexMap.keys();
788 foreach(const QString &namespaceName, indexedNamespaces) {
789 mutexLocker.relock();
790 if (m_cancel)
791 break;
792 mutexLocker.unlock();
793
794 if (!namespaces.contains(namespaceName)) {
795 indexMap.remove(namespaceName);
796 writeIndexMap(engine, indexMap);
797 removeDocuments(indexPath, namespaceName);
798 }
799 }
800 }
801 emit indexingFinished();
802}
803
804bool QHelpSearchIndexWriter::addDocuments(const QList<QUrl> docFiles,
805 const QHelpEngineCore &engine, const QStringList &attributes,
806 const QString &namespaceName, QCLuceneIndexWriter *writer,
807 QCLuceneAnalyzer &analyzer)
808{
809 QMutexLocker locker(&mutex);
810 const QString attrList = attributes.join(QLatin1String(" "));
811
812 locker.unlock();
813 foreach(const QUrl &url, docFiles) {
814 QCLuceneDocument document;
815 DocumentHelper helper(url.toString(), engine.fileData(url));
816 if (helper.addFieldsToDocument(&document, namespaceName, attrList))
817 writer->addDocument(document, analyzer);
818
819 locker.relock();
820 if (m_cancel)
821 return false;
822 locker.unlock();
823 }
824
825 return true;
826}
827
828void QHelpSearchIndexWriter::removeDocuments(const QString &indexPath,
829 const QString &namespaceName)
830{
831 if (namespaceName.isEmpty() || QCLuceneIndexReader::isLocked(indexPath))
832 return;
833
834 QCLuceneIndexReader reader = QCLuceneIndexReader::open(indexPath);
835 reader.deleteDocuments(QCLuceneTerm(QLatin1String("namespace"),
836 namespaceName));
837
838 reader.close();
839}
840
841bool QHelpSearchIndexWriter::writeIndexMap(QHelpEngineCore &engine,
842 const QMap<QString, QDateTime> &indexMap)
843{
844 QByteArray bArray;
845
846 QDataStream data(&bArray, QIODevice::ReadWrite);
847 data << indexMap;
848
849 return engine.setCustomValue(QLatin1String("CluceneIndexedNamespaces"),
850 bArray);
851}
852
853QList<QUrl> QHelpSearchIndexWriter::indexableFiles(QHelpEngineCore *helpEngine,
854 const QString &namespaceName, const QStringList &attributes) const
855{
856 QList<QUrl> docFiles = helpEngine->files(namespaceName, attributes,
857 QLatin1String("html"));
858 docFiles += helpEngine->files(namespaceName, attributes, QLatin1String("htm"));
859 docFiles += helpEngine->files(namespaceName, attributes, QLatin1String("txt"));
860
861 return docFiles;
862}
863
864
865 } // namespace clucene
866 } // namespace fulltextsearch
867} // namespace qt
868
869QT_END_NAMESPACE
Note: See TracBrowser for help on using the repository browser.