clang
20.0.0git
include
clang
Tooling
Syntax
Tokens.h
Go to the documentation of this file.
1
//===- Tokens.h - collect tokens from preprocessing --------------*- C++-*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
// Record tokens that a preprocessor emits and define operations to map between
9
// the tokens written in a file and tokens produced by the preprocessor.
10
//
11
// When running the compiler, there are two token streams we are interested in:
12
// - "spelled" tokens directly correspond to a substring written in some
13
// source file.
14
// - "expanded" tokens represent the result of preprocessing, parses consumes
15
// this token stream to produce the AST.
16
//
17
// Expanded tokens correspond directly to locations found in the AST, allowing
18
// to find subranges of the token stream covered by various AST nodes. Spelled
19
// tokens correspond directly to the source code written by the user.
20
//
21
// To allow composing these two use-cases, we also define operations that map
22
// between expanded and spelled tokens that produced them (macro calls,
23
// directives, etc).
24
//
25
//===----------------------------------------------------------------------===//
26
27
#ifndef LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H
28
#define LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H
29
30
#include "
clang/Basic/LangOptions.h
"
31
#include "
clang/Basic/SourceLocation.h
"
32
#include "
clang/Basic/SourceManager.h
"
33
#include "
clang/Basic/TokenKinds.h
"
34
#include "
clang/Lex/Token.h
"
35
#include "llvm/ADT/ArrayRef.h"
36
#include "llvm/ADT/DenseMap.h"
37
#include "llvm/ADT/StringRef.h"
38
#include "llvm/Support/Compiler.h"
39
#include "llvm/Support/raw_ostream.h"
40
#include <cstdint>
41
#include <tuple>
42
43
namespace
clang
{
44
class
Preprocessor;
45
46
namespace
syntax {
47
48
/// A half-open character range inside a particular file, the start offset is
49
/// included and the end offset is excluded from the range.
50
struct
FileRange
{
51
/// EXPECTS: File.isValid() && Begin <= End.
52
FileRange
(
FileID
File,
unsigned
BeginOffset,
unsigned
EndOffset);
53
/// EXPECTS: BeginLoc.isValid() && BeginLoc.isFileID().
54
FileRange
(
const
SourceManager
&
SM
,
SourceLocation
BeginLoc,
unsigned
Length);
55
/// EXPECTS: BeginLoc.isValid() && BeginLoc.isFileID(), Begin <= End and files
56
/// are the same.
57
FileRange
(
const
SourceManager
&
SM
,
SourceLocation
BeginLoc,
58
SourceLocation
EndLoc);
59
60
FileID
file
()
const
{
return
File
; }
61
/// Start is a start offset (inclusive) in the corresponding file.
62
unsigned
beginOffset
()
const
{
return
Begin; }
63
/// End offset (exclusive) in the corresponding file.
64
unsigned
endOffset
()
const
{
return
End; }
65
66
unsigned
length
()
const
{
return
End - Begin; }
67
68
/// Check if \p Offset is inside the range.
69
bool
contains
(
unsigned
Offset)
const
{
70
return
Begin <= Offset && Offset < End;
71
}
72
/// Check \p Offset is inside the range or equal to its endpoint.
73
bool
touches
(
unsigned
Offset)
const
{
74
return
Begin <= Offset && Offset <= End;
75
}
76
77
/// Gets the substring that this FileRange refers to.
78
llvm::StringRef
text
(
const
SourceManager
&