clang 20.0.0git
Tokens.h
Go to the documentation of this file.
1//===- Tokens.h - collect tokens from preprocessing --------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Record tokens that a preprocessor emits and define operations to map between
9// the tokens written in a file and tokens produced by the preprocessor.
10//
11// When running the compiler, there are two token streams we are interested in:
12// - "spelled" tokens directly correspond to a substring written in some
13// source file.
14// - "expanded" tokens represent the result of preprocessing, parses consumes
15// this token stream to produce the AST.
16//
17// Expanded tokens correspond directly to locations found in the AST, allowing
18// to find subranges of the token stream covered by various AST nodes. Spelled
19// tokens correspond directly to the source code written by the user.
20//
21// To allow composing these two use-cases, we also define operations that map
22// between expanded and spelled tokens that produced them (macro calls,
23// directives, etc).
24//
25//===----------------------------------------------------------------------===//
26
27#ifndef LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H
28#define LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H
29
34#include "clang/Lex/Token.h"
35#include "llvm/ADT/ArrayRef.h"
36#include "llvm/ADT/DenseMap.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/Support/Compiler.h"
39#include "llvm/Support/raw_ostream.h"
40#include <cstdint>
41#include <tuple>
42
43namespace clang {
44class Preprocessor;
45
46namespace syntax {
47
48/// A half-open character range inside a particular file, the start offset is
49/// included and the end offset is excluded from the range.
50struct FileRange {
51 /// EXPECTS: File.isValid() && Begin <= End.
52 FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset);
53 /// EXPECTS: BeginLoc.isValid() && BeginLoc.isFileID().
54 FileRange(const SourceManager &SM, SourceLocation BeginLoc, unsigned Length);
55 /// EXPECTS: BeginLoc.isValid() && BeginLoc.isFileID(), Begin <= End and files
56 /// are the same.
58 SourceLocation EndLoc);
59
60 FileID file() const { return File; }
61 /// Start is a start offset (inclusive) in the corresponding file.
62 unsigned beginOffset() const { return Begin; }
63 /// End offset (exclusive) in the corresponding file.
64 unsigned endOffset() const { return End; }
65
66 unsigned length() const { return End - Begin; }
67
68 /// Check if \p Offset is inside the range.
69 bool contains(unsigned Offset) const {
70 return Begin <= Offset && Offset < End;
71 }
72 /// Check \p Offset is inside the range or equal to its endpoint.
73 bool touches(unsigned Offset) const {
74 return Begin <= Offset && Offset <= End;
75 }
76
77 /// Gets the substring that this FileRange refers to.
78 llvm::StringRef text(const SourceManager &