Preprocessor.cpp source code [clang_source_code/lib/Lex/Preprocessor.cpp]

1	//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements the Preprocessor interface.
10	//
11	//===----------------------------------------------------------------------===//
12	//
13	// Options to support:
14	// -H - Print the name of each header file used.
15	// -d[DNI] - Dump various things.
16	// -fworking-directory - #line's with preprocessor's working dir.
17	// -fpreprocessed
18	// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19	// -W*
20	// -w
21	//
22	// Messages to emit:
23	// "Multiple include guards may be useful for:\n"
24	//
25	//===----------------------------------------------------------------------===//
26
27	#include "clang/Lex/Preprocessor.h"
28	#include "clang/Basic/FileManager.h"
29	#include "clang/Basic/FileSystemStatCache.h"
30	#include "clang/Basic/IdentifierTable.h"
31	#include "clang/Basic/LLVM.h"
32	#include "clang/Basic/LangOptions.h"
33	#include "clang/Basic/Module.h"
34	#include "clang/Basic/SourceLocation.h"
35	#include "clang/Basic/SourceManager.h"
36	#include "clang/Basic/TargetInfo.h"
37	#include "clang/Lex/CodeCompletionHandler.h"
38	#include "clang/Lex/ExternalPreprocessorSource.h"
39	#include "clang/Lex/HeaderSearch.h"
40	#include "clang/Lex/LexDiagnostic.h"
41	#include "clang/Lex/Lexer.h"
42	#include "clang/Lex/LiteralSupport.h"
43	#include "clang/Lex/MacroArgs.h"
44	#include "clang/Lex/MacroInfo.h"
45	#include "clang/Lex/ModuleLoader.h"
46	#include "clang/Lex/Pragma.h"
47	#include "clang/Lex/PreprocessingRecord.h"
48	#include "clang/Lex/PreprocessorLexer.h"
49	#include "clang/Lex/PreprocessorOptions.h"
50	#include "clang/Lex/ScratchBuffer.h"
51	#include "clang/Lex/Token.h"
52	#include "clang/Lex/TokenLexer.h"
53	#include "llvm/ADT/APInt.h"
54	#include "llvm/ADT/ArrayRef.h"
55	#include "llvm/ADT/DenseMap.h"
56	#include "llvm/ADT/SmallString.h"
57	#include "llvm/ADT/SmallVector.h"
58	#include "llvm/ADT/STLExtras.h"
59	#include "llvm/ADT/StringRef.h"
60	#include "llvm/ADT/StringSwitch.h"
61	#include "llvm/Support/Capacity.h"
62	#include "llvm/Support/ErrorHandling.h"
63	#include "llvm/Support/MemoryBuffer.h"
64	#include "llvm/Support/raw_ostream.h"
65	#include <algorithm>
66	#include <cassert>
67	#include <memory>
68	#include <string>
69	#include <utility>
70	#include <vector>
71
72	using namespace clang;
73
74	LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
75
76	ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
77
78	Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
79	DiagnosticsEngine &diags, LangOptions &opts,
80	SourceManager &SM, HeaderSearch &Headers,
81	ModuleLoader &TheModuleLoader,
82	IdentifierInfoLookup *IILookup, bool OwnsHeaders,
83	TranslationUnitKind TUKind)
84	: PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
85	FileMgr(Headers.getFileMgr()), SourceMgr(SM),
86	ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
87	TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
88	// As the language options may have not been loaded yet (when
89	// deserializing an ASTUnit), adding keywords to the identifier table is
90	// deferred to Preprocessor::Initialize().
91	Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
92	TUKind(TUKind), SkipMainFilePreamble(0, true),
93	CurSubmoduleState(&NullSubmoduleState) {
94	OwnsHeaderSearch = OwnsHeaders;
95
96	// Default to discarding comments.
97	KeepComments = false;
98	KeepMacroComments = false;
99	SuppressIncludeNotFoundError = false;
100
101	// Macro expansion is enabled.
102	DisableMacroExpansion = false;
103	MacroExpansionInDirectivesOverride = false;
104	InMacroArgs = false;
105	ArgMacro = nullptr;
106	InMacroArgPreExpansion = false;
107	NumCachedTokenLexers = 0;
108	PragmasEnabled = true;
109	ParsingIfOrElifDirective = false;
110	PreprocessedOutput = false;
111
112	// We haven't read anything from the external source.
113	ReadMacrosFromExternalSource = false;
114
115	// "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
116	// a macro. They get unpoisoned where it is allowed.
117	(Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
118	SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
119	if (getLangOpts().CPlusPlus2a) {
120	(Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
121	SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
122	} else {
123	Ident__VA_OPT__ = nullptr;
124	}
125
126	// Initialize the pragma handlers.
127	RegisterBuiltinPragmas();
128
129	// Initialize builtin macros like __LINE__ and friends.
130	RegisterBuiltinMacros();
131
132	if(LangOpts.Borland) {
133	Ident__exception_info = getIdentifierInfo("_exception_info");
134	Ident___exception_info = getIdentifierInfo("__exception_info");
135	Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
136	Ident__exception_code = getIdentifierInfo("_exception_code");
137	Ident___exception_code = getIdentifierInfo("__exception_code");
138	Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
139	Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
140	Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
141	Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
142	} else {
143	Ident__exception_info = Ident__exception_code = nullptr;
144	Ident__abnormal_termination = Ident___exception_info = nullptr;
145	Ident___exception_code = Ident___abnormal_termination = nullptr;
146	Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
147	Ident_AbnormalTermination = nullptr;
148	}
149
150	// If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
151	if (usingPCHWithPragmaHdrStop())
152	SkippingUntilPragmaHdrStop = true;
153
154	// If using a PCH with a through header, start skipping tokens.
155	if (!this->PPOpts->PCHThroughHeader.empty() &&
156	!this->PPOpts->ImplicitPCHInclude.empty())
157	SkippingUntilPCHThroughHeader = true;
158
159	if (this->PPOpts->GeneratePreamble)
160	PreambleConditionalStack.startRecording();
161	}
162
163	Preprocessor::~Preprocessor() {
164	(0) . __assert_fail ("BacktrackPositions.empty() && \"EnableBacktrack/Backtrack imbalance!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 164, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
165
166	IncludeMacroStack.clear();
167
168	// Destroy any macro definitions.
169	while (MacroInfoChain *I = MIChainHead) {
170	MIChainHead = I->Next;
171	I->~MacroInfoChain();
172	}
173
174	// Free any cached macro expanders.
175	// This populates MacroArgCache, so all TokenLexers need to be destroyed
176	// before the code below that frees up the MacroArgCache list.
177	std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
178	CurTokenLexer.reset();
179
180	// Free any cached MacroArgs.
181	for (MacroArgs *ArgList = MacroArgCache; ArgList;)
182	ArgList = ArgList->deallocate();
183
184	// Delete the header search info, if we own it.
185	if (OwnsHeaderSearch)
186	delete &HeaderInfo;
187	}
188
189	void Preprocessor::Initialize(const TargetInfo &Target,
190	const TargetInfo *AuxTarget) {
191	(0) . __assert_fail ("(!this->Target \|\| this->Target == &Target) && \"Invalid override of target information\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 192, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((!this->Target \|\| this->Target == &Target) &&
192	(0) . __assert_fail ("(!this->Target \|\| this->Target == &Target) && \"Invalid override of target information\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 192, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Invalid override of target information");
193	this->Target = &Target;
194
195	(0) . __assert_fail ("(!this->AuxTarget \|\| this->AuxTarget == AuxTarget) && \"Invalid override of aux target information.\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 196, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert((!this->AuxTarget \|\| this->AuxTarget == AuxTarget) &&
196	(0) . __assert_fail ("(!this->AuxTarget \|\| this->AuxTarget == AuxTarget) && \"Invalid override of aux target information.\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 196, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Invalid override of aux target information.");
197	this->AuxTarget = AuxTarget;
198
199	// Initialize information about built-ins.
200	BuiltinInfo.InitializeTarget(Target, AuxTarget);
201	HeaderInfo.setTarget(Target);
202
203	// Populate the identifier table with info about keywords for the current language.
204	Identifiers.AddKeywords(LangOpts);
205	}
206
207	void Preprocessor::InitializeForModelFile() {
208	NumEnteredSourceFiles = 0;
209
210	// Reset pragmas
211	PragmaHandlersBackup = std::move(PragmaHandlers);
212	PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
213	RegisterBuiltinPragmas();
214
215	// Reset PredefinesFileID
216	PredefinesFileID = FileID();
217	}
218
219	void Preprocessor::FinalizeForModelFile() {
220	NumEnteredSourceFiles = 1;
221
222	PragmaHandlers = std::move(PragmaHandlersBackup);
223	}
224
225	void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
226	llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
227	<< getSpelling(Tok) << "'";
228
229	if (!DumpFlags) return;
230
231	llvm::errs() << "\t";
232	if (Tok.isAtStartOfLine())
233	llvm::errs() << " [StartOfLine]";
234	if (Tok.hasLeadingSpace())
235	llvm::errs() << " [LeadingSpace]";
236	if (Tok.isExpandDisabled())
237	llvm::errs() << " [ExpandDisabled]";
238	if (Tok.needsCleaning()) {
239	const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
240	llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
241	<< "']";
242	}
243
244	llvm::errs() << "\tLoc=<";
245	DumpLocation(Tok.getLocation());
246	llvm::errs() << ">";
247	}
248
249	void Preprocessor::DumpLocation(SourceLocation Loc) const {
250	Loc.print(llvm::errs(), SourceMgr);
251	}
252
253	void Preprocessor::DumpMacro(const MacroInfo &MI) const {
254	llvm::errs() << "MACRO: ";
255	for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
256	DumpToken(MI.getReplacementToken(i));
257	llvm::errs() << " ";
258	}
259	llvm::errs() << "\n";
260	}
261
262	void Preprocessor::PrintStats() {
263	llvm::errs() << "\n*** Preprocessor Stats:\n";
264	llvm::errs() << NumDirectives << " directives found:\n";
265	llvm::errs() << " " << NumDefined << " #define.\n";
266	llvm::errs() << " " << NumUndefined << " #undef.\n";
267	llvm::errs() << " #include/#include_next/#import:\n";
268	llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
269	llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
270	llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
271	llvm::errs() << " " << NumElse << " #else/#elif.\n";
272	llvm::errs() << " " << NumEndif << " #endif.\n";
273	llvm::errs() << " " << NumPragma << " #pragma.\n";
274	llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
275
276	llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
277	<< NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
278	<< NumFastMacroExpanded << " on the fast path.\n";
279	llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
280	<< " token paste (##) operations performed, "
281	<< NumFastTokenPaste << " on the fast path.\n";
282
283	llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
284
285	llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
286	llvm::errs() << "\n Macro Expanded Tokens: "
287	<< llvm::capacity_in_bytes(MacroExpandedTokens);
288	llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
289	// FIXME: List information for all submodules.
290	llvm::errs() << "\n Macros: "
291	<< llvm::capacity_in_bytes(CurSubmoduleState->Macros);
292	llvm::errs() << "\n #pragma push_macro Info: "
293	<< llvm::capacity_in_bytes(PragmaPushMacroInfo);
294	llvm::errs() << "\n Poison Reasons: "
295	<< llvm::capacity_in_bytes(PoisonReasons);
296	llvm::errs() << "\n Comment Handlers: "
297	<< llvm::capacity_in_bytes(CommentHandlers) << "\n";
298	}
299
300	Preprocessor::macro_iterator
301	Preprocessor::macro_begin(bool IncludeExternalMacros) const {
302	if (IncludeExternalMacros && ExternalSource &&
303	!ReadMacrosFromExternalSource) {
304	ReadMacrosFromExternalSource = true;
305	ExternalSource->ReadDefinedMacros();
306	}
307
308	// Make sure we cover all macros in visible modules.
309	for (const ModuleMacro &Macro : ModuleMacros)
310	CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
311
312	return CurSubmoduleState->Macros.begin();
313	}
314
315	size_t Preprocessor::getTotalMemory() const {
316	return BP.getTotalMemory()
317	+ llvm::capacity_in_bytes(MacroExpandedTokens)
318	+ Predefines.capacity() /* Predefines buffer. */
319	// FIXME: Include sizes from all submodules, and include MacroInfo sizes,
320	// and ModuleMacros.
321	+ llvm::capacity_in_bytes(CurSubmoduleState->Macros)
322	+ llvm::capacity_in_bytes(PragmaPushMacroInfo)
323	+ llvm::capacity_in_bytes(PoisonReasons)
324	+ llvm::capacity_in_bytes(CommentHandlers);
325	}
326
327	Preprocessor::macro_iterator
328	Preprocessor::macro_end(bool IncludeExternalMacros) const {
329	if (IncludeExternalMacros && ExternalSource &&
330	!ReadMacrosFromExternalSource) {
331	ReadMacrosFromExternalSource = true;
332	ExternalSource->ReadDefinedMacros();
333	}
334
335	return CurSubmoduleState->Macros.end();
336	}
337
338	/// Compares macro tokens with a specified token value sequence.
339	static bool MacroDefinitionEquals(const MacroInfo *MI,
340	ArrayRef<TokenValue> Tokens) {
341	return Tokens.size() == MI->getNumTokens() &&
342	std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
343	}
344
345	StringRef Preprocessor::getLastMacroWithSpelling(
346	SourceLocation Loc,
347	ArrayRef<TokenValue> Tokens) const {
348	SourceLocation BestLocation;
349	StringRef BestSpelling;
350	for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
351	I != E; ++I) {
352	const MacroDirective::DefInfo
353	Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
354	if (!Def \|\| !Def.getMacroInfo())
355	continue;
356	if (!Def.getMacroInfo()->isObjectLike())
357	continue;
358	if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
359	continue;
360	SourceLocation Location = Def.getLocation();
361	// Choose the macro defined latest.
362	if (BestLocation.isInvalid() \|\|
363	(Location.isValid() &&
364	SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
365	BestLocation = Location;
366	BestSpelling = I->first->getName();
367	}
368	}
369	return BestSpelling;
370	}
371
372	void Preprocessor::recomputeCurLexerKind() {
373	if (CurLexer)
374	CurLexerKind = CLK_Lexer;
375	else if (CurTokenLexer)
376	CurLexerKind = CLK_TokenLexer;
377	else
378	CurLexerKind = CLK_CachingLexer;
379	}
380
381	bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
382	unsigned CompleteLine,
383	unsigned CompleteColumn) {
384	assert(File);
385	(0) . __assert_fail ("CompleteLine && CompleteColumn && \"Starts from 1.1\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 385, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CompleteLine && CompleteColumn && "Starts from 1:1");
386	(0) . __assert_fail ("!CodeCompletionFile && \"Already set\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 386, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!CodeCompletionFile && "Already set");
387
388	using llvm::MemoryBuffer;
389
390	// Load the actual file's contents.
391	bool Invalid = false;
392	const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
393	if (Invalid)
394	return true;
395
396	// Find the byte position of the truncation point.
397	const char *Position = Buffer->getBufferStart();
398	for (unsigned Line = 1; Line < CompleteLine; ++Line) {
399	for (; *Position; ++Position) {
400	if (Position != '\r' && Position != '\n')
401	continue;
402
403	// Eat \r\n or \n\r as a single line.
404	if ((Position[1] == '\r' \|\| Position[1] == '\n') &&
405	Position[0] != Position[1])
406	++Position;
407	++Position;
408	break;
409	}
410	}
411
412	Position += CompleteColumn - 1;
413
414	// If pointing inside the preamble, adjust the position at the beginning of
415	// the file after the preamble.
416	if (SkipMainFilePreamble.first &&
417	SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
418	if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
419	Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
420	}
421
422	if (Position > Buffer->getBufferEnd())
423	Position = Buffer->getBufferEnd();
424
425	CodeCompletionFile = File;
426	CodeCompletionOffset = Position - Buffer->getBufferStart();
427
428	auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
429	Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
430	char *NewBuf = NewBuffer->getBufferStart();
431	char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
432	*NewPos = '\0';
433	std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
434	SourceMgr.overrideFileContents(File, std::move(NewBuffer));
435
436	return false;
437	}
438
439	void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
440	bool IsAngled) {
441	if (CodeComplete)
442	CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
443	setCodeCompletionReached();
444	}
445
446	void Preprocessor::CodeCompleteNaturalLanguage() {
447	if (CodeComplete)
448	CodeComplete->CodeCompleteNaturalLanguage();
449	setCodeCompletionReached();
450	}
451
452	/// getSpelling - This method is used to get the spelling of a token into a
453	/// SmallVector. Note that the returned StringRef may not point to the
454	/// supplied buffer if a copy can be avoided.
455	StringRef Preprocessor::getSpelling(const Token &Tok,
456	SmallVectorImpl<char> &Buffer,
457	bool *Invalid) const {
458	// NOTE: this has to be checked before testing for an IdentifierInfo.
459	if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
460	// Try the fast path.
461	if (const IdentifierInfo *II = Tok.getIdentifierInfo())
462	return II->getName();
463	}
464
465	// Resize the buffer if we need to copy into it.
466	if (Tok.needsCleaning())
467	Buffer.resize(Tok.getLength());
468
469	const char *Ptr = Buffer.data();
470	unsigned Len = getSpelling(Tok, Ptr, Invalid);
471	return StringRef(Ptr, Len);
472	}
473
474	/// CreateString - Plop the specified string into a scratch buffer and return a
475	/// location for it. If specified, the source location provides a source
476	/// location for the token.
477	void Preprocessor::CreateString(StringRef Str, Token &Tok,
478	SourceLocation ExpansionLocStart,
479	SourceLocation ExpansionLocEnd) {
480	Tok.setLength(Str.size());
481
482	const char *DestPtr;
483	SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
484
485	if (ExpansionLocStart.isValid())
486	Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
487	ExpansionLocEnd, Str.size());
488	Tok.setLocation(Loc);
489
490	// If this is a raw identifier or a literal token, set the pointer data.
491	if (Tok.is(tok::raw_identifier))
492	Tok.setRawIdentifierData(DestPtr);
493	else if (Tok.isLiteral())
494	Tok.setLiteralData(DestPtr);
495	}
496
497	SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
498	auto &SM = getSourceManager();
499	SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
500	std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
501	bool Invalid = false;
502	StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
503	if (Invalid)
504	return SourceLocation();
505
506	// FIXME: We could consider re-using spelling for tokens we see repeatedly.
507	const char *DestPtr;
508	SourceLocation Spelling =
509	ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
510	return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
511	}
512
513	Module *Preprocessor::getCurrentModule() {
514	if (!getLangOpts().isCompilingModule())
515	return nullptr;
516
517	return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
518	}
519
520	//===----------------------------------------------------------------------===//
521	// Preprocessor Initialization Methods
522	//===----------------------------------------------------------------------===//
523
524	/// EnterMainSourceFile - Enter the specified FileID as the main source file,
525	/// which implicitly adds the builtin defines etc.
526	void Preprocessor::EnterMainSourceFile() {
527	// We do not allow the preprocessor to reenter the main file. Doing so will
528	// cause FileID's to accumulate information from both runs (e.g. #line
529	// information) and predefined macros aren't guaranteed to be set properly.
530	(0) . __assert_fail ("NumEnteredSourceFiles == 0 && \"Cannot reenter the main file!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 530, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
531	FileID MainFileID = SourceMgr.getMainFileID();
532
533	// If MainFileID is loaded it means we loaded an AST file, no need to enter
534	// a main file.
535	if (!SourceMgr.isLoadedFileID(MainFileID)) {
536	// Enter the main file source buffer.
537	EnterSourceFile(MainFileID, nullptr, SourceLocation());
538
539	// If we've been asked to skip bytes in the main file (e.g., as part of a
540	// precompiled preamble), do so now.
541	if (SkipMainFilePreamble.first > 0)
542	CurLexer->SetByteOffset(SkipMainFilePreamble.first,
543	SkipMainFilePreamble.second);
544
545	// Tell the header info that the main file was entered. If the file is later
546	// #imported, it won't be re-entered.
547	if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
548	HeaderInfo.IncrementIncludeCount(FE);
549	}
550
551	// Preprocess Predefines to populate the initial preprocessor state.
552	std::unique_ptr<llvm::MemoryBuffer> SB =
553	llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
554	(0) . __assert_fail ("SB && \"Cannot create predefined source buffer\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 554, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(SB && "Cannot create predefined source buffer");
555	FileID FID = SourceMgr.createFileID(std::move(SB));
556	(0) . __assert_fail ("FID.isValid() && \"Could not create FileID for predefines?\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 556, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(FID.isValid() && "Could not create FileID for predefines?");
557	setPredefinesFileID(FID);
558
559	// Start parsing the predefines.
560	EnterSourceFile(FID, nullptr, SourceLocation());
561
562	if (!PPOpts->PCHThroughHeader.empty()) {
563	// Lookup and save the FileID for the through header. If it isn't found
564	// in the search path, it's a fatal error.
565	const DirectoryLookup *CurDir;
566	const FileEntry *File = LookupFile(
567	SourceLocation(), PPOpts->PCHThroughHeader,
568	/isAngled=/false, /FromDir=/nullptr, /FromFile=/nullptr, CurDir,
569	/SearchPath=/nullptr, /RelativePath=/nullptr,
570	/SuggestedModule=/nullptr, /IsMapped=/nullptr,
571	/IsFrameworkFound=/nullptr);
572	if (!File) {
573	Diag(SourceLocation(), diag::err_pp_through_header_not_found)
574	<< PPOpts->PCHThroughHeader;
575	return;
576	}
577	setPCHThroughHeaderFileID(
578	SourceMgr.createFileID(File, SourceLocation(), SrcMgr::C_User));
579	}
580
581	// Skip tokens from the Predefines and if needed the main file.
582	if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) \|\|
583	(usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
584	SkipTokensWhileUsingPCH();
585	}
586
587	void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
588	(0) . __assert_fail ("PCHThroughHeaderFileID.isInvalid() && \"PCHThroughHeaderFileID already set!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 589, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(PCHThroughHeaderFileID.isInvalid() &&
589	(0) . __assert_fail ("PCHThroughHeaderFileID.isInvalid() && \"PCHThroughHeaderFileID already set!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 589, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "PCHThroughHeaderFileID already set!");
590	PCHThroughHeaderFileID = FID;
591	}
592
593	bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
594	(0) . __assert_fail ("PCHThroughHeaderFileID.isValid() && \"Invalid PCH through header FileID\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 595, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(PCHThroughHeaderFileID.isValid() &&
595	(0) . __assert_fail ("PCHThroughHeaderFileID.isValid() && \"Invalid PCH through header FileID\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 595, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Invalid PCH through header FileID");
596	return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
597	}
598
599	bool Preprocessor::creatingPCHWithThroughHeader() {
600	return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
601	PCHThroughHeaderFileID.isValid();
602	}
603
604	bool Preprocessor::usingPCHWithThroughHeader() {
605	return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
606	PCHThroughHeaderFileID.isValid();
607	}
608
609	bool Preprocessor::creatingPCHWithPragmaHdrStop() {
610	return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
611	}
612
613	bool Preprocessor::usingPCHWithPragmaHdrStop() {
614	return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
615	}
616
617	/// Skip tokens until after the #include of the through header or
618	/// until after a #pragma hdrstop is seen. Tokens in the predefines file
619	/// and the main file may be skipped. If the end of the predefines file
620	/// is reached, skipping continues into the main file. If the end of the
621	/// main file is reached, it's a fatal error.
622	void Preprocessor::SkipTokensWhileUsingPCH() {
623	bool ReachedMainFileEOF = false;
624	bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
625	bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
626	Token Tok;
627	while (true) {
628	bool InPredefines = (CurLexer->getFileID() == getPredefinesFileID());
629	CurLexer->Lex(Tok);
630	if (Tok.is(tok::eof) && !InPredefines) {
631	ReachedMainFileEOF = true;
632	break;
633	}
634	if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
635	break;
636	if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
637	break;
638	}
639	if (ReachedMainFileEOF) {
640	if (UsingPCHThroughHeader)
641	Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
642	<< PPOpts->PCHThroughHeader << 1;
643	else if (!PPOpts->PCHWithHdrStopCreate)
644	Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
645	}
646	}
647
648	void Preprocessor::replayPreambleConditionalStack() {
649	// Restore the conditional stack from the preamble, if there is one.
650	if (PreambleConditionalStack.isReplaying()) {
651	(0) . __assert_fail ("CurPPLexer && \"CurPPLexer is null when calling replayPreambleConditionalStack.\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 652, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CurPPLexer &&
652	(0) . __assert_fail ("CurPPLexer && \"CurPPLexer is null when calling replayPreambleConditionalStack.\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 652, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "CurPPLexer is null when calling replayPreambleConditionalStack.");
653	CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
654	PreambleConditionalStack.doneReplaying();
655	if (PreambleConditionalStack.reachedEOFWhileSkipping())
656	SkipExcludedConditionalBlock(
657	PreambleConditionalStack.SkipInfo->HashTokenLoc,
658	PreambleConditionalStack.SkipInfo->IfTokenLoc,
659	PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
660	PreambleConditionalStack.SkipInfo->FoundElse,
661	PreambleConditionalStack.SkipInfo->ElseLoc);
662	}
663	}
664
665	void Preprocessor::EndSourceFile() {
666	// Notify the client that we reached the end of the source file.
667	if (Callbacks)
668	Callbacks->EndOfMainFile();
669	}
670
671	//===----------------------------------------------------------------------===//
672	// Lexer Event Handling.
673	//===----------------------------------------------------------------------===//
674
675	/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
676	/// identifier information for the token and install it into the token,
677	/// updating the token kind accordingly.
678	IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
679	(0) . __assert_fail ("!Identifier.getRawIdentifier().empty() && \"No raw identifier data!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 679, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
680
681	// Look up this token, see if it is a macro, or if it is a language keyword.
682	IdentifierInfo *II;
683	if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
684	// No cleaning needed, just use the characters from the lexed buffer.
685	II = getIdentifierInfo(Identifier.getRawIdentifier());
686	} else {
687	// Cleaning needed, alloca a buffer, clean into it, then use the buffer.
688	SmallString<64> IdentifierBuffer;
689	StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
690
691	if (Identifier.hasUCN()) {
692	SmallString<64> UCNIdentifierBuffer;
693	expandUCNs(UCNIdentifierBuffer, CleanedStr);
694	II = getIdentifierInfo(UCNIdentifierBuffer);
695	} else {
696	II = getIdentifierInfo(CleanedStr);
697	}
698	}
699
700	// Update the token info (identifier info and appropriate token kind).
701	Identifier.setIdentifierInfo(II);
702	if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
703	getSourceManager().isInSystemHeader(Identifier.getLocation()))
704	Identifier.setKind(tok::identifier);
705	else
706	Identifier.setKind(II->getTokenID());
707
708	return II;
709	}
710
711	void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
712	PoisonReasons[II] = DiagID;
713	}
714
715	void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
716	assert(Ident__exception_code && Ident__exception_info);
717	assert(Ident___exception_code && Ident___exception_info);
718	Ident__exception_code->setIsPoisoned(Poison);
719	Ident___exception_code->setIsPoisoned(Poison);
720	Ident_GetExceptionCode->setIsPoisoned(Poison);
721	Ident__exception_info->setIsPoisoned(Poison);
722	Ident___exception_info->setIsPoisoned(Poison);
723	Ident_GetExceptionInfo->setIsPoisoned(Poison);
724	Ident__abnormal_termination->setIsPoisoned(Poison);
725	Ident___abnormal_termination->setIsPoisoned(Poison);
726	Ident_AbnormalTermination->setIsPoisoned(Poison);
727	}
728
729	void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
730	(0) . __assert_fail ("Identifier.getIdentifierInfo() && \"Can't handle identifiers without identifier info!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 731, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Identifier.getIdentifierInfo() &&
731	(0) . __assert_fail ("Identifier.getIdentifierInfo() && \"Can't handle identifiers without identifier info!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 731, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Can't handle identifiers without identifier info!");
732	llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
733	PoisonReasons.find(Identifier.getIdentifierInfo());
734	if(it == PoisonReasons.end())
735	Diag(Identifier, diag::err_pp_used_poisoned_id);
736	else
737	Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
738	}
739
740	/// Returns a diagnostic message kind for reporting a future keyword as
741	/// appropriate for the identifier and specified language.
742	static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II,
743	const LangOptions &LangOpts) {
744	(0) . __assert_fail ("II.isFutureCompatKeyword() && \"diagnostic should not be needed\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 744, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
745
746	if (LangOpts.CPlusPlus)
747	return llvm::StringSwitch<diag::kind>(II.getName())
748	#define CXX11_KEYWORD(NAME, FLAGS) \
749	.Case(#NAME, diag::warn_cxx11_keyword)
750	#define CXX2A_KEYWORD(NAME, FLAGS) \
751	.Case(#NAME, diag::warn_cxx2a_keyword)
752	#include "clang/Basic/TokenKinds.def"
753	;
754
755	llvm_unreachable(
756	"Keyword not known to come from a newer Standard or proposed Standard");
757	}
758
759	void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
760	(0) . __assert_fail ("II.isOutOfDate() && \"not out of date\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 760, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(II.isOutOfDate() && "not out of date");
761	getExternalSource()->updateOutOfDateIdentifier(II);
762	}
763
764	/// HandleIdentifier - This callback is invoked when the lexer reads an
765	/// identifier. This callback looks up the identifier in the map and/or
766	/// potentially macro expands it or turns it into a named token (like 'for').
767	///
768	/// Note that callers of this method are guarded by checking the
769	/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
770	/// IdentifierInfo methods that compute these properties will need to change to
771	/// match.
772	bool Preprocessor::HandleIdentifier(Token &Identifier) {
773	(0) . __assert_fail ("Identifier.getIdentifierInfo() && \"Can't handle identifiers without identifier info!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 774, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Identifier.getIdentifierInfo() &&
774	(0) . __assert_fail ("Identifier.getIdentifierInfo() && \"Can't handle identifiers without identifier info!\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 774, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Can't handle identifiers without identifier info!");
775
776	IdentifierInfo &II = *Identifier.getIdentifierInfo();
777
778	// If the information about this identifier is out of date, update it from
779	// the external source.
780	// We have to treat __VA_ARGS__ in a special way, since it gets
781	// serialized with isPoisoned = true, but our preprocessor may have
782	// unpoisoned it if we're defining a C99 macro.
783	if (II.isOutOfDate()) {
784	bool CurrentIsPoisoned = false;
785	const bool IsSpecialVariadicMacro =
786	&II == Ident__VA_ARGS__ \|\| &II == Ident__VA_OPT__;
787	if (IsSpecialVariadicMacro)
788	CurrentIsPoisoned = II.isPoisoned();
789
790	updateOutOfDateIdentifier(II);
791	Identifier.setKind(II.getTokenID());
792
793	if (IsSpecialVariadicMacro)
794	II.setIsPoisoned(CurrentIsPoisoned);
795	}
796
797	// If this identifier was poisoned, and if it was not produced from a macro
798	// expansion, emit an error.
799	if (II.isPoisoned() && CurPPLexer) {
800	HandlePoisonedIdentifier(Identifier);
801	}
802
803	// If this is a macro to be expanded, do it.
804	if (MacroDefinition MD = getMacroDefinition(&II)) {
805	auto *MI = MD.getMacroInfo();
806	(0) . __assert_fail ("MI && \"macro definition with no macro info?\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 806, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(MI && "macro definition with no macro info?");
807	if (!DisableMacroExpansion) {
808	if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
809	// C99 6.10.3p10: If the preprocessing token immediately after the
810	// macro name isn't a '(', this macro should not be expanded.
811	if (!MI->isFunctionLike() \|\| isNextPPTokenLParen())
812	return HandleMacroExpandedIdentifier(Identifier, MD);
813	} else {
814	// C99 6.10.3.4p2 says that a disabled macro may never again be
815	// expanded, even if it's in a context where it could be expanded in the
816	// future.
817	Identifier.setFlag(Token::DisableExpand);
818	if (MI->isObjectLike() \|\| isNextPPTokenLParen())
819	Diag(Identifier, diag::pp_disabled_macro_expansion);
820	}
821	}
822	}
823
824	// If this identifier is a keyword in a newer Standard or proposed Standard,
825	// produce a warning. Don't warn if we're not considering macro expansion,
826	// since this identifier might be the name of a macro.
827	// FIXME: This warning is disabled in cases where it shouldn't be, like
828	// "#define constexpr constexpr", "int constexpr;"
829	if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
830	Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
831	<< II.getName();
832	// Don't diagnose this keyword again in this translation unit.
833	II.setIsFutureCompatKeyword(false);
834	}
835
836	// If this is an extension token, diagnose its use.
837	// We avoid diagnosing tokens that originate from macro definitions.
838	// FIXME: This warning is disabled in cases where it shouldn't be,
839	// like "#define TY typeof", "TY(1) x".
840	if (II.isExtensionToken() && !DisableMacroExpansion)
841	Diag(Identifier, diag::ext_token_used);
842
843	// If this is the 'import' contextual keyword following an '@', note
844	// that the next token indicates a module name.
845	//
846	// Note that we do not treat 'import' as a contextual
847	// keyword when we're in a caching lexer, because caching lexers only get
848	// used in contexts where import declarations are disallowed.
849	//
850	// Likewise if this is the C++ Modules TS import keyword.
851	if (((LastTokenWasAt && II.isModulesImport()) \|\|
852	Identifier.is(tok::kw_import)) &&
853	!InMacroArgs && !DisableMacroExpansion &&
854	(getLangOpts().Modules \|\| getLangOpts().DebuggerSupport) &&
855	CurLexerKind != CLK_CachingLexer) {
856	ModuleImportLoc = Identifier.getLocation();
857	ModuleImportPath.clear();
858	ModuleImportExpectsIdentifier = true;
859	CurLexerKind = CLK_LexAfterModuleImport;
860	}
861	return true;
862	}
863
864	void Preprocessor::Lex(Token &Result) {
865	// We loop here until a lex function returns a token; this avoids recursion.
866	bool ReturnedToken;
867	do {
868	switch (CurLexerKind) {
869	case CLK_Lexer:
870	ReturnedToken = CurLexer->Lex(Result);
871	break;
872	case CLK_TokenLexer:
873	ReturnedToken = CurTokenLexer->Lex(Result);
874	break;
875	case CLK_CachingLexer:
876	CachingLex(Result);
877	ReturnedToken = true;
878	break;
879	case CLK_LexAfterModuleImport:
880	LexAfterModuleImport(Result);
881	ReturnedToken = true;
882	break;
883	}
884	} while (!ReturnedToken);
885
886	if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
887	// Remember the identifier before code completion token.
888	setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
889	setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc());
890	// Set IdenfitierInfo to null to avoid confusing code that handles both
891	// identifiers and completion tokens.
892	Result.setIdentifierInfo(nullptr);
893	}
894
895	LastTokenWasAt = Result.is(tok::at);
896	}
897
898	/// Lex a header-name token (including one formed from header-name-tokens if
899	/// \p AllowConcatenation is \c true).
900	///
901	/// \param FilenameTok Filled in with the next token. On success, this will
902	/// be either an angle_header_name or a string_literal token. On
903	/// failure, it will be whatever other token was found instead.
904	/// \param AllowMacroExpansion If \c true, allow the header name to be formed
905	/// by macro expansion (concatenating tokens as necessary if the first
906	/// token is a '<').
907	/// \return \c true if we reached EOD or EOF while looking for a > token in
908	/// a concatenated header name and diagnosed it. \c false otherwise.
909	bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
910	// Lex using header-name tokenization rules if tokens are being lexed from
911	// a file. Just grab a token normally if we're in a macro expansion.
912	if (CurPPLexer)
913	CurPPLexer->LexIncludeFilename(FilenameTok);
914	else
915	Lex(FilenameTok);
916
917	// This could be a <foo/bar.h> file coming from a macro expansion. In this
918	// case, glue the tokens together into an angle_string_literal token.
919	SmallString<128> FilenameBuffer;
920	if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
921	SourceLocation Start = FilenameTok.getLocation();
922	SourceLocation End;
923	FilenameBuffer.push_back('<');
924
925	// Consume tokens until we find a '>'.
926	// FIXME: A header-name could be formed starting or ending with an
927	// alternative token. It's not clear whether that's ill-formed in all
928	// cases.
929	while (FilenameTok.isNot(tok::greater)) {
930	Lex(FilenameTok);
931	if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
932	Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
933	Diag(Start, diag::note_matching) << tok::less;
934	return true;
935	}
936
937	End = FilenameTok.getLocation();
938
939	// FIXME: Provide code completion for #includes.
940	if (FilenameTok.is(tok::code_completion)) {
941	setCodeCompletionReached();
942	Lex(FilenameTok);
943	continue;
944	}
945
946	// Append the spelling of this token to the buffer. If there was a space
947	// before it, add it now.
948	if (FilenameTok.hasLeadingSpace())
949	FilenameBuffer.push_back(' ');
950
951	// Get the spelling of the token, directly into FilenameBuffer if
952	// possible.
953	size_t PreAppendSize = FilenameBuffer.size();
954	FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
955
956	const char *BufPtr = &FilenameBuffer[PreAppendSize];
957	unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
958
959	// If the token was spelled somewhere else, copy it into FilenameBuffer.
960	if (BufPtr != &FilenameBuffer[PreAppendSize])
961	memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
962
963	// Resize FilenameBuffer to the correct size.
964	if (FilenameTok.getLength() != ActualLen)
965	FilenameBuffer.resize(PreAppendSize + ActualLen);
966	}
967
968	FilenameTok.startToken();
969	FilenameTok.setKind(tok::header_name);
970	CreateString(FilenameBuffer, FilenameTok, Start, End);
971	} else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
972	// Convert a string-literal token of the form " h-char-sequence "
973	// (produced by macro expansion) into a header-name token.
974	//
975	// The rules for header-names don't quite match the rules for
976	// string-literals, but all the places where they differ result in
977	// undefined behavior, so we can and do treat them the same.
978	//
979	// A string-literal with a prefix or suffix is not translated into a
980	// header-name. This could theoretically be observable via the C++20
981	// context-sensitive header-name formation rules.
982	StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
983	if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
984	FilenameTok.setKind(tok::header_name);
985	}
986
987	return false;
988	}
989
990	/// Lex a token following the 'import' contextual keyword.
991	///
992	void Preprocessor::LexAfterModuleImport(Token &Result) {
993	// Figure out what kind of lexer we actually have.
994	recomputeCurLexerKind();
995
996	// Lex the next token.
997	Lex(Result);
998
999	// The token sequence
1000	//
1001	// import identifier (. identifier)*
1002	//
1003	// indicates a module import directive. We already saw the 'import'
1004	// contextual keyword, so now we're looking for the identifiers.
1005	if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1006	// We expected to see an identifier here, and we did; continue handling
1007	// identifiers.
1008	ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1009	Result.getLocation()));
1010	ModuleImportExpectsIdentifier = false;
1011	CurLexerKind = CLK_LexAfterModuleImport;
1012	return;
1013	}
1014
1015	// If we're expecting a '.' or a ';', and we got a '.', then wait until we
1016	// see the next identifier. (We can also see a '[[' that begins an
1017	// attribute-specifier-seq here under the C++ Modules TS.)
1018	if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1019	ModuleImportExpectsIdentifier = true;
1020	CurLexerKind = CLK_LexAfterModuleImport;
1021	return;
1022	}
1023
1024	// If we have a non-empty module path, load the named module.
1025	if (!ModuleImportPath.empty()) {
1026	// Under the Modules TS, the dot is just part of the module name, and not
1027	// a real hierarchy separator. Flatten such module names now.
1028	//
1029	// FIXME: Is this the right level to be performing this transformation?
1030	std::string FlatModuleName;
1031	if (getLangOpts().ModulesTS) {
1032	for (auto &Piece : ModuleImportPath) {
1033	if (!FlatModuleName.empty())
1034	FlatModuleName += ".";
1035	FlatModuleName += Piece.first->getName();
1036	}
1037	SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1038	ModuleImportPath.clear();
1039	ModuleImportPath.push_back(
1040	std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1041	}
1042
1043	Module *Imported = nullptr;
1044	if (getLangOpts().Modules) {
1045	Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1046	ModuleImportPath,
1047	Module::Hidden,
1048	/IsIncludeDirective=/false);
1049	if (Imported)
1050	makeModuleVisible(Imported, ModuleImportLoc);
1051	}
1052	if (Callbacks && (getLangOpts().Modules \|\| getLangOpts().DebuggerSupport))
1053	Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1054	}
1055	}
1056
1057	void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) {
1058	CurSubmoduleState->VisibleModules.setVisible(
1059	M, Loc, [](Module *) {},
1060	[&](ArrayRef<Module > Path, Module Conflict, StringRef Message) {
1061	// FIXME: Include the path in the diagnostic.
1062	// FIXME: Include the import location for the conflicting module.
1063	Diag(ModuleImportLoc, diag::warn_module_conflict)
1064	<< Path[0]->getFullModuleName()
1065	<< Conflict->getFullModuleName()
1066	<< Message;
1067	});
1068
1069	// Add this module to the imports list of the currently-built submodule.
1070	if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1071	BuildingSubmoduleStack.back().M->Imports.insert(M);
1072	}
1073
1074	bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1075	const char *DiagnosticTag,
1076	bool AllowMacroExpansion) {
1077	// We need at least one string literal.
1078	if (Result.isNot(tok::string_literal)) {
1079	Diag(Result, diag::err_expected_string_literal)
1080	<< /Source='in...'/0 << DiagnosticTag;
1081	return false;
1082	}
1083
1084	// Lex string literal tokens, optionally with macro expansion.
1085	SmallVector<Token, 4> StrToks;
1086	do {
1087	StrToks.push_back(Result);
1088
1089	if (Result.hasUDSuffix())
1090	Diag(Result, diag::err_invalid_string_udl);
1091
1092	if (AllowMacroExpansion)
1093	Lex(Result);
1094	else
1095	LexUnexpandedToken(Result);
1096	} while (Result.is(tok::string_literal));
1097
1098	// Concatenate and parse the strings.
1099	StringLiteralParser Literal(StrToks, *this);
1100	(0) . __assert_fail ("Literal.isAscii() && \"Didn't allow wide strings in\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 1100, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Literal.isAscii() && "Didn't allow wide strings in");
1101
1102	if (Literal.hadError)
1103	return false;
1104
1105	if (Literal.Pascal) {
1106	Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1107	<< /Source='in...'/0 << DiagnosticTag;
1108	return false;
1109	}
1110
1111	String = Literal.GetString();
1112	return true;
1113	}
1114
1115	bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1116	assert(Tok.is(tok::numeric_constant));
1117	SmallString<8> IntegerBuffer;
1118	bool NumberInvalid = false;
1119	StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1120	if (NumberInvalid)
1121	return false;
1122	NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1123	if (Literal.hadError \|\| !Literal.isIntegerLiteral() \|\| Literal.hasUDSuffix())
1124	return false;
1125	llvm::APInt APVal(64, 0);
1126	if (Literal.GetIntegerValue(APVal))
1127	return false;
1128	Lex(Tok);
1129	Value = APVal.getLimitedValue();
1130	return true;
1131	}
1132
1133	void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1134	(0) . __assert_fail ("Handler && \"NULL comment handler\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 1134, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Handler && "NULL comment handler");
1135	(0) . __assert_fail ("std..find(CommentHandlers.begin(), CommentHandlers.end(), Handler) == CommentHandlers.end() && \"Comment handler already registered\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 1136, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
1136	(0) . __assert_fail ("std..find(CommentHandlers.begin(), CommentHandlers.end(), Handler) == CommentHandlers.end() && \"Comment handler already registered\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 1136, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> CommentHandlers.end() && "Comment handler already registered");
1137	CommentHandlers.push_back(Handler);
1138	}
1139
1140	void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1141	std::vector<CommentHandler *>::iterator Pos =
1142	std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
1143	(0) . __assert_fail ("Pos != CommentHandlers.end() && \"Comment handler not registered\"", "/home/seafit/code_projects/clang_source/clang/lib/Lex/Preprocessor.cpp", 1143, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1144	CommentHandlers.erase(Pos);
1145	}
1146
1147	bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1148	bool AnyPendingTokens = false;
1149	for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1150	HEnd = CommentHandlers.end();
1151	H != HEnd; ++H) {
1152	if ((H)->HandleComment(this, Comment))
1153	AnyPendingTokens = true;
1154	}
1155	if (!AnyPendingTokens \|\| getCommentRetentionState())
1156	return false;
1157	Lex(result);
1158	return true;
1159	}
1160
1161	ModuleLoader::~ModuleLoader() = default;
1162
1163	CommentHandler::~CommentHandler() = default;
1164
1165	CodeCompletionHandler::~CodeCompletionHandler() = default;
1166
1167	void Preprocessor::createPreprocessingRecord() {
1168	if (Record)
1169	return;
1170
1171	Record = new PreprocessingRecord(getSourceManager());
1172	addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1173	}
1174