Cold-Air-Networks/mirror-llvm-project
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 7

[flang] Don't perform macro replacement on exponents (#136176)

Browse source 
See new test. I inadvertently broke this behavior with a recent fix for
another problem, because the effects of the overloaded
TokenSequence::Put() member function on token merging were confusing.
Rename and document the various overloads.
This commit is contained in:
Peter Klausler 2025-04-18 12:52:04 -07:00 committed by GitHub
parent 544940846d
commit aac53ad4d6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 55 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
flang/include/flang/Parser/token-sequence.h
View file

@ -35,10 +35,10 @@ class Prescanner;
class TokenSequence {
public:
TokenSequence() {}
TokenSequence(const TokenSequence &that) { Put(that); }
TokenSequence(const TokenSequence &that) { CopyAll(that); }
TokenSequence(
const TokenSequence &that, std::size_t at, std::size_t count = 1) {
Put(that, at, count);
AppendRange(that, at, count);
}
TokenSequence(TokenSequence &&that)
: start_{std::move(that.start_)}, nextStart_{that.nextStart_},
@ -48,7 +48,7 @@ public:
TokenSequence &operator=(const TokenSequence &that) {
clear();
Put(that);
CopyAll(that);
return *this;
}
TokenSequence &operator=(TokenSequence &&that);
@ -100,14 +100,26 @@ public:
start_.pop_back();
}
void Put(const TokenSequence &);
void Put(const TokenSequence &, ProvenanceRange);
void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
// These append characters with provenance and then close the token.
// When the last token of this sequence remains open beforehand,
// the new characters are appended to it.
void Put(const char *, std::size_t, Provenance);
void Put(const CharBlock &, Provenance);
void Put(const std::string &, Provenance);
void Put(llvm::raw_string_ostream &, Provenance);
// Appends a full copy of another sequence. When the last token of this
// sequence remains open beforehand, it is closed before the new text
// is appended.
void CopyAll(const TokenSequence &);
// Copies a range of tokens from another sequence. If the last token of
// this sequence remains open, the first token of the copied range will be
// appended to it.
void AppendRange(
const TokenSequence &, std::size_t at, std::size_t tokens = 1);
// Copies tokens (via Put above) with new provenance.
void CopyWithProvenance(const TokenSequence &, ProvenanceRange);
Provenance GetCharProvenance(std::size_t) const;
Provenance GetTokenProvenance(
std::size_t token, std::size_t offset = 0) const;

38
flang/lib/Parser/preprocessor.cpp
View file

@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
continue;
}
}
result.Put(token, firstToken + j, 1);
result.AppendRange(token, firstToken + j, 1);
}
return result;
}
@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
}
} else if (pasting && text.TokenAt(j).IsBlank()) {
} else {
result.Put(text, j, 1);
result.AppendRange(text, j, 1);
pasting = false;
}
}
@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
CHECK(resultSize > 0 &&
result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
result.pop_back();
result.Put(Stringify(args[index], prescanner.allSources()));
result.CopyAll(Stringify(args[index], prescanner.allSources()));
} else {
const TokenSequence *arg{&args[index]};
std::optional<TokenSequence> replaced;
@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
}
}
result.Put(DEREF(arg));
result.CopyAll(DEREF(arg));
}
} else if (bytes == 11 && isVariadic_ &&
token.ToString() == "__VA_ARGS__") {
@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
if (k > argumentCount()) {
result.Put(","s, commaProvenance);
}
result.Put(args[k]);
result.CopyAll(args[k]);
}
} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
}
}
result.Put(replacement_, j);
result.AppendRange(replacement_, j);
}
}
return TokenPasting(std::move(result));
@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
inIfExpression](std::size_t after, const TokenSequence &replacement,
std::size_t pFLMOffset) {
if (after < input.SizeInTokens()) {
result.Put(replacement, 0, pFLMOffset);
result.AppendRange(replacement, 0, pFLMOffset);
TokenSequence suffix;
suffix.Put(
suffix.AppendRange(
replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
suffix.Put(input, after, input.SizeInTokens() - after);
suffix.AppendRange(input, after, input.SizeInTokens() - after);
auto further{ReplaceMacros(
suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
// still not closed
**partialFunctionLikeMacro += result.SizeInTokens();
}
result.Put(further);
result.CopyAll(further);
return true;
} else {
if (partialFunctionLikeMacro) {
@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
for (; j < tokens; ++j) {
CharBlock token{input.TokenAt(j)};
if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
result.Put(input, j);
result.AppendRange(input, j);
continue;
}
// Process identifier in replacement text.
@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
}
if (it == definitions_.end()) {
result.Put(input, j);
result.AppendRange(input, j);
continue;
}
Definition *def{&it->second};
if (def->isDisabled()) {
result.Put(input, j);
result.AppendRange(input, j);
continue;
}
if (!def->isFunctionLike()) {
@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
ProvenanceRange use{input.GetTokenProvenanceRange(j)};
ProvenanceRange newRange{
allSources_.AddMacroCall(from, use, replaced.ToString())};
result.Put(replaced, newRange);
result.CopyWithProvenance(replaced, newRange);
}
} else {
// Possible function-like macro call. Skip spaces and newlines to see
@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
if (!leftParen) {
if (partialFunctionLikeMacro) {
*partialFunctionLikeMacro = result.SizeInTokens();
result.Put(input, j, tokens - j);
result.AppendRange(input, j, tokens - j);
return result;
} else {
result.Put(input, j);
result.AppendRange(input, j);
continue;
}
}
@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
if (k >= tokens && partialFunctionLikeMacro) {
*partialFunctionLikeMacro = result.SizeInTokens();
result.Put(input, j, tokens - j);
result.AppendRange(input, j, tokens - j);
return result;
} else if (k >= tokens || argStart.size() < def->argumentCount() ||
(argStart.size() > def->argumentCount() && !def->isVariadic())) {
result.Put(input, j);
result.AppendRange(input, j);
continue;
}
std::vector<TokenSequence> args;
@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
ProvenanceRange newRange{
allSources_.AddMacroCall(from, use, replaced.ToString())};
result.Put(replaced, newRange);
result.CopyWithProvenance(replaced, newRange);
}
j = k; // advance to the terminal ')'
}

13
flang/lib/Parser/prescan.cpp
View file

@ -903,8 +903,7 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
EmitCharAndAdvance(possible, *at_);
}
possible.CloseToken();
tokens.CloseToken();
tokens.Put(possible);
tokens.AppendRange(possible, 0); // appends to current token
return true;
}
// Not an exponent; backtrack
@ -937,9 +936,9 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
!preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
// "_foo" is not defined, but "foo" is
tokens.Put(separate); // '_' "foo"
tokens.CopyAll(separate); // '_' "foo"
} else {
tokens.Put(withUnderscore); // "_foo"
tokens.CopyAll(withUnderscore); // "_foo"
}
return true;
}
@ -1016,7 +1015,7 @@ void Prescanner::QuotedCharacterLiteral(
ppTokens.Put(id, GetProvenance(idStart));
if (auto replaced{
preprocessor_.MacroReplacement(ppTokens, *this)}) {
tokens.Put(*replaced);
tokens.CopyAll(*replaced);
at_ = &idStart[idLen - 1];
NextLine();
continue; // try again on the next line
@ -1836,7 +1835,7 @@ bool Prescanner::CompilerDirectiveContinuation(
}
if (ok) {
tokens.pop_back(); // delete original '&'
tokens.Put(followingTokens, startAt, following - startAt);
tokens.AppendRange(followingTokens, startAt, following - startAt);
tokens.RemoveRedundantBlanks();
} else {
nextLine_ = origNextLine;
@ -1866,7 +1865,7 @@ bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
}
followingTokens.RemoveRedundantBlanks();
tokens.pop_back(); // delete original '&'
tokens.Put(followingTokens);
tokens.CopyAll(followingTokens);
return true;
}
}

13
flang/lib/Parser/token-sequence.cpp
View file

@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {
return false;
}
void TokenSequence::Put(const TokenSequence &that) {
void TokenSequence::CopyAll(const TokenSequence &that) {
if (nextStart_ < char_.size()) {
start_.push_back(nextStart_);
}
@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {
provenances_.Put(that.provenances_);
}
void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
void TokenSequence::CopyWithProvenance(
const TokenSequence &that, ProvenanceRange range) {
std::size_t offset{0};
std::size_t tokens{that.SizeInTokens()};
for (std::size_t j{0}; j < tokens; ++j) {
@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
CHECK(offset == range.size());
}
void TokenSequence::Put(
void TokenSequence::AppendRange(
const TokenSequence &that, std::size_t at, std::size_t tokens) {
ProvenanceRange provenance;
std::size_t offset{0};
@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {
TokenSequence result;
for (std::size_t j{0}; j < tokens; ++j) {
if (!TokenAt(j).IsBlank() || start_[j] < firstChar) {
result.Put(*this, j);
result.AppendRange(*this, j);
}
}
swap(result);
@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
for (std::size_t j{0}; j < tokens; ++j) {
bool isBlank{TokenAt(j).IsBlank()};
if (!isBlank || !lastWasBlank || start_[j] < firstChar) {
result.Put(*this, j);
result.AppendRange(*this, j);
}
lastWasBlank = isBlank;
}
@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(
} else {
TokenSequence result;
if (j > 0) {
result.Put(*this, 0, j - 1);
result.AppendRange(*this, 0, j - 1);
}
swap(result);
return *this;

5
flang/test/Preprocessing/exponent-bug.F90 Normal file
View file

@ -0,0 +1,5 @@
! RUN: %flang -E %s 2>&1 | FileCheck %s
! CHECK: 3.14159e00
#define e00 e666
print *, 3.14159e00
end