[Clang] Support libstdc++ workarounds when using -E (#210802)

We have a libstdc++ workaround in place to address a hack in libstdc++15
that is used in the definition of `std::format_kind`. GCC accepts the
hack, while Clang does not. libstdc++ was eventually updated to remove
the hack, but we still need the workaround for some versions of
libstdc++15 (see #139560 for more information).

Whether this workaround (and others that address libstdc++ hacks) is
active depends on the value of `__GLIBCXX__`. This stops working if
someone first preprocesses the input (via `-E`) and then attempts to
compile the preprocessed code with Clang (see #160314): since
preprocessing has already happened, `__GLIBCXX__` will be undefined and
the workaround will not be applied.

In case of this particular workaround, this results in `#include
<format>` failing to compile on some versions of libstdc++ if you’re
using separate preprocessing.

To resolve this problem, this patch introduces a new pragma (`#pragma
clang __set_pp_state MACRO_NAME INTEGER`). This is intended as a general
solution to preserving preprocessor state across runs of the
preprocessor.

Currently, the only supported value for `MACRO_NAME`is `__GLIBCXX__`.
When we encounter this form of the pragma, we update
`Preprocessor::CXXStandardLibraryVersion` (we don’t actually define
`__GLIBCXX__` or do anything w/ that macro here). In `-E` mode, the
pragma is retained in the output, and a pragma is emitted for `#define
__GLIBCXX__` if printing macro definitions is disabled.

Things like `#undef __GLIBCXX__` and redefinitions of `__GLIBCXX__` are
deliberately left unhandled (i.e. you get what you get), since users
shouldn’t ever be undefining or redefining this macro anyway (and since
it starts w/ `__`, it would also be UB to do so).

Fixes #160314.

clang/docs/ReleaseNotes.md

@@ -411,6 +411,10 @@ features cannot lower the translation-unit ABI level;
   operator required an access check that ran while an enclosing declaration
   was still being parsed. (#GH210692)
 
+- A workaround that was introduced to fix an issue with the `<format>` header present in some versions of
+  libstdc++15 has been extended to support preprocessed input. Previously, splitting the preprocessing and
+  compilation step would result in the fix not being applied. (#GH160314)
+
 #### Bug Fixes to AST Handling
 
 - Fixed a non-deterministic ordering of unused local typedefs that made

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -866,6 +866,13 @@ def err_pp_include_in_arc_cf_code_audited : Error<
 def err_pp_eof_in_arc_cf_code_audited : Error<
   "'#pragma clang arc_cf_code_audited' was not ended within this file">;
 
+def err_pp_pragma_set_pp_state_expected_name : Error<
+  "expected identifier after '#pragma clang __set_pp_state'">;
+def err_pp_pragma_set_pp_state_expected_int_after : Error<
+  "expected integer after '#pragma clang __set_pp_state %0'">;
+def err_pp_pragma_set_pp_state_invalid_arg : Error<
+  "invalid argument %0 in '#pragma clang __set_pp_state'">;
+
 def warn_pp_date_time : Warning<
   "expansion of date or time macro is not reproducible">,
   ShowInSystemHeader, DefaultIgnore, InGroup<DiagGroup<"date-time">>;

clang/include/clang/Lex/PPCallbacks.h

@@ -342,6 +342,10 @@ public:
   /// is read.
   virtual void PragmaAssumeNonNullEnd(SourceLocation Loc) {}
 
+  /// Callback invoked when a \#pragma clang __set_pp_state directive is read.
+  virtual void PragmaSetPPState(SourceLocation Loc, IdentifierInfo *MacroName,
+                                std::uint64_t Value) {}
+
   /// Called by Preprocessor::HandleMacroExpandedIdentifier when a
   /// macro invocation is found.
   virtual void MacroExpands(const Token &MacroNameTok,
@@ -700,6 +704,12 @@ public:
     Second->PragmaAssumeNonNullEnd(Loc);
   }
 
+  void PragmaSetPPState(SourceLocation Loc, IdentifierInfo *MacroName,
+                        std::uint64_t Value) override {
+    First->PragmaSetPPState(Loc, MacroName, Value);
+    Second->PragmaSetPPState(Loc, MacroName, Value);
+  }
+
   void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
                     SourceRange Range, const MacroArgs *Args) override {
     First->MacroExpands(MacroNameTok, MD, Range, Args);

clang/include/clang/Lex/Preprocessor.h

@@ -2827,6 +2827,7 @@ private:
 
 public:
   std::optional<std::uint64_t> getStdLibCxxVersion();
+  void setStdLibCxxVersion(std::uint64_t Version);
   bool NeedsStdLibCxxWorkaroundBefore(std::uint64_t FixedVersion);
 
 private:
@@ -2987,6 +2988,9 @@ private:
   // Pragmas.
   void HandlePragmaDirective(PragmaIntroducer Introducer);
 
+  // Cached identifiers used to implement __set_pp_state.
+  IdentifierInfo *Ident__GLIBCXX__;
+
 public:
   void HandlePragmaOnce(Token &OnceTok);
   void HandlePragmaMark(Token &MarkTok);
@@ -2998,8 +3002,13 @@ public:
   void HandlePragmaIncludeAlias(Token &Tok);
   void HandlePragmaModuleBuild(Token &Tok);
   void HandlePragmaHdrstop(Token &Tok);
+  void HandlePragmaSetPPState(PragmaIntroducer Introducer, Token &Tok);
   IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
 
+  /// Check whether this is a macro name that can be used as an argument to
+  /// '#pragma clang __set_pp_state'.
+  bool isPragmaSetPPStateMacro(IdentifierInfo *II);
+
   // Return true and store the first token only if any CommentHandler
   // has inserted some tokens and getCommentRetentionState() is false.
   bool HandleComment(Token &result, SourceRange Comment);

clang/lib/Frontend/PrintPreprocessedOutput.cpp

@@ -29,10 +29,12 @@
 using namespace clang;
 
 /// PrintMacroDefinition - Print a macro definition in a form that will be
-/// properly accepted back as a definition.
-static void PrintMacroDefinition(const IdentifierInfo &II, const MacroInfo &MI,
+/// properly accepted back as a definition. If 'II' is nullptr, only the
+/// expansion will be printed.
+static void PrintMacroDefinition(const IdentifierInfo *II, const MacroInfo &MI,
                                  Preprocessor &PP, raw_ostream *OS) {
-  *OS << "#define " << II.getName();
+  if (II)
+    *OS << "#define " << II->getName();
 
   if (MI.isFunctionLike()) {
     *OS << '(';
@@ -182,6 +184,8 @@ public:
   void PragmaExecCharsetPop(SourceLocation Loc) override;
   void PragmaAssumeNonNullBegin(SourceLocation Loc) override;
   void PragmaAssumeNonNullEnd(SourceLocation Loc) override;
+  void PragmaSetPPState(SourceLocation Loc, IdentifierInfo *MacroName,
+                        std::uint64_t Value) override;
 
   /// Insert whitespace before emitting the next token.
   ///
@@ -559,22 +563,38 @@ void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, StringRef S) {
 /// MacroDefined - This hook is called whenever a macro definition is seen.
 void PrintPPOutputPPCallbacks::MacroDefined(const Token &MacroNameTok,
                                             const MacroDirective *MD) {
+  bool ShouldEmitDefine = true;
   const MacroInfo *MI = MD->getMacroInfo();
+  SourceLocation DefLoc = MI->getDefinitionLoc();
+
   // Print out macro definitions in -dD mode and when we have -fdirectives-only
   // for C++20 header units.
   if ((!DumpDefines && !DirectivesOnly) ||
       // Ignore __FILE__ etc.
-      MI->isBuiltinMacro())
-    return;
-
-  SourceLocation DefLoc = MI->getDefinitionLoc();
-  if (DirectivesOnly && !MI->isUsed()) {
+      MI->isBuiltinMacro()) {
+    ShouldEmitDefine = false;
+  } else if (DirectivesOnly && !MI->isUsed()) {
     SourceManager &SM = PP.getSourceManager();
     if (SM.isInPredefinedFile(DefLoc))
-      return;
+      ShouldEmitDefine = false;
   }
+
+  IdentifierInfo *MacroName = MacroNameTok.getIdentifierInfo();
+  if (!ShouldEmitDefine) {
+    // Preserve macro definitions of macros that can be used with
+    // '#pragma clang __set_pp_state' as pragmas if printing '#define's
+    // is disabled.
+    if (PP.isPragmaSetPPStateMacro(MacroName)) {
+      MoveToLine(DefLoc, /*RequireStartOfLine=*/true);
+      *OS << "#pragma clang __set_pp_state " << MacroName->getName();
+      PrintMacroDefinition(/*II=*/nullptr, *MI, PP, OS);
+      setEmittedDirectiveOnThisLine();
+    }
+    return;
+  }
+
   MoveToLine(DefLoc, /*RequireStartOfLine=*/true);
-  PrintMacroDefinition(*MacroNameTok.getIdentifierInfo(), *MI, PP, OS);
+  PrintMacroDefinition(MacroName, *MI, PP, OS);
   setEmittedDirectiveOnThisLine();
 }
 
@@ -752,6 +772,15 @@ PragmaAssumeNonNullEnd(SourceLocation Loc) {
   setEmittedDirectiveOnThisLine();
 }
 
+void PrintPPOutputPPCallbacks::PragmaSetPPState(SourceLocation Loc,
+                                                IdentifierInfo *MacroName,
+                                                std::uint64_t Value) {
+  MoveToLine(Loc, /*RequireStartOfLine=*/true);
+  *OS << "#pragma clang __set_pp_state " << MacroName->getName() << " "
+      << Value;
+  setEmittedDirectiveOnThisLine();
+}
+
 void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
                                                          bool RequireSpace,
                                                          bool RequireSameLine) {
@@ -1093,7 +1122,7 @@ static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {
     // Ignore computed macros like __LINE__ and friends.
     if (MI.isBuiltinMacro()) continue;
 
-    PrintMacroDefinition(*MacrosByID[i].first, MI, PP, OS);
+    PrintMacroDefinition(MacrosByID[i].first, MI, PP, OS);
     *OS << '\n';
   }
 }

clang/lib/Lex/PPExpressions.cpp

@@ -983,9 +983,9 @@ Preprocessor::EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
 }
 
 static std::optional<CXXStandardLibraryVersionInfo>
-getCXXStandardLibraryVersion(Preprocessor &PP, StringRef MacroName,
+getCXXStandardLibraryVersion(Preprocessor &PP, IdentifierInfo *MacroName,
                              CXXStandardLibraryVersionInfo::Library Lib) {
-  MacroInfo *Macro = PP.getMacroInfo(PP.getIdentifierInfo(MacroName));
+  MacroInfo *Macro = PP.getMacroInfo(MacroName);
   if (!Macro || Macro->getNumTokens() != 1 || !Macro->isObjectLike())
     return std::nullopt;
 
@@ -1008,7 +1008,7 @@ getCXXStandardLibraryVersion(Preprocessor &PP, StringRef MacroName,
 std::optional<uint64_t> Preprocessor::getStdLibCxxVersion() {
   if (!CXXStandardLibraryVersion)
     CXXStandardLibraryVersion = getCXXStandardLibraryVersion(
-        *this, "__GLIBCXX__", CXXStandardLibraryVersionInfo::LibStdCXX);
+        *this, Ident__GLIBCXX__, CXXStandardLibraryVersionInfo::LibStdCXX);
   if (!CXXStandardLibraryVersion)
     return std::nullopt;
 
@@ -1018,6 +1018,13 @@ std::optional<uint64_t> Preprocessor::getStdLibCxxVersion() {
   return std::nullopt;
 }
 
+void Preprocessor::setStdLibCxxVersion(std::uint64_t Version) {
+  CXXStandardLibraryVersion = {
+      CXXStandardLibraryVersionInfo::LibStdCXX,
+      Version,
+  };
+}
+
 bool Preprocessor::NeedsStdLibCxxWorkaroundBefore(uint64_t FixedVersion) {
   assert(FixedVersion >= 2000'00'00 && FixedVersion <= 2100'00'00 &&
          "invalid value for __GLIBCXX__");

clang/lib/Lex/Pragma.cpp

@@ -913,6 +913,47 @@ void Preprocessor::HandlePragmaHdrstop(Token &Tok) {
     SkippingUntilPragmaHdrStop = false;
 }
 
+bool Preprocessor::isPragmaSetPPStateMacro(IdentifierInfo *MacroName) {
+  return MacroName == Ident__GLIBCXX__;
+}
+
+void Preprocessor::HandlePragmaSetPPState(PragmaIntroducer Introducer,
+                                          Token &Tok) {
+  // Lex the macro name we want to set.
+  LexUnexpandedToken(Tok);
+  if (!Tok.getIdentifierInfo()) {
+    Diag(Tok.getLocation(), diag::err_pp_pragma_set_pp_state_expected_name);
+    return;
+  }
+
+  IdentifierInfo *MacroName = Tok.getIdentifierInfo();
+  if (!isPragmaSetPPStateMacro(MacroName)) {
+    Diag(Tok.getLocation(), diag::err_pp_pragma_set_pp_state_invalid_arg)
+        << MacroName;
+    return;
+  }
+
+  // Lex the integer argument.
+  Lex(Tok);
+  std::uint64_t Value;
+  if (!Tok.is(tok::numeric_constant) ||
+      !parseSimpleIntegerLiteral(Tok, Value)) {
+    Diag(Tok.getLocation(), diag::err_pp_pragma_set_pp_state_expected_int_after)
+        // Don't pass an IdentifierInfo* here to avoid quoting.
+        << MacroName->getName();
+    return;
+  }
+
+  // Update the state.
+  if (MacroName->getName() == "__GLIBCXX__")
+    setStdLibCxxVersion(Value);
+  else
+    llvm_unreachable("forgot to handle a possible argument to __set_pp_state");
+
+  if (Callbacks)
+    Callbacks->PragmaSetPPState(Introducer.Loc, MacroName, Value);
+}
+
 /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
 /// If 'Namespace' is non-null, then it is a token required to exist on the
 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
@@ -2148,6 +2189,25 @@ struct PragmaFinalHandler : public PragmaHandler {
   }
 };
 
+/// "\#pragma clang __set_pp_state ..."
+///
+/// This pragma takes an identifier+value pair and sets some internal state in
+/// the compiler; it is intended primarily to preserve preprocessor state that
+/// is required for compilation to function properly across preprocessor runs
+/// if '-E' is used. This is an internal pragma that should not be used by
+/// users.
+///
+/// The syntax is
+/// \code
+///   #pragma clang __set_pp_state glibcxx_version INTEGER
+/// \endcode
+struct PragmaSetPPStateHandler : PragmaHandler {
+  PragmaSetPPStateHandler() : PragmaHandler("__set_pp_state") {}
+  void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
+                    Token &Tok) override {
+    PP.HandlePragmaSetPPState(Introducer, Tok);
+  }
+};
 } // namespace
 
 /// RegisterBuiltinPragmas - Install the standard preprocessor pragmas:
@@ -2179,6 +2239,7 @@ void Preprocessor::RegisterBuiltinPragmas() {
   AddPragmaHandler("clang", new PragmaDeprecatedHandler());
   AddPragmaHandler("clang", new PragmaRestrictExpansionHandler());
   AddPragmaHandler("clang", new PragmaFinalHandler());
+  AddPragmaHandler("clang", new PragmaSetPPStateHandler());
 
   // #pragma clang module ...
   auto *ModuleHandler = new PragmaNamespace("module");

clang/lib/Lex/Preprocessor.cpp

@@ -159,6 +159,8 @@ Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts,
     Ident_AbnormalTermination = nullptr;
   }
 
+  Ident__GLIBCXX__ = getIdentifierInfo("__GLIBCXX__");
+
   // Default incremental processing to -fincremental-extensions, clients can
   // override with `enableIncrementalProcessing` if desired.
   IncrementalProcessing = LangOpts.IncrementalExtensions;

clang/test/Preprocessor/pragma_set_pp_state.cpp

@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -E -verify %s
+
+#pragma clang __set_pp_state // expected-error {{expected identifier after '#pragma clang __set_pp_state'}}
+#pragma clang __set_pp_state 123984 // expected-error {{expected identifier after '#pragma clang __set_pp_state'}}
+
+#pragma clang __set_pp_state foo // expected-error {{invalid argument 'foo' in '#pragma clang __set_pp_state'}}
+#pragma clang __set_pp_state void // expected-error {{invalid argument 'void' in '#pragma clang __set_pp_state'}}
+
+#pragma clang __set_pp_state __GLIBCXX__ foo // expected-error {{expected integer after '#pragma clang __set_pp_state __GLIBCXX__'}}
+#pragma clang __set_pp_state __GLIBCXX__ 100000000000000000000000000000 // expected-error {{expected integer after '#pragma clang __set_pp_state __GLIBCXX__'}}
+#pragma clang __set_pp_state __GLIBCXX__ 42.0 // expected-error {{expected integer after '#pragma clang __set_pp_state __GLIBCXX__'}}
+
+#pragma clang __set_pp_state __GLIBCXX__ 42L
+#pragma clang __set_pp_state __GLIBCXX__ 42
+
+// Check that we treat the identifier after '__set_pp_state' literally.
+#define MACRO __GLIBCXX__
+#pragma clang __set_pp_state MACRO // expected-error {{invalid argument 'MACRO' in '#pragma clang __set_pp_state'}}
+
+// Check that we treat '__set_pp_state' literally.
+#define __set_pp_state foobar
+#pragma clang __set_pp_state __GLIBCXX__ 42
+
+// The pragma does *not* define __GLIBCXX__!
+#ifdef __GLIBCXX__
+#   error __set_pp_state __GLIBCXX__ should not define __GLIBCXX__
+#endif

clang/test/SemaCXX/libstdcxx_format_kind_hack.cpp

@@ -1,12 +1,46 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify %s
+// Check that we accept the program if '__GLIBCXX__' is defined:
+// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify %s -DDEFINE_GLIBCXX
+
+// Check that we preserve the value of __GLIBCXX__ via a pragma when preprocessing:
+// RUN: %clang_cc1 -E -std=c++23 %s -o %t.ii -DDEFINE_GLIBCXX
+// RUN: FileCheck --input-file=%t.ii %s
+
+// Check that the preprocessed file compiles with no diagnostics:
+// RUN: echo '// expected-no-diagnostics' >> %t.ii
+// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify %t.ii
+
+// Check that we accept the program if the pragma is present:
+// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify %s -DUSE_PRAGMA
+
+// Check that we preserve the pragma when preprocessing:
+// RUN: %clang_cc1 -E -std=c++23 %s -o %t.ii -DUSE_PRAGMA
+// RUN: FileCheck --input-file=%t.ii %s
+
+// Check that the preprocessed file compiles with no diagnostics:
+// RUN: echo '// expected-no-diagnostics' >> %t.ii
+// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify %t.ii
+
+// Irrespective of whether we used the pragma directly or defined __GLIBCXX__,
+// the preprocessed output should contain the pragma:
+// CHECK: #pragma clang __set_pp_state __GLIBCXX__ 20250513
 
 // expected-no-diagnostics
 
 // Primary variable template std::format_kind is defined as followed since
 // libstdc++ 15.1, which triggers compilation error introduced by GH134522.
 // This file tests the workaround.
+//
+// Since the workaround relies on '__GLIBCXX__' being defined, we emit a pragma
+// that ensures '__GLIBCXX__' is defined if the user first preprocesses the file
+// with '-E' before passing the output of that back to Clang.
 
-#define __GLIBCXX__ 20250513
+#ifdef DEFINE_GLIBCXX
+#   define __GLIBCXX__ 20250513
+#endif
+
+#ifdef USE_PRAGMA
+#   pragma clang __set_pp_state __GLIBCXX__ 20250513
+#endif
 
 namespace std {
   template<typename _Rg>