[clang][modules-driver] Generate jobs from Standard library module manifest entries (#182182)

This patch is part of a series to support driver-managed module builds.

To support imports of the Standard library modules (std and
std.compat), the driver must generate frontend jobs for each module
before performing the dependency scan.
The source paths for these modules, along with additional information
required to precompile them, are provided by the Standard library
modules manifest.

This change implements the parsing and handling of the manifest in the
modules-driver.
This change is part of an effort to split #152770 into smaller, more
manageable pieces.

RFC for driver-managed module builds:

https://discourse.llvm.org/t/rfc-modules-support-simple-c-20-modules-use-from-the-clang-driver-without-a-build-system
This commit is contained in:
Naveen Seth Hanig 2026-02-25 22:00:16 +01:00 committed by GitHub
parent 7be9d66c29
commit 84764dfc2a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 495 additions and 0 deletions

View file

@ -611,6 +611,18 @@ def err_drv_reduced_module_output_overrided : Warning<
"please consider use '-fmodule-output=' to specify the output file for reduced BMI explicitly">,
InGroup<DiagGroup<"reduced-bmi-output-overrided">>;
def remark_performing_driver_managed_module_build : Remark<
"performing driver managed module build">, InGroup<ModulesDriver>;
def remark_modules_manifest_not_found : Remark<
"standard modules manifest file not found; import of standard library "
"modules not supported">,
InGroup<ModulesDriver>;
def remark_using_modules_manifest : Remark<
"using standard modules manifest file '%0'">,
InGroup<ModulesDriver>;
def err_modules_manifest_failed_parse : Error<
"failure while parsing standard modules manifest: '%0'">;
def warn_drv_delayed_template_parsing_after_cxx20 : Warning<
"-fdelayed-template-parsing is deprecated after C++20">,
InGroup<DiagGroup<"delayed-template-parsing-in-cxx20">>;

View file

@ -754,6 +754,7 @@ def ModuleConflict : DiagGroup<"module-conflict">;
def ModuleFileExtension : DiagGroup<"module-file-extension">;
def ModuleIncludeDirectiveTranslation : DiagGroup<"module-include-translation">;
def ModuleMap : DiagGroup<"module-map">;
def ModulesDriver : DiagGroup<"modules-driver">;
def RoundTripCC1Args : DiagGroup<"round-trip-cc1-args">;
def NewlineEOF : DiagGroup<"newline-eof">;
def Nullability : DiagGroup<"nullability">;

View file

@ -279,6 +279,9 @@ public:
const list_type &getJobs() const { return Jobs; }
// Returns and transfers ownership of all jobs, leaving this list empty.
list_type takeJobs() { return std::exchange(Jobs, {}); };
bool empty() const { return Jobs.empty(); }
size_type size() const { return Jobs.size(); }
iterator begin() { return Jobs.begin(); }

View file

@ -0,0 +1,81 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file defines functionality to support driver managed builds for
/// compilations which use Clang modules or standard C++20 named modules.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_DRIVER_MODULESDRIVER_H
#define LLVM_CLANG_DRIVER_MODULESDRIVER_H
#include "clang/Basic/LLVM.h"
#include "clang/Driver/Types.h"
#include "llvm/Support/Error.h"
namespace llvm::vfs {
class FileSystem;
} // namespace llvm::vfs
namespace clang {
class DiagnosticsEngine;
namespace driver {
class Compilation;
} // namespace driver
} // namespace clang
namespace clang::driver::modules {
/// The parsed Standard library module manifest.
struct StdModuleManifest {
struct Module {
struct LocalArguments {
std::vector<std::string> SystemIncludeDirs;
};
bool IsStdlib = false;
std::string LogicalName;
std::string SourcePath;
std::optional<LocalArguments> LocalArgs;
};
std::vector<Module> Modules;
};
/// Reads the Standard library module manifest at \p ManifestPath.
///
/// Assumes that all file paths specified in the manifest are relative to
/// \p ManifestPath and converts them to absolute.
///
/// \returns The parsed manifest on success; otherwise, a \c llvm::FileError
/// or \c llvm::json::ParseError.
llvm::Expected<StdModuleManifest>
readStdModuleManifest(llvm::StringRef ManifestPath, llvm::vfs::FileSystem &VFS);
/// Constructs compilation inputs for each module listed in the provided
/// Standard library module manifest.
///
/// \param ManifestEntries All entries of the Standard library module manifest.
/// \param C The Compilation being built.
/// \param Inputs The input list to which the new module inputs are appended.
void buildStdModuleManifestInputs(
ArrayRef<StdModuleManifest::Module> ManifestEntries, Compilation &C,
InputList &Inputs);
/// Scans the compilation inputs for module dependencies and adjusts the
/// compilation to build and supply those modules as required.
///
/// \param C The Compilation being built.
/// \param ManifestEntries All entries of the Standard library module manifest.
void runModulesDriver(Compilation &C,
ArrayRef<StdModuleManifest::Module> ManifestEntries);
} // namespace clang::driver::modules
#endif // LLVM_CLANG_DRIVER_MODULESDRIVER_H

View file

@ -3582,6 +3582,17 @@ def fmodules_single_module_parse_mode :
Group<f_Group>, Flags<[]>, Visibility<[CC1Option]>,
MarshallingInfoFlag<PreprocessorOpts<"SingleModuleParseMode">>;
def fmodules_driver :
Flag<["-"], "fmodules-driver">,
Group<f_Group>,
Visibility<[ClangOption]>,
HelpText<"Enable support for driver managed module builds (experimental)">;
def fno_modules_driver :
Flag<["-"], "fno-modules-driver">,
Group<f_Group>,
Visibility<[ClangOption]>,
HelpText<"Disable support for driver managed module builds (experimental)">;
def fincremental_extensions :
Flag<["-"], "fincremental-extensions">,
Group<f_Group>, Visibility<[ClangOption, CC1Option]>,

View file

@ -22,6 +22,7 @@ add_clang_library(clangDriver
Distro.cpp
Driver.cpp
Job.cpp
ModulesDriver.cpp
Multilib.cpp
MultilibBuilder.cpp
OffloadBundler.cpp

View file

@ -61,6 +61,7 @@
#include "clang/Driver/Compilation.h"
#include "clang/Driver/InputInfo.h"
#include "clang/Driver/Job.h"
#include "clang/Driver/ModulesDriver.h"
#include "clang/Driver/Phases.h"
#include "clang/Driver/SanitizerArgs.h"
#include "clang/Driver/Tool.h"
@ -90,6 +91,7 @@
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/IOSandbox.h"
#include "llvm/Support/JSON.h"
#include "llvm/Support/MD5.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/PrettyStackTrace.h"
@ -1825,6 +1827,48 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
// Populate the tool chains for the offloading devices, if any.
CreateOffloadingDeviceToolChains(*C, Inputs);
bool UseModulesDriver = C->getArgs().hasFlag(
options::OPT_fmodules_driver, options::OPT_fno_modules_driver, false);
modules::StdModuleManifest ModulesManifest;
if (UseModulesDriver) {
Diags.Report(diag::remark_performing_driver_managed_module_build);
// Read the Standard library module manifest and, if available, add all
// discovered modules to this Compilation. Jobs for modules specified in
// the manifest that are not required by any command-line input are pruned
// later.
const auto StdModuleManifestPath =
GetStdModuleManifestPath(*C, C->getDefaultToolChain());
if (!llvm::sys::fs::exists(StdModuleManifestPath))
Diags.Report(diag::remark_modules_manifest_not_found);
else {
Diags.Report(diag::remark_using_modules_manifest)
<< StdModuleManifestPath;
if (auto ManifestOrErr =
modules::readStdModuleManifest(StdModuleManifestPath, getVFS())) {
ModulesManifest = std::move(*ManifestOrErr);
// Only allow on-demand imports of standard library modules for now.
llvm::erase_if(ModulesManifest.Modules, [](const auto &ModuleEntry) {
return !ModuleEntry.IsStdlib;
});
modules::buildStdModuleManifestInputs(ModulesManifest.Modules, *C,
Inputs);
} else {
llvm::handleAllErrors(
ManifestOrErr.takeError(),
[&](llvm::json::ParseError &Err) {
Diags.Report(diag::err_modules_manifest_failed_parse)
<< Err.message();
},
[&](llvm::FileError &Err) {
Diags.Report(diag::err_cannot_open_file)
<< Err.getFileName() << Err.messageWithoutFileInfo();
});
}
}
}
// Construct the list of abstract actions to perform for this compilation. On
// MachO targets this uses the driver-driver and universal actions.
if (TC.getTriple().isOSBinFormatMachO())
@ -1839,6 +1883,9 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
BuildJobs(*C);
if (UseModulesDriver)
modules::runModulesDriver(*C, ModulesManifest.Modules);
return C;
}

View file

@ -0,0 +1,216 @@
//===--- ModulesDriver.cpp - Driver managed module builds -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file defines functionality to support driver managed builds for
/// compilations which use Clang modules or standard C++20 named modules.
///
//===----------------------------------------------------------------------===//
#include "clang/Driver/ModulesDriver.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/LLVM.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Job.h"
#include "clang/Driver/Tool.h"
#include "clang/Driver/ToolChain.h"
#include "llvm/Support/JSON.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/VirtualFileSystem.h"
using namespace llvm::opt;
using namespace clang;
using namespace driver;
using namespace modules;
namespace clang::driver::modules {
static bool fromJSON(const llvm::json::Value &Params,
StdModuleManifest::Module::LocalArguments &LocalArgs,
llvm::json::Path P) {
llvm::json::ObjectMapper O(Params, P);
return O.mapOptional("system-include-directories",
LocalArgs.SystemIncludeDirs);
}
static bool fromJSON(const llvm::json::Value &Params,
StdModuleManifest::Module &ModuleEntry,
llvm::json::Path P) {
llvm::json::ObjectMapper O(Params, P);
return O.map("is-std-library", ModuleEntry.IsStdlib) &&
O.map("logical-name", ModuleEntry.LogicalName) &&
O.map("source-path", ModuleEntry.SourcePath) &&
O.mapOptional("local-arguments", ModuleEntry.LocalArgs);
}
static bool fromJSON(const llvm::json::Value &Params,
StdModuleManifest &Manifest, llvm::json::Path P) {
llvm::json::ObjectMapper O(Params, P);
return O.map("modules", Manifest.Modules);
}
} // namespace clang::driver::modules
/// Parses the Standard library module manifest from \p Buffer.
static Expected<StdModuleManifest> parseManifest(StringRef Buffer) {
auto ParsedOrErr = llvm::json::parse(Buffer);
if (!ParsedOrErr)
return ParsedOrErr.takeError();
StdModuleManifest Manifest;
llvm::json::Path::Root Root;
if (!fromJSON(*ParsedOrErr, Manifest, Root))
return Root.getError();
return Manifest;
}
/// Converts each file path in manifest from relative to absolute.
///
/// Each file path in the manifest is expected to be relative the manifest's
/// location \p ManifestPath itself.
static void makeManifestPathsAbsolute(
MutableArrayRef<StdModuleManifest::Module> ManifestEntries,
StringRef ManifestPath) {
StringRef ManifestDir = llvm::sys::path::parent_path(ManifestPath);
SmallString<256> TempPath;
auto PrependManifestDir = [&](StringRef Path) {
TempPath = ManifestDir;
llvm::sys::path::append(TempPath, Path);
return std::string(TempPath);
};
for (auto &Entry : ManifestEntries) {
Entry.SourcePath = PrependManifestDir(Entry.SourcePath);
if (!Entry.LocalArgs)
continue;
for (auto &IncludeDir : Entry.LocalArgs->SystemIncludeDirs)
IncludeDir = PrependManifestDir(IncludeDir);
}
}
Expected<StdModuleManifest>
driver::modules::readStdModuleManifest(StringRef ManifestPath,
llvm::vfs::FileSystem &VFS) {
auto MemBufOrErr = VFS.getBufferForFile(ManifestPath);
if (!MemBufOrErr)
return llvm::createFileError(ManifestPath, MemBufOrErr.getError());
auto ManifestOrErr = parseManifest((*MemBufOrErr)->getBuffer());
if (!ManifestOrErr)
return ManifestOrErr.takeError();
auto Manifest = std::move(*ManifestOrErr);
makeManifestPathsAbsolute(Manifest.Modules, ManifestPath);
return Manifest;
}
void driver::modules::buildStdModuleManifestInputs(
ArrayRef<StdModuleManifest::Module> ManifestEntries, Compilation &C,
InputList &Inputs) {
DerivedArgList &Args = C.getArgs();
const OptTable &Opts = C.getDriver().getOpts();
for (const auto &Entry : ManifestEntries) {
auto *InputArg =
makeInputArg(Args, Opts, Args.MakeArgString(Entry.SourcePath));
Inputs.emplace_back(types::TY_CXXModule, InputArg);
}
}
using ManifestEntryLookup =
llvm::DenseMap<StringRef, const StdModuleManifest::Module *>;
/// Builds a mapping from a module's source path to its entry in the manifest.
static ManifestEntryLookup
buildManifestLookupMap(ArrayRef<StdModuleManifest::Module> ManifestEntries) {
ManifestEntryLookup ManifestEntryBySource;
for (auto &Entry : ManifestEntries) {
const bool Inserted =
ManifestEntryBySource.try_emplace(Entry.SourcePath, &Entry).second;
assert(Inserted &&
"Manifest defines multiple modules with the same source path.");
}
return ManifestEntryBySource;
}
/// Returns the manifest entry corresponding to \p Job, or \c nullptr if none
/// exists.
static const StdModuleManifest::Module *
getManifestEntryForCommand(const Command &Job,
const ManifestEntryLookup &ManifestEntryBySource) {
for (const auto &II : Job.getInputInfos()) {
if (const auto It = ManifestEntryBySource.find(II.getFilename());
It != ManifestEntryBySource.end())
return It->second;
}
return nullptr;
}
/// Adds all \p SystemIncludeDirs to the \p CC1Args of \p Job.
static void
addSystemIncludeDirsFromManifest(Compilation &C, Command &Job,
ArgStringList &CC1Args,
ArrayRef<std::string> SystemIncludeDirs) {
const ToolChain &TC = Job.getCreator().getToolChain();
const DerivedArgList &TCArgs =
C.getArgsForToolChain(&TC, Job.getSource().getOffloadingArch(),
Job.getSource().getOffloadingDeviceKind());
for (const auto &IncludeDir : SystemIncludeDirs)
TC.addSystemInclude(TCArgs, CC1Args, IncludeDir);
}
static bool isCC1Job(const Command &Job) {
return StringRef(Job.getCreator().getName()) == "clang";
}
/// Apply command-line modifications specific for inputs originating from the
/// Standard library module manifest.
static void applyArgsForStdModuleManifestInputs(
Compilation &C, const ManifestEntryLookup &ManifestEntryBySource,
MutableArrayRef<std::unique_ptr<Command>> Jobs) {
for (auto &Job : Jobs) {
if (!isCC1Job(*Job))
continue;
const auto *Entry = getManifestEntryForCommand(*Job, ManifestEntryBySource);
if (!Entry)
continue;
auto CC1Args = Job->getArguments();
if (Entry->IsStdlib)
CC1Args.push_back("-Wno-reserved-module-identifier");
if (Entry->LocalArgs)
addSystemIncludeDirsFromManifest(C, *Job, CC1Args,
Entry->LocalArgs->SystemIncludeDirs);
Job->replaceArguments(CC1Args);
}
}
void driver::modules::runModulesDriver(
Compilation &C, ArrayRef<StdModuleManifest::Module> ManifestEntries) {
llvm::PrettyStackTraceString CrashInfo("Running modules driver.");
auto Jobs = C.getJobs().takeJobs();
const auto ManifestEntryBySource = buildManifestLookupMap(ManifestEntries);
// Apply manifest-entry specific command-line modifications before the scan as
// they might affect it.
applyArgsForStdModuleManifestInputs(C, ManifestEntryBySource, Jobs);
// TODO: Run the dependency scan.
// TODO: Prune jobs for modules specified in the manifest that are not
// required by any command-line input.
// TODO: Reorder and modify jobs based on the discovered dependencies.
// Add jobs back to the Compilation.
for (auto &Job : Jobs)
C.addCommand(std::move(Job));
}

View file

@ -0,0 +1,41 @@
// Verify that a malformed standard modules manifest triggers an error.
// RUN: split-file %s %t
// The standard library modules manifest (libc++.modules.json) is discovered
// relative to the installed C++ standard library runtime libraries
// We need to create them in order for Clang to find the manifest.
// RUN: rm -rf %t && split-file %s %t && cd %t
// RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu
// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so
// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a
// Add the standard module manifest itself.
// RUN: cat %t/libc++.modules.json.in > \
// RUN: %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json
// RUN: not %clang -std=c++20 \
// RUN: -fmodules-driver -Rmodules-driver \
// RUN: -stdlib=libc++ \
// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \
// RUN: --target=x86_64-linux-gnu \
// RUN: -fmodules-cache-path=%t/modules-cache \
// RUN: %t/main.cpp \
// RUN: -### 2>&1 \
// RUN: | sed 's:\\\\\?:/:g' \
// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: remark: using standard modules manifest file '[[PREFIX]]/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json' [-Rmodules-driver]
// CHECK: error: failure while parsing standard modules manifest: '[5:7, byte=57]: Invalid JSON value (true?)'
//--- main.cpp
// empty
//--- libc++.modules.json.in
{
"version": 1,
"revision": 1,
"modules": [
this is malformed.
]
}

View file

@ -0,0 +1,65 @@
// Checks that -cc1 command lines generated for inputs from the Standard Library
// module manifest are correctly adjusted, specifically that:
// - manifest-specified local arguments are applied to the corresponding entry.
// - "-Wno-reserved-module-identifier" is added for Standard library modules.
// The standard library modules manifest (libc++.modules.json) is discovered
// relative to the installed C++ standard library runtime libraries
// We need to create them in order for Clang to find the manifest.
// RUN: rm -rf %t && split-file %s %t && cd %t
// RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu
// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so
// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a
// RUN: sed "s|DIR|%/t|g" %t/libc++.modules.json.in > \
// RUN: %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json
// RUN: mkdir -p %t/Inputs/usr/lib/share/libc++/v1
// RUN: cat %t/std.cppm > %t/Inputs/usr/lib/share/libc++/v1/std.cppm
// RUN: %clang -std=c++20 \
// RUN: -fmodules-driver -Rmodules-driver \
// RUN: -stdlib=libc++ \
// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \
// RUN: --target=x86_64-linux-gnu \
// RUN: -fmodules-cache-path=%t/modules-cache \
// RUN: %t/main.cpp \
// RUN: -### 2>&1 \
// RUN: | sed 's:\\\\\?:/:g' \
// RUN: | FileCheck %s -check-prefix=MAIN-CC1 -check-prefix=STDLIB-MOD-CC1 -DPREFIX=%/t
// MAIN-CC1: "-cc1"
// MAIN-CC1-SAME: "main.cpp"
// MAIN-CC1-NOT: "-Wno-reserved-module-identifier"
// MAIN-CC1-NOT: "-internal-isystem" "[[PREFIX]]/Inputs/usr/lib/x86_64-linux-gnu/../share/libc++/v1/"
// STDLIB-MOD-CC1: "-cc1"
// STDLIB-MOD-CC1-SAME: "[[PREFIX]]/Inputs/usr/lib/x86_64-linux-gnu/../share/libc++/v1/std.cppm"
// STDLIB-MOD-CC1-SAME: "-Wno-reserved-module-identifier"
// STDLIB-MOD-CC1-SAME: "-internal-isystem" "[[PREFIX]]/Inputs/usr/lib/x86_64-linux-gnu/../share/libc++/v1/"
//--- main.cpp
import std;
int main() {}
//--- std.cppm
export module std;
//--- libc++.modules.json.in
{
"version": 1,
"revision": 1,
"modules": [
{
"logical-name": "std",
"source-path": "../share/libc++/v1/std.cppm",
"is-std-library": true,
"local-arguments": {
"system-include-directories": [
"../share/libc++/v1/"
]
}
}
]
}

View file

@ -0,0 +1,17 @@
// Tests the remark for when the Standard library modules manifest is
// not found.
// RUN: split-file %s %t
//--- main.cpp
// empty
// RUN: %clang -std=c++23 -nostdlib -fmodules \
// RUN: -fmodules-driver -Rmodules-driver \
// RUN: -fmodule-map-file=%t/module.modulemap %t/main.cpp \
// RUN: -fmodules-cache-path=%t/modules-cache \
// RUN: %t/main.cpp -### 2>&1 \
// RUN: | sed 's:\\\\\?:/:g' \
// RUN: | FileCheck -DPREFIX=%/t %s
// CHECK: clang: remark: standard modules manifest file not found; import of standard library modules not supported [-Rmodules-driver]