[BOLT] Support multiple perf data inputs

Allow multiple -p/-perfdata args (comma-separated and repeated).
Process them in parallel with `--perfdata-jobs/-pj` (default 4,
0 for all available hardware threads). Keep YAML/DataReader
inputs single-profile only.

This also enables density computation with multiple perf files.
Prior to that, it was impossible to compute total density as
it wasn't computed from symbolized profiles.

Depends on
#199323
#199322
#199321
#199320

Test Plan:
Updated pre-aggregated-perf.test and perf_test

For a large binary (300MB text) and 7 ETM perf data files
(6GB each), compared to parallel perf2bolt + merge-fdata:
* processing time is equal: 670+3s -> 669s (within noise),
* output is equivalent: sorted fdata is equal.

Reviewers:
paschalis-mpeis, rafaelauler, yozhu, ayermolo, yavtuk, maksfb, yota9

Reviewed By: paschalis-mpeis, rafaelauler

Pull Request: https://github.com/llvm/llvm-project/pull/199324

bolt/include/bolt/Profile/DataAggregator.h

@@ -51,9 +51,7 @@ class BoltAddressTranslation;
 /// specified by the user.
 class DataAggregator : public DataReader {
 public:
-  explicit DataAggregator(StringRef Filename) : DataReader(Filename) {
-    start();
-  }
+  explicit DataAggregator(StringRef Filename) : DataReader(Filename) {}
 
   ~DataAggregator();
 
@@ -69,6 +67,10 @@ public:
 
   Error readProfile(BinaryContext &BC) override;
 
+  /// Add an additional perf.data or pre-aggregated profile input to be merged
+  /// into this aggregation job.
+  void addInputFile(StringRef Filename);
+
   bool mayHaveProfileData(const BinaryFunction &BF) override;
 
   /// Set Bolt Address Translation Table when processing samples collected in
@@ -147,6 +149,10 @@ private:
   std::unordered_map<uint64_t, uint64_t> BasicSamples;
   std::vector<PerfMemSample> MemSamples;
 
+  /// Perf.data or pre-aggregated inputs to aggregate and merge into this
+  /// reader.
+  std::vector<std::string> InputFilenames;
+
   /// Filter pre-aggregated entries belonging to a DSO with this buildid.
   /// Set when processing a shared library, empty implies main binary.
   StringRef FilterBuildID;
@@ -394,6 +400,9 @@ private:
   /// Parse this aggregator's input file.
   void parseInput();
 
+  /// Merge parsed profile data from another aggregation job.
+  void mergeFrom(const DataAggregator &Other);
+
   /// Mark binary functions covered by parsed profile data.
   void markFunctionsWithProfile();
 

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -94,7 +94,7 @@ extern llvm::cl::opt<bool> HotText;
 extern llvm::cl::opt<bool> Hugify;
 extern llvm::cl::opt<bool> Instrument;
 extern llvm::cl::opt<std::string> OutputFilename;
-extern llvm::cl::opt<std::string> PerfData;
+extern llvm::cl::list<std::string> PerfData;
 extern llvm::cl::opt<bool> PrintCacheMetrics;
 extern llvm::cl::opt<bool> PrintSections;
 extern llvm::cl::opt<bool> UpdateBranchProtection;

bolt/lib/Profile/DataAggregator.cpp

@@ -28,6 +28,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Parallel.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Regex.h"
@@ -114,6 +115,16 @@ MaxSamples("max-samples",
   cl::Hidden,
   cl::cat(AggregatorCategory));
 
+static cl::opt<unsigned>
+    PerfDataJobs("perfdata-jobs",
+                 cl::desc("number of perf data files to process in parallel, "
+                          "0 = all HW threads (default 4)"),
+                 cl::init(4), cl::cat(AggregatorCategory),
+                 cl::sub(cl::SubCommand::getAll()));
+
+static cl::alias PerfDataJobsAlias("pj", cl::desc("Alias for --perfdata-jobs"),
+                                   cl::aliasopt(PerfDataJobs));
+
 extern cl::opt<opts::ProfileFormatKind> ProfileFormat;
 extern cl::opt<bool> ProfileWritePseudoProbes;
 extern cl::opt<std::string> SaveProfile;
@@ -169,6 +180,27 @@ std::vector<SectionNameAndRange> getTextSections(const BinaryContext *BC) {
 
 DataAggregator::~DataAggregator() { deleteTempFiles(); }
 
+void DataAggregator::addInputFile(StringRef Filename) {
+  InputFilenames.emplace_back(Filename);
+}
+
+void DataAggregator::mergeFrom(const DataAggregator &Other) {
+  Traces.insert(Traces.end(), Other.Traces.begin(), Other.Traces.end());
+
+  for (const auto &[PC, Count] : Other.BasicSamples)
+    BasicSamples[PC] += Count;
+
+  MemSamples.insert(MemSamples.end(), Other.MemSamples.begin(),
+                    Other.MemSamples.end());
+  Returns.insert(Other.Returns.begin(), Other.Returns.end());
+  EventNames.insert(Other.EventNames.begin(), Other.EventNames.end());
+
+  NumTraces += Other.NumTraces;
+  NumInvalidTraces += Other.NumInvalidTraces;
+  NumLongRangeTraces += Other.NumLongRangeTraces;
+  NumTotalSamples += Other.NumTotalSamples;
+}
+
 void DataAggregator::markFunctionsWithProfile() {
   std::unordered_set<uint64_t> Samples;
   std::unordered_set<BinaryFunction *> Funcs;
@@ -228,10 +260,6 @@ void DataAggregator::findPerfExecutable() {
 void DataAggregator::start() {
   outs() << "PERF2BOLT: Starting data aggregation job for " << Filename << "\n";
 
-  // Turn on heatmap building if requested by --heatmap flag.
-  if (!opts::HeatmapMode && opts::HeatmapOutput.getNumOccurrences())
-    opts::HeatmapMode = opts::HeatmapModeKind::HM_Optional;
-
   // Don't launch perf for pre-aggregated files or when perf input is specified
   // by the user.
   if (opts::ReadPreAggregated || !opts::ReadPerfEvents.empty())
@@ -707,6 +735,8 @@ void DataAggregator::imputeFallThroughs() {
 }
 
 void DataAggregator::parseInput() {
+  start();
+
   if (opts::ReadPreAggregated)
     parsePreAggregated();
   else
@@ -714,9 +744,14 @@ void DataAggregator::parseInput() {
 }
 
 Error DataAggregator::preprocessProfile(BinaryContext &BC) {
+  // Turn on heatmap building if requested by --heatmap flag.
+  if (!opts::HeatmapMode && opts::HeatmapOutput.getNumOccurrences())
+    opts::HeatmapMode = opts::HeatmapModeKind::HM_Optional;
+
   this->BC = &BC;
 
   if (opts::GeneratePerfTextProfile) {
+    start();
     if (Error E = generatePerfTextData()) {
       deleteTempFiles();
       exit(1);
@@ -724,7 +759,21 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
     exit(0);
   }
 
-  parseInput();
+  SmallVector<DataAggregator *, 1> Aggregators(1, this);
+  for (StringRef InputFilename : InputFilenames) {
+    auto *DA = Aggregators.emplace_back(new DataAggregator(InputFilename));
+    DA->BC = &BC;
+  }
+
+  ThreadPoolStrategy SavedStrategy = parallel::strategy;
+  parallel::strategy = hardware_concurrency(opts::PerfDataJobs);
+  parallelForEach(Aggregators, [](DataAggregator *DA) { DA->parseInput(); });
+  parallel::strategy = SavedStrategy;
+
+  for (DataAggregator *DA : llvm::drop_begin(Aggregators)) {
+    mergeFrom(*DA);
+    delete DA;
+  }
 
   markFunctionsWithProfile();
 

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -473,6 +473,13 @@ Error RewriteInstance::setProfile(StringRef Filename) {
     return errorCodeToError(make_error_code(errc::no_such_file_or_directory));
 
   if (ProfileReader) {
+    if (DataAggregator::checkPerfDataMagic(Filename) &&
+        // Poor man's RTTI
+        ProfileReader->getReaderName() == StringRef("perf data aggregator")) {
+      static_cast<DataAggregator *>(ProfileReader.get())
+          ->addInputFile(Filename);
+      return Error::success();
+    }
     // Already exists
     return make_error<StringError>(Twine("multiple profiles specified: ") +
                                        ProfileReader->getFilename() + " and " +

bolt/lib/Utils/CommandLineOpts.cpp

@@ -243,15 +243,14 @@ OutputFilename("o",
   cl::Optional,
   cl::cat(BoltOutputCategory));
 
-cl::opt<std::string> PerfData("perfdata", cl::desc("<data file>"), cl::Optional,
-                              cl::cat(AggregatorCategory),
-                              cl::sub(cl::SubCommand::getAll()));
+cl::list<std::string> PerfData("perfdata", cl::CommaSeparated,
+                               cl::desc("<data file>"),
+                               cl::cat(AggregatorCategory),
+                               cl::sub(cl::SubCommand::getAll()));
 
-static cl::alias
-PerfDataA("p",
-  cl::desc("alias for -perfdata"),
-  cl::aliasopt(PerfData),
-  cl::cat(AggregatorCategory));
+static cl::alias PerfDataA("p", cl::CommaSeparated,
+                           cl::desc("alias for -perfdata"),
+                           cl::aliasopt(PerfData), cl::cat(AggregatorCategory));
 
 cl::opt<bool> PrintCacheMetrics(
     "print-cache-metrics",

bolt/test/X86/pre-aggregated-perf.test

@@ -33,8 +33,13 @@ CHECK-WARNING: BOLT-INFO: Functions with density >= 21.7 account for 97.00% tota
 RUN: llvm-bolt %t.exe -data %t -o %t.null | FileCheck %s
 RUN: llvm-bolt %t.exe -data %t.new -o %t.null | FileCheck %s
 RUN: llvm-bolt %t.exe -p %p/Inputs/pre-aggregated.txt --pa -o %t.null | FileCheck %s
+RUN: llvm-bolt %t.exe --pa -perfdata %p/Inputs/pre-aggregated.txt -perfdata %p/Inputs/pre-aggregated.txt -o %t.multi-perfdata.null | FileCheck %s --check-prefix=CHECK-MULTI
+RUN: llvm-bolt %t.exe --pa -p %p/Inputs/pre-aggregated.txt,%p/Inputs/pre-aggregated.txt -o %t.multi-perfdata-comma.null | FileCheck %s --check-prefix=CHECK-MULTI
+RUN: not llvm-bolt %t.exe --pa -p %p/Inputs/pre-aggregated.txt -p %t.missing -o %t.multi-perfdata-missing.null 2>&1 | FileCheck %s --check-prefix=CHECK-MISSING
 
 CHECK: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
+CHECK-MULTI: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
+CHECK-MISSING: {{.*}}missing': No such file or directory.
 
 RUN: FileCheck %s -check-prefix=PERF2BOLT --input-file %t
 RUN: FileCheck %s -check-prefix=NEWFORMAT --input-file %t.new

bolt/test/perf2bolt/perf_test.test

@@ -7,11 +7,23 @@ RUN: perf record -Fmax -e cycles:u -o %t2 -- %t
 RUN: perf2bolt %t -p=%t2 -o %t3 -ba -ignore-build-id --show-density \
 RUN:   --heatmap %t.hm 2>&1 | FileCheck %s
 RUN: FileCheck %s --input-file %t.hm-section-hotness.csv --check-prefix=CHECK-HM
+# Multiple perf.data files as input
+RUN: perf2bolt %t -p=%t2 -p %t2 -o %t3.multi -ba -ignore-build-id \
+RUN:   | FileCheck %s --check-prefix=CHECK-MULTI
+RUN: perf2bolt %t -p=%t2,%t2 -o %t3.comma -ba -ignore-build-id \
+RUN:   | FileCheck %s --check-prefix=CHECK-MULTI
+RUN: cmp %t3.multi %t3.comma
+# Check counts are 2x the original
+RUN: merge-fdata %t3 %t3 | sort > %t3.x2
+RUN: sort %t3.multi > %t3.multi.x2
+RUN: cmp %t3.x2 %t3.multi.x2
 
 CHECK-NOT: PERF2BOLT-ERROR
 CHECK-NOT: !! WARNING !! This high mismatch ratio indicates the input binary is probably not the same binary used during profiling collection.
 CHECK: HEATMAP: building heat map
 CHECK: BOLT-INFO: Functions with density >= {{.*}} account for 99.00% total sample counts.
+CHECK-MULTI: PERF2BOLT: Starting data aggregation job
+CHECK-MULTI: PERF2BOLT: Starting data aggregation job
 
 RUN: %clang %S/Inputs/perf_test.c -no-pie -fuse-ld=lld -o %t4
 RUN: perf record -Fmax -e cycles:u -o %t5 -- %t4

bolt/tools/driver/llvm-bolt.cpp

@@ -234,8 +234,9 @@ int main(int argc, char **argv) {
       }
 
       if (!opts::PerfData.empty()) {
-        if (Error E = RI.setProfile(opts::PerfData))
-          report_error(opts::PerfData, std::move(E));
+        for (StringRef Filename : opts::PerfData)
+          if (Error E = RI.setProfile(Filename))
+            report_error(Filename, std::move(E));
       } else if (opts::AggregateOnly) {
         errs() << ToolName << ": missing required -perfdata option.\n";
         exit(1);

bolt/tools/heatmap/heatmap.cpp

@@ -121,8 +121,9 @@ int main(int argc, char **argv) {
       report_error("RewriteInstance", std::move(E));
 
     RewriteInstance &RI = *RIOrErr.get();
-    if (Error E = RI.setProfile(opts::PerfData))
-      report_error(opts::PerfData, std::move(E));
+    for (StringRef Filename : opts::PerfData)
+      if (Error E = RI.setProfile(Filename))
+        report_error(Filename, std::move(E));
 
     if (Error E = RI.run())
       report_error(opts::InputFilename, std::move(E));