mirror-llvm-project/libcxx/utils/benchmark-historical

#!/usr/bin/env python3

import argparse
import logging
import os
import pathlib
import subprocess
import sys
import tempfile

PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))

def directory_path(string):
    if os.path.isdir(string):
        return pathlib.Path(string)
    else:
        raise NotADirectoryError(string)

def whitespace_separated(stream):
    """
    Iterate over a stream, yielding whitespace-delimited elements.
    """
    for line in stream:
        for element in line.split():
            yield element

def resolve_commit(git_repo, commit):
    """
    Resolve the full commit SHA from any tree-ish.
    """
    return subprocess.check_output(['git', '-C', git_repo, 'rev-parse', commit], text=True).strip()


def main(argv):
    parser = argparse.ArgumentParser(
        prog='benchmark-historical',
        description='Run the libc++ benchmarks against the commits provided on standard input and store the results in '
                    'LNT format in a directory. This makes it easy to generate historical benchmark results of libc++ '
                    'for analysis purposes. This script\'s usage is optimized to be run on a set of commits and then '
                    're-run on a potentially-overlapping set of commits, such as after pulling new commits with Git.')
    parser.add_argument('--output', '-o', type=pathlib.Path, required=True,
        help='Path to the directory where the resulting .lnt files are stored.')
    parser.add_argument('--compiler', type=str, required=True,
        help='Path to the compiler to use to build libc++ and run the tests.')
    parser.add_argument('--commit-list', type=argparse.FileType('r'), default=sys.stdin,
        help='Path to a file containing a whitespace separated list of commits to test. '
             'By default, this is read from standard input.')
    parser.add_argument('--existing', type=str, choices=['skip', 'overwrite', 'append'], default='skip',
        help='This option instructs what to do when data for a commit already exists in the output directory. '
             'Selecting "skip" instructs the tool to skip generating data for a commit that already has data, '
             '"overwrite" will overwrite the existing data with the newly-generated one, and "append" will '
             'append the new data to the existing one. By default, the tool uses "skip".')
    parser.add_argument('lit_options', nargs=argparse.REMAINDER,
        help='Optional arguments passed to lit when running the tests. Should be provided last and '
             'separated from other arguments with a `--`.')
    parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),
        help='Optional path to the Git repository to use. By default, the current working directory is used.')
    parser.add_argument('--dry-run', action='store_true',
        help='Do not actually run anything, just print what would be done.')
    args = parser.parse_args(argv)

    logging.getLogger().setLevel(logging.INFO)

    # Gather lit options
    lit_options = []
    if args.lit_options:
        if args.lit_options[0] != '--':
            raise ArgumentError('For clarity, Lit options must be separated from other options by --')
        lit_options = args.lit_options[1:]

    # Process commits one by one. Commits just need to be whitespace separated: we also handle
    # the case where there is more than one commit per line.
    for commit in whitespace_separated(args.commit_list):
        commit = resolve_commit(args.git_repo, commit) # resolve e.g. HEAD to a real SHA

        output_file = args.output / (commit + '.lnt')
        if output_file.exists() and args.existing == 'skip':
            logging.info(f'Skipping {commit} which already has data in {output_file}')
            continue
        else:
            logging.info(f'Benchmarking {commit} against test-suite in {args.git_repo}')

        with tempfile.TemporaryDirectory() as libcxx_install_dir:
            with tempfile.TemporaryDirectory() as build_dir:
                build_cmd = [PARENT_DIR / 'build-at-commit', '--git-repo', args.git_repo,
                                                             '--commit', commit,
                                                             '--install-dir', libcxx_install_dir,
                                                             '--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo',
                                                                   f'-DCMAKE_CXX_COMPILER={args.compiler}']

                test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo,
                                                           '--libcxx-installation', libcxx_install_dir,
                                                           '--compiler', args.compiler,
                                                           '--build-dir', build_dir]
                test_cmd += ['--'] + lit_options

                if args.dry_run:
                    logging.info(f'Running {" ".join(str(a) for a in build_cmd)}')
                    logging.info(f'Running {" ".join(str(a) for a in test_cmd)}')
                    continue

                subprocess.check_call(build_cmd)
                subprocess.call(test_cmd)
                output_file.parent.mkdir(parents=True, exist_ok=True)
                mode = 'a' if args.existing == 'append' else 'w'
                if output_file.exists() and args.existing == 'append':
                    logging.info(f'Appending to existing data for {commit}')
                elif output_file.exists() and args.existing == 'overwrite':
                    logging.info(f'Overwriting existing data for {commit}')
                else:
                    logging.info(f'Writing data for {commit}')
                with open(output_file, mode) as out:
                    subprocess.check_call([(PARENT_DIR / 'consolidate-benchmarks'), build_dir], stdout=out)

if __name__ == '__main__':
    main(sys.argv[1:])