[OFFLOAD] Add plugin with support for Intel oneAPI Level Zero (#158900)
Add a new nextgen plugin that supports GPU devices through the Intel oneAPI Level Zero library. The plugin is not enabled by default and needs to be added to LIBOMPTARGET_PLUGINS_TO_BUILD explicitely. --------- Co-authored-by: Alexey Sachkov <alexey.sachkov@intel.com> Co-authored-by: Nick Sarnie <nick.sarnie@intel.com> Co-authored-by: Joseph Huber <huberjn@outlook.com>
This commit is contained in:
Alex Duran
GitHub
parent
24d9550b27
commit
f125c8db5c
26 changed files with 5888 additions and 1 deletions
|
|
@ -152,7 +152,7 @@ endif()
|
|||
|
||||
set(LIBOMPTARGET_ALL_PLUGIN_TARGETS amdgpu cuda host)
|
||||
set(LIBOMPTARGET_PLUGINS_TO_BUILD "all" CACHE STRING
|
||||
"Semicolon-separated list of plugins to use: cuda, amdgpu, host or \"all\".")
|
||||
"Semicolon-separated list of plugins to use: cuda, amdgpu, level_zero, host or \"all\".")
|
||||
|
||||
if(LIBOMPTARGET_PLUGINS_TO_BUILD STREQUAL "all")
|
||||
set(LIBOMPTARGET_PLUGINS_TO_BUILD ${LIBOMPTARGET_ALL_PLUGIN_TARGETS})
|
||||
|
|
@ -176,6 +176,18 @@ if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$"
|
|||
list(REMOVE_ITEM LIBOMPTARGET_PLUGINS_TO_BUILD "cuda")
|
||||
endif()
|
||||
endif()
|
||||
if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64)$" AND
|
||||
CMAKE_SYSTEM_NAME MATCHES "Linux|Windows"))
|
||||
if("level_zero" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
|
||||
message(STATUS "Not building Level Zero plugin: it is only supported on "
|
||||
"Linux/Windows x86_64 or ppc64le hosts")
|
||||
list(REMOVE_ITEM LIBOMPTARGET_PLUGINS_TO_BUILD "level_zero")
|
||||
endif()
|
||||
endif()
|
||||
if("level_zero" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD AND
|
||||
NOT LIBOMPTARGET_DEP_LEVEL_ZERO_FOUND)
|
||||
list(REMOVE_ITEM LIBOMPTARGET_PLUGINS_TO_BUILD "level_zero")
|
||||
endif()
|
||||
message(STATUS "Building the offload library with support for "
|
||||
"the \"${LIBOMPTARGET_PLUGINS_TO_BUILD}\" plugins")
|
||||
|
||||
|
|
@ -214,6 +226,7 @@ set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} riscv64-unknown-linux
|
|||
set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} riscv64-unknown-linux-gnu-LTO")
|
||||
set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} loongarch64-unknown-linux-gnu")
|
||||
set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} loongarch64-unknown-linux-gnu-LTO")
|
||||
set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} spirv64-intel")
|
||||
|
||||
# Once the plugins for the different targets are validated, they will be added to
|
||||
# the list of supported targets in the current system.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,16 @@ endif()
|
|||
find_package(FFI QUIET)
|
||||
set(LIBOMPTARGET_DEP_LIBFFI_FOUND ${FFI_FOUND})
|
||||
|
||||
################################################################################
|
||||
# Looking for offload-arch...
|
||||
################################################################################
|
||||
if(TARGET offload-arch)
|
||||
get_property(LIBOMPTARGET_OFFLOAD_ARCH TARGET offload-arch PROPERTY LOCATION)
|
||||
else()
|
||||
find_program(LIBOMPTARGET_OFFLOAD_ARCH NAMES offload-arch
|
||||
PATHS ${LLVM_TOOLS_BINARY_DIR})
|
||||
endif()
|
||||
|
||||
################################################################################
|
||||
# Looking for NVIDIA GPUs...
|
||||
################################################################################
|
||||
|
|
@ -89,4 +99,27 @@ if(LIBOMPTARGET_AMDGPU_ARCH)
|
|||
endif()
|
||||
endif()
|
||||
|
||||
################################################################################
|
||||
# Looking for Level0
|
||||
################################################################################
|
||||
find_path(LIBOMPTARGET_DEP_LEVEL_ZERO_INCLUDE_DIR NAMES level_zero/ze_api.h)
|
||||
|
||||
if(NOT LIBOMPTARGET_DEP_LEVEL_ZERO_INCLUDE_DIR)
|
||||
set(LIBOMPTARGET_DEP_LEVEL_ZERO_FOUND FALSE)
|
||||
else()
|
||||
set(LIBOMPTARGET_DEP_LEVEL_ZERO_FOUND TRUE)
|
||||
find_library(LIBOMPTARGET_DEP_LEVEL_ZERO_LIBRARY NAMES ze_loader)
|
||||
endif()
|
||||
|
||||
if(LIBOMPTARGET_OFFLOAD_ARCH)
|
||||
execute_process(COMMAND ${LIBOMPTARGET_OFFLOAD_ARCH} "--only=intel"
|
||||
OUTPUT_VARIABLE LIBOMPTARGET_INTELGPU_ARCH_OUTPUT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
string(REPLACE "\n" ";" intelgpu_arch_list "${LIBOMPTARGET_INTELGPU_ARCH_OUTPUT}")
|
||||
if(intelgpu_arch_list)
|
||||
set(LIBOMPTARGET_FOUND_INTELGPU_GPU TRUE)
|
||||
set(LIBOMPTARGET_INTELGPU_DETECTED_ARCH_LIST "${intelgpu_arch_list}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(OPENMP_PTHREAD_LIB ${LLVM_PTHREAD_LIB})
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ enum TargetAllocTy : int32_t {
|
|||
TARGET_ALLOC_HOST,
|
||||
TARGET_ALLOC_SHARED,
|
||||
TARGET_ALLOC_DEFAULT,
|
||||
TARGET_ALLOC_LAST = TARGET_ALLOC_DEFAULT
|
||||
};
|
||||
|
||||
struct DeviceTy;
|
||||
|
|
|
|||
|
|
@ -282,5 +282,21 @@ template <size_t Requested, size_t Required> constexpr void verboseAssert() {
|
|||
return dlwrap::SYM_USE##_Trait::get()(x0, x1, x2, x3, x4, x5, x6, x7, x8, \
|
||||
x9, x10); \
|
||||
}
|
||||
#define DLWRAP_INSTANTIATE_12(SYM_DEF, SYM_USE, T) \
|
||||
T::ReturnType SYM_DEF(typename T::template arg<0>::type x0, \
|
||||
typename T::template arg<1>::type x1, \
|
||||
typename T::template arg<2>::type x2, \
|
||||
typename T::template arg<3>::type x3, \
|
||||
typename T::template arg<4>::type x4, \
|
||||
typename T::template arg<5>::type x5, \
|
||||
typename T::template arg<6>::type x6, \
|
||||
typename T::template arg<7>::type x7, \
|
||||
typename T::template arg<8>::type x8, \
|
||||
typename T::template arg<9>::type x9, \
|
||||
typename T::template arg<10>::type x10, \
|
||||
typename T::template arg<11>::type x11) { \
|
||||
return dlwrap::SYM_USE##_Trait::get()(x0, x1, x2, x3, x4, x5, x6, x7, x8, \
|
||||
x9, x10, x11); \
|
||||
}
|
||||
|
||||
#endif // OMPTARGET_SHARED_DLWRAP_H
|
||||
|
|
|
|||
|
|
@ -1292,6 +1292,14 @@ struct GenericPluginTy {
|
|||
/// Create a new global handler for the underlying plugin.
|
||||
virtual GenericGlobalHandlerTy *createGlobalHandler() = 0;
|
||||
|
||||
/// Get the reference to the device with a certain device id.
|
||||
const GenericDeviceTy &getDevice(int32_t DeviceId) const {
|
||||
assert(isValidDeviceId(DeviceId) && "Invalid device id");
|
||||
assert(Devices[DeviceId] && "Device is uninitialized");
|
||||
|
||||
return *Devices[DeviceId];
|
||||
}
|
||||
|
||||
/// Get the reference to the device with a certain device id.
|
||||
GenericDeviceTy &getDevice(int32_t DeviceId) {
|
||||
assert(isValidDeviceId(DeviceId) && "Invalid device id");
|
||||
|
|
@ -1560,6 +1568,13 @@ public:
|
|||
/// object and return immediately.
|
||||
int32_t async_barrier(omp_interop_val_t *Interop);
|
||||
|
||||
/// Returns a Range over all the devices in the plugin that can be
|
||||
/// used in a for loop:
|
||||
/// for (&Device : GenericPluginRef.getDevicesRange()) {
|
||||
auto getDevicesRange() {
|
||||
return llvm::make_range(Devices.begin(), Devices.end());
|
||||
}
|
||||
|
||||
private:
|
||||
/// Indicates if the platform runtime has been fully initialized.
|
||||
bool Initialized = false;
|
||||
|
|
|
|||
61
offload/plugins-nextgen/level_zero/CMakeLists.txt
Normal file
61
offload/plugins-nextgen/level_zero/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# Create the library and add the default arguments.
|
||||
add_target_library(omptarget.rtl.level_zero LEVEL_ZERO)
|
||||
|
||||
set(LEVEL_ZERO_SRC_FILES
|
||||
src/L0Context.cpp
|
||||
src/L0Device.cpp
|
||||
src/L0Kernel.cpp
|
||||
src/L0Memory.cpp
|
||||
src/L0Program.cpp
|
||||
src/L0Plugin.cpp
|
||||
src/L0Program.cpp
|
||||
src/L0Options.cpp
|
||||
)
|
||||
|
||||
target_sources(omptarget.rtl.level_zero PRIVATE
|
||||
${LEVEL_ZERO_SRC_FILES}
|
||||
)
|
||||
|
||||
target_include_directories(omptarget.rtl.level_zero PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/include
|
||||
)
|
||||
|
||||
target_include_directories(omptarget.rtl.level_zero PRIVATE
|
||||
${LIBOMPTARGET_INCLUDE_DIR}
|
||||
${LIBOMPTARGET_DEP_LEVEL_ZERO_INCLUDE_DIR}
|
||||
${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
|
||||
${LIBOMPTARGET_OMP_HEADER_DIR}
|
||||
)
|
||||
|
||||
if (EXISTS ${LIBOMPTARGET_DEP_LEVEL_ZERO_LIBRARY} AND NOT "level_zero" IN_LIST LIBOMPTARGET_DLOPEN_PLUGINS)
|
||||
message(STATUS "Building Level Zero NG plugin linked against level_zero library")
|
||||
if(UNIX)
|
||||
target_link_libraries(omptarget.rtl.level_zero PRIVATE
|
||||
${LIBOMPTARGET_DEP_LEVEL_ZERO_LIBRARY})
|
||||
elseif(WIN32)
|
||||
# Full path to the Level Zero library is recognized as a linker option, so we
|
||||
# separate directory and file name
|
||||
cmake_path(GET LIBOMPTARGET_DEP_LEVEL_ZERO_LIBRARY FILENAME LEVEL_ZERO_LIBRARY_NAME)
|
||||
cmake_path(GET LIBOMPTARGET_DEP_LEVEL_ZERO_LIBRARY PARENT_PATH LEVEL_ZERO_LIBRARY_PATH)
|
||||
target_link_libraries(omptarget.rtl.level_zero PRIVATE ${LEVEL_ZERO_LIBRARY_NAME}
|
||||
${LIBOMP_LIB_FILE})
|
||||
target_link_directories(omptarget.rtl.level_zero PRIVATE ${LEVEL_ZERO_LIBRARY_PATH})
|
||||
else()
|
||||
message(FATAL_ERROR "Missing platform support")
|
||||
endif()
|
||||
else()
|
||||
message(STATUS "Building Level Zero NG plugin for dlopened level_zero")
|
||||
target_sources(omptarget.rtl.level_zero PRIVATE src/L0DynWrapper.cpp)
|
||||
endif()
|
||||
|
||||
option(LIBOMPTARGET_FORCE_LEVELZERO_TESTS "Build Level Zero libomptarget tests" OFF)
|
||||
if (LIBOMPTARGET_FOUND_INTELGPU_GPU OR LIBOMPTARGET_FORCE_LEVELZERO_TESTS)
|
||||
# Report to the parent scope that we are building a plugin for intelgpu
|
||||
set(LIBOMPTARGET_SYSTEM_TARGETS
|
||||
"${LIBOMPTARGET_SYSTEM_TARGETS} spirv64-intel" PARENT_SCOPE)
|
||||
list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.level_zero")
|
||||
set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
|
||||
else()
|
||||
message(STATUS "Not generating Level Zero tests, no supported devices detected."
|
||||
" Use 'LIBOMPTARGET_FORCE_INTELGPU_TESTS' to override.")
|
||||
endif()
|
||||
44
offload/plugins-nextgen/level_zero/include/AsyncQueue.h
Normal file
44
offload/plugins-nextgen/level_zero/include/AsyncQueue.h
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Async Queue wrapper for Level Zero.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_ASYNCQUEUE_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_ASYNCQUEUE_H
|
||||
|
||||
#include <tuple>
|
||||
|
||||
#include "L0Memory.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
/// Abstract queue that supports asynchronous command submission.
|
||||
struct AsyncQueueTy {
|
||||
/// List of events attached to submitted commands.
|
||||
llvm::SmallVector<ze_event_handle_t> WaitEvents;
|
||||
/// Pending staging buffer to host copies.
|
||||
llvm::SmallVector<std::tuple<void *, void *, size_t>> H2MList;
|
||||
/// Pending USM memory copy commands that must wait for kernel completion.
|
||||
llvm::SmallVector<std::tuple<const void *, void *, size_t>> USM2MList;
|
||||
/// Kernel event not signaled.
|
||||
ze_event_handle_t KernelEvent = nullptr;
|
||||
/// Clear data.
|
||||
void reset() {
|
||||
WaitEvents.clear();
|
||||
H2MList.clear();
|
||||
USM2MList.clear();
|
||||
KernelEvent = nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
using AsyncQueuePoolTy = ObjPool<AsyncQueueTy>;
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_ASYNCQUEUE_H
|
||||
142
offload/plugins-nextgen/level_zero/include/L0Context.h
Normal file
142
offload/plugins-nextgen/level_zero/include/L0Context.h
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Level Zero Context abstraction.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0CONTEXT_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0CONTEXT_H
|
||||
|
||||
#include "L0Memory.h"
|
||||
#include "PerThreadTable.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
class LevelZeroPluginTy;
|
||||
|
||||
class L0ContextTLSTy {
|
||||
StagingBufferTy StagingBuffer;
|
||||
|
||||
public:
|
||||
StagingBufferTy &getStagingBuffer() { return StagingBuffer; }
|
||||
const StagingBufferTy &getStagingBuffer() const { return StagingBuffer; }
|
||||
|
||||
Error deinit() { return StagingBuffer.clear(); }
|
||||
};
|
||||
|
||||
struct L0ContextTLSTableTy
|
||||
: public PerThreadContainer<
|
||||
std::unordered_map<ze_context_handle_t, L0ContextTLSTy>> {
|
||||
Error deinit() {
|
||||
return PerThreadTable::deinit(
|
||||
[](L0ContextTLSTy &Entry) -> auto { return Entry.deinit(); });
|
||||
}
|
||||
};
|
||||
|
||||
/// Driver and context-specific resources. We assume a single context per
|
||||
/// driver.
|
||||
class L0ContextTy {
|
||||
/// The plugin that created this context.
|
||||
LevelZeroPluginTy &Plugin;
|
||||
|
||||
/// Level Zero Driver handle.
|
||||
ze_driver_handle_t zeDriver = nullptr;
|
||||
|
||||
/// Common Level Zero context.
|
||||
ze_context_handle_t zeContext = nullptr;
|
||||
|
||||
/// API version supported by the Level Zero driver.
|
||||
ze_api_version_t APIVersion = ZE_API_VERSION_CURRENT;
|
||||
|
||||
/// Imported external pointers. Track this only for user-directed
|
||||
/// imports/releases.
|
||||
llvm::DenseMap<uintptr_t, size_t> ImportedPtrs;
|
||||
|
||||
/// Common event pool.
|
||||
EventPoolTy EventPool;
|
||||
|
||||
/// Host Memory allocator for this driver.
|
||||
MemAllocatorTy HostMemAllocator;
|
||||
|
||||
public:
|
||||
/// Named constants for checking the imported external pointer regions.
|
||||
static constexpr int32_t ImportNotExist = -1;
|
||||
static constexpr int32_t ImportUnknown = 0;
|
||||
static constexpr int32_t ImportExist = 1;
|
||||
|
||||
/// Create context, initialize event pool and extension functions.
|
||||
L0ContextTy(LevelZeroPluginTy &Plugin, ze_driver_handle_t zeDriver,
|
||||
int32_t DriverId)
|
||||
: Plugin(Plugin), zeDriver(zeDriver) {}
|
||||
|
||||
L0ContextTy(const L0ContextTy &) = delete;
|
||||
L0ContextTy(L0ContextTy &&) = delete;
|
||||
L0ContextTy &operator=(const L0ContextTy &) = delete;
|
||||
L0ContextTy &operator=(const L0ContextTy &&) = delete;
|
||||
|
||||
/// Release resources.
|
||||
~L0ContextTy() = default;
|
||||
|
||||
Error init();
|
||||
Error deinit();
|
||||
|
||||
LevelZeroPluginTy &getPlugin() const { return Plugin; }
|
||||
|
||||
StagingBufferTy &getStagingBuffer();
|
||||
|
||||
/// Add imported external pointer region.
|
||||
void addImported(void *Ptr, size_t Size) {
|
||||
(void)ImportedPtrs.try_emplace(reinterpret_cast<uintptr_t>(Ptr), Size);
|
||||
}
|
||||
|
||||
/// Remove imported external pointer region.
|
||||
void removeImported(void *Ptr) {
|
||||
(void)ImportedPtrs.erase(reinterpret_cast<uintptr_t>(Ptr));
|
||||
}
|
||||
/// Check if imported regions contain the specified region.
|
||||
int32_t checkImported(void *Ptr, size_t Size) const {
|
||||
uintptr_t LB = reinterpret_cast<uintptr_t>(Ptr);
|
||||
uintptr_t UB = LB + Size;
|
||||
// We do not expect a large number of user-directed imports, so use simple
|
||||
// logic.
|
||||
for (auto &I : ImportedPtrs) {
|
||||
uintptr_t ILB = I.first;
|
||||
uintptr_t IUB = ILB + I.second;
|
||||
if (LB >= ILB && UB <= IUB)
|
||||
return ImportExist;
|
||||
if ((LB >= ILB && LB < IUB) || (UB > ILB && UB <= IUB))
|
||||
return ImportUnknown;
|
||||
}
|
||||
return ImportNotExist;
|
||||
}
|
||||
|
||||
ze_driver_handle_t getZeDriver() const { return zeDriver; }
|
||||
|
||||
/// Return context associated with the driver.
|
||||
ze_context_handle_t getZeContext() const { return zeContext; }
|
||||
|
||||
/// Return driver API version.
|
||||
ze_api_version_t getDriverAPIVersion() const { return APIVersion; }
|
||||
|
||||
/// Return the event pool of this driver.
|
||||
EventPoolTy &getEventPool() { return EventPool; }
|
||||
const EventPoolTy &getEventPool() const { return EventPool; }
|
||||
|
||||
bool supportsLargeMem() const {
|
||||
// Large memory support is available since API version 1.1.
|
||||
return getDriverAPIVersion() >= ZE_API_VERSION_1_1;
|
||||
}
|
||||
|
||||
const MemAllocatorTy &getHostMemAllocator() const { return HostMemAllocator; }
|
||||
MemAllocatorTy &getHostMemAllocator() { return HostMemAllocator; }
|
||||
};
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0CONTEXT_H
|
||||
48
offload/plugins-nextgen/level_zero/include/L0Defs.h
Normal file
48
offload/plugins-nextgen/level_zero/include/L0Defs.h
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// External and other auxilary definitions.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0DEFS_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0DEFS_H
|
||||
|
||||
#include "PluginInterface.h"
|
||||
#include "Shared/Requirements.h"
|
||||
#include "omptarget.h"
|
||||
|
||||
enum class AllocOptionTy : int32_t {
|
||||
ALLOC_OPT_NONE = 0,
|
||||
ALLOC_OPT_REDUCTION_SCRATCH = 1,
|
||||
ALLOC_OPT_REDUCTION_COUNTER = 2,
|
||||
ALLOC_OPT_HOST_MEM = 3,
|
||||
ALLOC_OPT_SLM = 4,
|
||||
};
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
/// Default alignmnet for allocation.
|
||||
constexpr size_t L0DefaultAlignment = 0;
|
||||
/// Default staging buffer size for host to device copy (16KB).
|
||||
constexpr size_t L0StagingBufferSize = (1 << 14);
|
||||
/// Default staging buffer count.
|
||||
constexpr size_t L0StagingBufferCount = 64;
|
||||
/// USM allocation threshold where preallocation does not pay off (128MB).
|
||||
constexpr size_t L0UsmPreAllocThreshold = (128 << 20);
|
||||
/// Host USM allocation threshold where preallocation does not pay off (8MB).
|
||||
constexpr size_t L0HostUsmPreAllocThreshold = (8 << 20);
|
||||
/// Maximum wait time.
|
||||
constexpr uint64_t L0DefaultTimeout = std::numeric_limits<uint64_t>::max();
|
||||
/// Generic L0 handle type.
|
||||
using ZeHandleTy = void *;
|
||||
|
||||
using error::ErrorCode;
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0DEFS_H
|
||||
646
offload/plugins-nextgen/level_zero/include/L0Device.h
Normal file
646
offload/plugins-nextgen/level_zero/include/L0Device.h
Normal file
|
|
@ -0,0 +1,646 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// GenericDevice instatiation for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0DEVICE_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0DEVICE_H
|
||||
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
|
||||
#include "PerThreadTable.h"
|
||||
|
||||
#include "AsyncQueue.h"
|
||||
#include "L0Context.h"
|
||||
#include "L0Program.h"
|
||||
#include "PluginInterface.h"
|
||||
#include "TLS.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
using OmpInteropTy = omp_interop_val_t *;
|
||||
class LevelZeroPluginTy;
|
||||
|
||||
// clang-format off
|
||||
enum class PCIIdTy : int32_t {
|
||||
None = 0x0000,
|
||||
SKL = 0x1900,
|
||||
KBL = 0x5900,
|
||||
CFL = 0x3E00,
|
||||
CFL_2 = 0x9B00,
|
||||
ICX = 0x8A00,
|
||||
TGL = 0xFF20,
|
||||
TGL_2 = 0x9A00,
|
||||
DG1 = 0x4900,
|
||||
RKL = 0x4C00,
|
||||
ADLS = 0x4600,
|
||||
RTL = 0xA700,
|
||||
MTL = 0x7D00,
|
||||
PVC = 0x0B00,
|
||||
DG2_ATS_M = 0x4F00,
|
||||
DG2_ATS_M_2 = 0x5600,
|
||||
LNL = 0x6400,
|
||||
BMG = 0xE200,
|
||||
};
|
||||
|
||||
/// Device type enumeration common to compiler and runtime.
|
||||
enum class DeviceArchTy : uint64_t {
|
||||
DeviceArch_None = 0,
|
||||
DeviceArch_Gen = 0x0001, // Gen 9, Gen 11 or Xe
|
||||
DeviceArch_XeLPG = 0x0002,
|
||||
DeviceArch_XeHPC = 0x0004,
|
||||
DeviceArch_XeHPG = 0x0008,
|
||||
DeviceArch_Xe2LP = 0x0010,
|
||||
DeviceArch_Xe2HP = 0x0020,
|
||||
DeviceArch_x86_64 = 0x0100
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
struct L0DeviceIdTy {
|
||||
ze_device_handle_t zeId;
|
||||
int32_t RootId;
|
||||
int32_t SubId;
|
||||
int32_t CCSId;
|
||||
|
||||
L0DeviceIdTy(ze_device_handle_t Device, int32_t RootId, int32_t SubId = -1,
|
||||
int32_t CCSId = -1)
|
||||
: zeId(Device), RootId(RootId), SubId(SubId), CCSId(CCSId) {}
|
||||
};
|
||||
|
||||
class L0DeviceTLSTy {
|
||||
/// Command list for each device.
|
||||
ze_command_list_handle_t CmdList = nullptr;
|
||||
|
||||
/// Main copy command list for each device.
|
||||
ze_command_list_handle_t CopyCmdList = nullptr;
|
||||
|
||||
/// Command queue for each device.
|
||||
ze_command_queue_handle_t CmdQueue = nullptr;
|
||||
|
||||
/// Main copy command queue for each device.
|
||||
ze_command_queue_handle_t CopyCmdQueue = nullptr;
|
||||
|
||||
/// Immediate command list for each device.
|
||||
ze_command_list_handle_t ImmCmdList = nullptr;
|
||||
|
||||
/// Immediate copy command list for each device.
|
||||
ze_command_list_handle_t ImmCopyCmdList = nullptr;
|
||||
|
||||
public:
|
||||
L0DeviceTLSTy() = default;
|
||||
~L0DeviceTLSTy() {
|
||||
// assert all fields are nullptr on destruction.
|
||||
assert(!CmdList && !CopyCmdList && !CmdQueue && !CopyCmdQueue &&
|
||||
!ImmCmdList && !ImmCopyCmdList &&
|
||||
"L0DeviceTLSTy destroyed without clearing resources");
|
||||
}
|
||||
|
||||
L0DeviceTLSTy(const L0DeviceTLSTy &) = delete;
|
||||
L0DeviceTLSTy(L0DeviceTLSTy &&Other) {
|
||||
CmdList = std::exchange(Other.CmdList, nullptr);
|
||||
CopyCmdList = std::exchange(Other.CopyCmdList, nullptr);
|
||||
CmdQueue = std::exchange(Other.CmdQueue, nullptr);
|
||||
CopyCmdQueue = std::exchange(Other.CopyCmdQueue, nullptr);
|
||||
ImmCmdList = std::exchange(Other.ImmCmdList, nullptr);
|
||||
ImmCopyCmdList = std::exchange(Other.ImmCopyCmdList, nullptr);
|
||||
}
|
||||
|
||||
Error deinit() {
|
||||
// destroy all lists and queues.
|
||||
if (CmdList)
|
||||
CALL_ZE_RET_ERROR(zeCommandListDestroy, CmdList);
|
||||
if (CopyCmdList)
|
||||
CALL_ZE_RET_ERROR(zeCommandListDestroy, CopyCmdList);
|
||||
if (ImmCmdList)
|
||||
CALL_ZE_RET_ERROR(zeCommandListDestroy, ImmCmdList);
|
||||
if (ImmCopyCmdList)
|
||||
CALL_ZE_RET_ERROR(zeCommandListDestroy, ImmCopyCmdList);
|
||||
if (CmdQueue)
|
||||
CALL_ZE_RET_ERROR(zeCommandQueueDestroy, CmdQueue);
|
||||
if (CopyCmdQueue)
|
||||
CALL_ZE_RET_ERROR(zeCommandQueueDestroy, CopyCmdQueue);
|
||||
|
||||
CmdList = nullptr;
|
||||
CopyCmdList = nullptr;
|
||||
CmdQueue = nullptr;
|
||||
CopyCmdQueue = nullptr;
|
||||
ImmCmdList = nullptr;
|
||||
ImmCopyCmdList = nullptr;
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
L0DeviceTLSTy &operator=(const L0DeviceTLSTy &) = delete;
|
||||
L0DeviceTLSTy &operator=(L0DeviceTLSTy &&) = delete;
|
||||
|
||||
ze_command_list_handle_t getCmdList() const { return CmdList; }
|
||||
void setCmdList(ze_command_list_handle_t _CmdList) { CmdList = _CmdList; }
|
||||
|
||||
ze_command_list_handle_t getCopyCmdList() const { return CopyCmdList; }
|
||||
void setCopyCmdList(ze_command_list_handle_t _CopyCmdList) {
|
||||
CopyCmdList = _CopyCmdList;
|
||||
}
|
||||
|
||||
ze_command_list_handle_t getImmCmdList() const { return ImmCmdList; }
|
||||
void setImmCmdList(ze_command_list_handle_t ImmCmdListIn) {
|
||||
ImmCmdList = ImmCmdListIn;
|
||||
}
|
||||
|
||||
ze_command_list_handle_t getImmCopyCmdList() const { return ImmCopyCmdList; }
|
||||
void setImmCopyCmdList(ze_command_list_handle_t ImmCopyCmdListIn) {
|
||||
ImmCopyCmdList = ImmCopyCmdListIn;
|
||||
}
|
||||
|
||||
ze_command_queue_handle_t getCmdQueue() const { return CmdQueue; }
|
||||
void setCmdQueue(ze_command_queue_handle_t CmdQueueIn) {
|
||||
CmdQueue = CmdQueueIn;
|
||||
}
|
||||
|
||||
ze_command_queue_handle_t getCopyCmdQueue() const { return CopyCmdQueue; }
|
||||
void setCopyCmdQueue(ze_command_queue_handle_t CopyCmdQueueIn) {
|
||||
CopyCmdQueue = CopyCmdQueueIn;
|
||||
}
|
||||
};
|
||||
|
||||
struct L0DeviceTLSTableTy
|
||||
: public PerThreadContainer<std::vector<L0DeviceTLSTy>, 8> {
|
||||
Error deinit() {
|
||||
return PerThreadTable::deinit(
|
||||
[](L0DeviceTLSTy &Entry) { return Entry.deinit(); });
|
||||
}
|
||||
};
|
||||
|
||||
class L0DeviceTy final : public GenericDeviceTy {
|
||||
// Level Zero Context for this Device.
|
||||
L0ContextTy &l0Context;
|
||||
|
||||
// Level Zero handle for this Device.
|
||||
ze_device_handle_t zeDevice;
|
||||
// Device Properties.
|
||||
ze_device_properties_t DeviceProperties{};
|
||||
ze_device_compute_properties_t ComputeProperties{};
|
||||
ze_device_memory_properties_t MemoryProperties{};
|
||||
ze_device_cache_properties_t CacheProperties{};
|
||||
|
||||
/// Devices' default target allocation kind for internal allocation.
|
||||
int32_t AllocKind = TARGET_ALLOC_DEVICE;
|
||||
|
||||
DeviceArchTy DeviceArch = DeviceArchTy::DeviceArch_None;
|
||||
|
||||
std::string DeviceName;
|
||||
|
||||
/// Common indirect access flags for this device.
|
||||
ze_kernel_indirect_access_flags_t IndirectAccessFlags = 0;
|
||||
|
||||
/// Device UUID for toplevel devices only.
|
||||
std::string DeviceUuid;
|
||||
|
||||
/// L0 Device ID as string.
|
||||
std::string zeId;
|
||||
|
||||
/// Command queue group ordinals for each device.
|
||||
static constexpr uint32_t MaxOrdinal =
|
||||
std::numeric_limits<decltype(MaxOrdinal)>::max();
|
||||
std::pair<uint32_t, uint32_t> ComputeOrdinal{MaxOrdinal, 0};
|
||||
/// Command queue group ordinals for copying.
|
||||
std::pair<uint32_t, uint32_t> CopyOrdinal{MaxOrdinal, 0};
|
||||
|
||||
/// Command queue index for each device.
|
||||
uint32_t ComputeIndex = 0;
|
||||
|
||||
bool IsAsyncEnabled = false;
|
||||
|
||||
/// Lock for this device.
|
||||
std::mutex Mutex;
|
||||
|
||||
/// Contains all modules (possibly from multiple device images) to handle
|
||||
/// dynamic link across multiple images
|
||||
llvm::SmallVector<ze_module_handle_t> GlobalModules;
|
||||
|
||||
/// L0 programs created for this device
|
||||
std::list<L0ProgramTy> Programs;
|
||||
|
||||
/// MemAllocator for this device.
|
||||
MemAllocatorTy MemAllocator;
|
||||
|
||||
DeviceArchTy computeArch() const;
|
||||
|
||||
/// Get default compute group ordinal. Returns Ordinal-NumQueues pair.
|
||||
std::pair<uint32_t, uint32_t> findComputeOrdinal();
|
||||
|
||||
/// Get copy command queue group ordinal. Returns Ordinal-NumQueues pair.
|
||||
std::pair<uint32_t, uint32_t> findCopyOrdinal(bool LinkCopy = false);
|
||||
|
||||
public:
|
||||
L0DeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices,
|
||||
ze_device_handle_t zeDevice, L0ContextTy &DriverInfo,
|
||||
const std::string_view zeId, int32_t ComputeIndex)
|
||||
: GenericDeviceTy(Plugin, DeviceId, NumDevices, SPIRVGridValues),
|
||||
l0Context(DriverInfo), zeDevice(zeDevice), zeId(zeId),
|
||||
ComputeIndex(ComputeIndex) {
|
||||
DeviceProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
DeviceProperties.pNext = nullptr;
|
||||
ComputeProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_COMPUTE_PROPERTIES;
|
||||
ComputeProperties.pNext = nullptr;
|
||||
MemoryProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_MEMORY_PROPERTIES;
|
||||
MemoryProperties.pNext = nullptr;
|
||||
CacheProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_CACHE_PROPERTIES;
|
||||
CacheProperties.pNext = nullptr;
|
||||
}
|
||||
|
||||
static L0DeviceTy &makeL0Device(GenericDeviceTy &Device) {
|
||||
return static_cast<L0DeviceTy &>(Device);
|
||||
}
|
||||
LevelZeroPluginTy &getPlugin() {
|
||||
return reinterpret_cast<LevelZeroPluginTy &>(Plugin);
|
||||
}
|
||||
|
||||
L0DeviceTLSTy &getTLS();
|
||||
|
||||
Error setContext() override { return Plugin::success(); }
|
||||
Error initImpl(GenericPluginTy &Plugin) override;
|
||||
Error deinitImpl() override;
|
||||
ze_device_handle_t getZeDevice() const { return zeDevice; }
|
||||
|
||||
const L0ContextTy &getL0Context() const { return l0Context; }
|
||||
L0ContextTy &getL0Context() { return l0Context; }
|
||||
|
||||
const std::string_view getName() const { return DeviceName; }
|
||||
const char *getNameCStr() const { return DeviceName.c_str(); }
|
||||
|
||||
const std::string_view getZeId() const { return zeId; }
|
||||
const char *getZeIdCStr() const { return zeId.c_str(); }
|
||||
|
||||
std::mutex &getMutex() { return Mutex; }
|
||||
|
||||
uint32_t getComputeIndex() const { return ComputeIndex; }
|
||||
ze_kernel_indirect_access_flags_t getIndirectFlags() const {
|
||||
return IndirectAccessFlags;
|
||||
}
|
||||
|
||||
size_t getNumGlobalModules() const { return GlobalModules.size(); }
|
||||
void addGlobalModule(ze_module_handle_t Module) {
|
||||
GlobalModules.push_back(Module);
|
||||
}
|
||||
ze_module_handle_t *getGlobalModulesArray() { return GlobalModules.data(); }
|
||||
|
||||
L0ProgramTy *getProgramFromImage(MemoryBufferRef Image) {
|
||||
for (auto &PGM : Programs)
|
||||
if (PGM.getMemoryBuffer() == Image)
|
||||
return &PGM;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Error buildAllKernels() {
|
||||
for (auto &PGM : Programs) {
|
||||
if (auto Err = PGM.loadModuleKernels())
|
||||
return Err;
|
||||
}
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
// Add a new program to the device. Return a reference to the new program.
|
||||
L0ProgramTy &addProgram(int32_t ImageId,
|
||||
std::unique_ptr<MemoryBuffer> &&Image) {
|
||||
Programs.emplace_back(ImageId, *this, std::move(Image));
|
||||
return Programs.back();
|
||||
}
|
||||
|
||||
const L0ProgramTy &getLastProgram() const { return Programs.back(); }
|
||||
L0ProgramTy &getLastProgram() { return Programs.back(); }
|
||||
// Device properties getters.
|
||||
uint32_t getVendorId() const { return DeviceProperties.vendorId; }
|
||||
bool isGPU() const { return DeviceProperties.type == ZE_DEVICE_TYPE_GPU; }
|
||||
|
||||
uint32_t getPCIId() const { return DeviceProperties.deviceId; }
|
||||
uint32_t getNumThreadsPerEU() const {
|
||||
return DeviceProperties.numThreadsPerEU;
|
||||
}
|
||||
uint32_t getSIMDWidth() const { return DeviceProperties.physicalEUSimdWidth; }
|
||||
uint32_t getNumEUsPerSubslice() const {
|
||||
return DeviceProperties.numEUsPerSubslice;
|
||||
}
|
||||
uint32_t getNumSubslicesPerSlice() const {
|
||||
return DeviceProperties.numSubslicesPerSlice;
|
||||
}
|
||||
uint32_t getNumSlices() const { return DeviceProperties.numSlices; }
|
||||
uint32_t getNumSubslices() const {
|
||||
return DeviceProperties.numSubslicesPerSlice * DeviceProperties.numSlices;
|
||||
}
|
||||
uint32_t getNumEUs() const {
|
||||
return DeviceProperties.numEUsPerSubslice * getNumSubslices();
|
||||
}
|
||||
uint32_t getTotalThreads() const {
|
||||
return DeviceProperties.numThreadsPerEU * getNumEUs();
|
||||
}
|
||||
uint32_t getNumThreadsPerSubslice() const {
|
||||
return getNumEUsPerSubslice() * getNumThreadsPerEU();
|
||||
}
|
||||
uint32_t getClockRate() const { return DeviceProperties.coreClockRate; }
|
||||
|
||||
uint32_t getMaxSharedLocalMemory() const {
|
||||
return ComputeProperties.maxSharedLocalMemory;
|
||||
}
|
||||
uint32_t getMaxGroupSize() const {
|
||||
return ComputeProperties.maxTotalGroupSize;
|
||||
}
|
||||
uint32_t getMaxGroupCount() const {
|
||||
return getMaxGroupCountX() * getMaxGroupCountY() * getMaxGroupCountZ();
|
||||
}
|
||||
|
||||
uint32_t getMaxGroupSizeX() const { return ComputeProperties.maxGroupSizeX; }
|
||||
uint32_t getMaxGroupSizeY() const { return ComputeProperties.maxGroupSizeY; }
|
||||
uint32_t getMaxGroupSizeZ() const { return ComputeProperties.maxGroupSizeZ; }
|
||||
uint32_t getMaxGroupCountX() const {
|
||||
return ComputeProperties.maxGroupCountX;
|
||||
}
|
||||
uint32_t getMaxGroupCountY() const {
|
||||
return ComputeProperties.maxGroupCountY;
|
||||
}
|
||||
uint32_t getMaxGroupCountZ() const {
|
||||
return ComputeProperties.maxGroupCountZ;
|
||||
}
|
||||
uint32_t getMemoryClockRate() const { return MemoryProperties.maxClockRate; }
|
||||
uint64_t getGlobalMemorySize() const { return MemoryProperties.totalSize; }
|
||||
size_t getCacheSize() const { return CacheProperties.cacheSize; }
|
||||
uint64_t getMaxMemAllocSize() const {
|
||||
return DeviceProperties.maxMemAllocSize;
|
||||
}
|
||||
|
||||
int32_t getAllocKind() const { return AllocKind; }
|
||||
DeviceArchTy getDeviceArch() const { return DeviceArch; }
|
||||
bool isDeviceArch(DeviceArchTy Arch) const { return DeviceArch == Arch; }
|
||||
|
||||
static bool isDiscrete(uint32_t PCIId) {
|
||||
switch (static_cast<PCIIdTy>(PCIId & 0xFF00)) {
|
||||
case PCIIdTy::DG1:
|
||||
case PCIIdTy::PVC:
|
||||
case PCIIdTy::DG2_ATS_M:
|
||||
case PCIIdTy::DG2_ATS_M_2:
|
||||
case PCIIdTy::BMG:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool isDiscrete(ze_device_handle_t Device) {
|
||||
ze_device_properties_t PR{};
|
||||
PR.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
PR.pNext = nullptr;
|
||||
CALL_ZE_RET(false, zeDeviceGetProperties, Device, &PR);
|
||||
return isDiscrete(PR.deviceId);
|
||||
}
|
||||
|
||||
bool isDiscreteDevice() { return isDiscrete(getPCIId()); }
|
||||
bool isDeviceIPorNewer(uint32_t Version) const;
|
||||
|
||||
const std::string_view getUuid() const { return DeviceUuid; }
|
||||
|
||||
uint32_t getComputeEngine() const { return ComputeOrdinal.first; }
|
||||
uint32_t getNumComputeQueues() const { return ComputeOrdinal.second; }
|
||||
|
||||
bool hasMainCopyEngine() const { return CopyOrdinal.first != MaxOrdinal; }
|
||||
uint32_t getMainCopyEngine() const { return CopyOrdinal.first; }
|
||||
|
||||
bool deviceRequiresImmCmdList() const {
|
||||
constexpr uint32_t BMGIP = 0x05003000;
|
||||
return isDeviceIPorNewer(BMGIP);
|
||||
}
|
||||
bool asyncEnabled() const { return IsAsyncEnabled; }
|
||||
bool useImmForCompute() const { return true; }
|
||||
bool useImmForCopy() const { return true; }
|
||||
bool useImmForInterop() const { return true; }
|
||||
|
||||
void reportDeviceInfo() const;
|
||||
|
||||
// Command queues related functions.
|
||||
/// Create a command list with given ordinal and flags.
|
||||
Expected<ze_command_list_handle_t>
|
||||
createCmdList(ze_context_handle_t Context, ze_device_handle_t Device,
|
||||
uint32_t Ordinal, ze_command_list_flags_t Flags,
|
||||
const std::string_view DeviceIdStr);
|
||||
|
||||
/// Create a command list with default flags.
|
||||
Expected<ze_command_list_handle_t>
|
||||
createCmdList(ze_context_handle_t Context, ze_device_handle_t Device,
|
||||
uint32_t Ordinal, const std::string_view DeviceIdStr);
|
||||
|
||||
Expected<ze_command_list_handle_t> getCmdList();
|
||||
|
||||
/// Create a command queue with given ordinal and flags.
|
||||
Expected<ze_command_queue_handle_t>
|
||||
createCmdQueue(ze_context_handle_t Context, ze_device_handle_t Device,
|
||||
uint32_t Ordinal, uint32_t Index,
|
||||
ze_command_queue_flags_t Flags,
|
||||
const std::string_view DeviceIdStr);
|
||||
|
||||
/// Create a command queue with default flags.
|
||||
Expected<ze_command_queue_handle_t>
|
||||
createCmdQueue(ze_context_handle_t Context, ze_device_handle_t Device,
|
||||
uint32_t Ordinal, uint32_t Index,
|
||||
const std::string_view DeviceIdStr, bool InOrder = false);
|
||||
|
||||
/// Create a new command queue for the given OpenMP device ID.
|
||||
Expected<ze_command_queue_handle_t> createCommandQueue(bool InOrder = false);
|
||||
|
||||
/// Create an immediate command list.
|
||||
Expected<ze_command_list_handle_t>
|
||||
createImmCmdList(uint32_t Ordinal, uint32_t Index, bool InOrder = false);
|
||||
|
||||
/// Create an immediate command list for computing.
|
||||
Expected<ze_command_list_handle_t> createImmCmdList(bool InOrder = false) {
|
||||
return createImmCmdList(getComputeEngine(), getComputeIndex(), InOrder);
|
||||
}
|
||||
|
||||
/// Create an immediate command list for copying.
|
||||
Expected<ze_command_list_handle_t> createImmCopyCmdList();
|
||||
Expected<ze_command_queue_handle_t> getCmdQueue();
|
||||
Expected<ze_command_list_handle_t> getCopyCmdList();
|
||||
Expected<ze_command_queue_handle_t> getCopyCmdQueue();
|
||||
Expected<ze_command_list_handle_t> getImmCmdList();
|
||||
Expected<ze_command_list_handle_t> getImmCopyCmdList();
|
||||
|
||||
/// Enqueue copy command.
|
||||
Error enqueueMemCopy(void *Dst, const void *Src, size_t Size,
|
||||
__tgt_async_info *AsyncInfo = nullptr,
|
||||
bool UseCopyEngine = true);
|
||||
|
||||
/// Enqueue asynchronous copy command.
|
||||
Error enqueueMemCopyAsync(void *Dst, const void *Src, size_t Size,
|
||||
__tgt_async_info *AsyncInfo, bool CopyTo = true);
|
||||
|
||||
/// Enqueue fill command.
|
||||
Error enqueueMemFill(void *Ptr, const void *Pattern, size_t PatternSize,
|
||||
size_t Size);
|
||||
|
||||
/// Driver related functions.
|
||||
|
||||
/// Reurn the driver handle for this device.
|
||||
ze_driver_handle_t getZeDriver() const { return l0Context.getZeDriver(); }
|
||||
|
||||
/// Return context for this device.
|
||||
ze_context_handle_t getZeContext() const { return l0Context.getZeContext(); }
|
||||
|
||||
/// Return driver API version for this device.
|
||||
ze_api_version_t getDriverAPIVersion() const {
|
||||
return l0Context.getDriverAPIVersion();
|
||||
}
|
||||
|
||||
/// Return an event from the driver associated to this device.
|
||||
Expected<ze_event_handle_t> getEvent() {
|
||||
return l0Context.getEventPool().getEvent();
|
||||
}
|
||||
|
||||
/// Release event to the pool associated to this device.
|
||||
Error releaseEvent(ze_event_handle_t Event) {
|
||||
return l0Context.getEventPool().releaseEvent(Event, *this);
|
||||
}
|
||||
|
||||
StagingBufferTy &getStagingBuffer() { return l0Context.getStagingBuffer(); }
|
||||
|
||||
bool supportsLargeMem() const { return l0Context.supportsLargeMem(); }
|
||||
|
||||
// Allocation related routines.
|
||||
|
||||
/// Data alloc.
|
||||
Expected<void *> dataAlloc(
|
||||
size_t Size, size_t Align, int32_t Kind, intptr_t Offset, bool UserAlloc,
|
||||
bool DevMalloc = false,
|
||||
uint32_t MemAdvice = std::numeric_limits<decltype(MemAdvice)>::max(),
|
||||
AllocOptionTy AllocOpt = AllocOptionTy::ALLOC_OPT_NONE);
|
||||
|
||||
/// Data delete.
|
||||
Error dataDelete(void *Ptr);
|
||||
|
||||
/// Return the memory allocation type for the specified memory location.
|
||||
uint32_t getMemAllocType(const void *Ptr) const;
|
||||
|
||||
const MemAllocatorTy &getDeviceMemAllocator() const { return MemAllocator; }
|
||||
MemAllocatorTy &getDeviceMemAllocator() { return MemAllocator; }
|
||||
|
||||
MemAllocatorTy &getMemAllocator(int32_t Kind) {
|
||||
if (Kind == TARGET_ALLOC_HOST)
|
||||
return l0Context.getHostMemAllocator();
|
||||
return getDeviceMemAllocator();
|
||||
}
|
||||
|
||||
MemAllocatorTy &getMemAllocator(const void *Ptr) {
|
||||
if (ZE_MEMORY_TYPE_HOST == getMemAllocType(Ptr))
|
||||
return l0Context.getHostMemAllocator();
|
||||
return getDeviceMemAllocator();
|
||||
}
|
||||
|
||||
Error makeMemoryResident(void *Mem, size_t Size);
|
||||
|
||||
// Generic device interface implementation.
|
||||
Expected<DeviceImageTy *>
|
||||
loadBinaryImpl(std::unique_ptr<MemoryBuffer> &&TgtImage,
|
||||
int32_t ImageId) override;
|
||||
Error unloadBinaryImpl(DeviceImageTy *Image) override;
|
||||
Expected<void *> allocate(size_t Size, void *HstPtr,
|
||||
TargetAllocTy Kind) override;
|
||||
Error free(void *TgtPtr, TargetAllocTy Kind = TARGET_ALLOC_DEFAULT) override;
|
||||
|
||||
Expected<void *> dataLockImpl(void *HstPtr, int64_t Size) override {
|
||||
return Plugin::error(error::ErrorCode::UNKNOWN,
|
||||
"dataLockImpl not supported");
|
||||
}
|
||||
Error dataUnlockImpl(void *HstPtr) override { return Plugin::success(); }
|
||||
|
||||
Expected<bool> isPinnedPtrImpl(void *, void *&, void *&,
|
||||
size_t &) const override {
|
||||
// Don't need to do anything, this is handled by the driver.
|
||||
return false;
|
||||
}
|
||||
|
||||
Expected<bool> isAccessiblePtrImpl(const void *Ptr, size_t Size) override;
|
||||
Error dataFence(__tgt_async_info *Async) override;
|
||||
Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
|
||||
int64_t Size,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
|
||||
Error synchronizeImpl(__tgt_async_info &AsyncInfo,
|
||||
bool ReleaseQueue) override;
|
||||
Error queryAsyncImpl(__tgt_async_info &AsyncInfo) override;
|
||||
Error dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
|
||||
Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
|
||||
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstDev,
|
||||
void *DstPtr, int64_t Size,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
|
||||
Error initAsyncInfoImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) override;
|
||||
Expected<bool>
|
||||
hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) override;
|
||||
|
||||
Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
|
||||
AsyncInfoWrapperTy &AsyncInfo) override {
|
||||
return Plugin::error(ErrorCode::UNIMPLEMENTED,
|
||||
"enqueueHostCallImpl not implemented yet");
|
||||
}
|
||||
|
||||
// Event routines are used to ensure ordering between dataTransfers. Instead
|
||||
// of adding extra events in the queues, we make sure they're ordered by
|
||||
// using the events from the data submission APIs so we don't need to support
|
||||
// these routines.
|
||||
// They still need to report succes to indicate the event are handled
|
||||
// somewhere waitEvent and syncEvent should remain unimplemented.
|
||||
Expected<bool> isEventCompleteImpl(void *EventPtr,
|
||||
AsyncInfoWrapperTy &) override {
|
||||
return true;
|
||||
}
|
||||
|
||||
Error createEventImpl(void **EventPtrStorage) override {
|
||||
return Plugin::success();
|
||||
}
|
||||
Error destroyEventImpl(void *EventPtr) override { return Plugin::success(); }
|
||||
Error recordEventImpl(void *EventPtr,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) override {
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error waitEventImpl(void *EventPtr,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) override {
|
||||
return Plugin::error(error::ErrorCode::UNKNOWN, "%s not implemented yet\n",
|
||||
__func__);
|
||||
}
|
||||
|
||||
Error syncEventImpl(void *EventPtr) override {
|
||||
return Plugin::error(error::ErrorCode::UNKNOWN, "%s not implemented yet\n",
|
||||
__func__);
|
||||
}
|
||||
|
||||
Expected<InfoTreeNode> obtainInfoImpl() override;
|
||||
uint64_t getClockFrequency() const override { return getClockRate(); }
|
||||
uint64_t getHardwareParallelism() const override { return getTotalThreads(); }
|
||||
Error getDeviceMemorySize(uint64_t &DSize) override {
|
||||
DSize = getGlobalMemorySize();
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error getDeviceStackSize(uint64_t &V) override {
|
||||
V = 0;
|
||||
return Plugin::success();
|
||||
}
|
||||
Expected<GenericKernelTy &> constructKernel(const char *Name) override;
|
||||
|
||||
Error setDeviceStackSize(uint64_t V) override { return Plugin::success(); }
|
||||
|
||||
Expected<omp_interop_val_t *>
|
||||
createInterop(int32_t InteropType, interop_spec_t &InteropSpec) override;
|
||||
Error releaseInterop(omp_interop_val_t *Interop) override;
|
||||
|
||||
interop_spec_t selectInteropPreference(int32_t InteropType,
|
||||
int32_t NumPrefers,
|
||||
interop_spec_t *Prefers) override;
|
||||
};
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0DEVICE_H
|
||||
28
offload/plugins-nextgen/level_zero/include/L0Interop.h
Normal file
28
offload/plugins-nextgen/level_zero/include/L0Interop.h
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Interop support for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0INTEROP_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0INTEROP_H
|
||||
|
||||
namespace llvm::omp::target::plugin::L0Interop {
|
||||
|
||||
/// Level Zero interop property.
|
||||
struct Property {
|
||||
// Use this when command queue needs to be accessed as
|
||||
// the targetsync field in interop will be changed if preferred type is sycl.
|
||||
ze_command_queue_handle_t CommandQueue;
|
||||
ze_command_list_handle_t ImmCmdList;
|
||||
};
|
||||
|
||||
} // namespace llvm::omp::target::plugin::L0Interop
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0INTEROP_H
|
||||
150
offload/plugins-nextgen/level_zero/include/L0Kernel.h
Normal file
150
offload/plugins-nextgen/level_zero/include/L0Kernel.h
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// GenericKernel implementation for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0KERNEL_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0KERNEL_H
|
||||
|
||||
#include "AsyncQueue.h"
|
||||
#include "L0Defs.h"
|
||||
#include "L0Trace.h"
|
||||
#include "PluginInterface.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
class L0DeviceTy;
|
||||
class L0ProgramTy;
|
||||
|
||||
/// Loop descriptor.
|
||||
struct TgtLoopDescTy {
|
||||
int64_t Lb = 0; // The lower bound of the i-th loop.
|
||||
int64_t Ub = 0; // The upper bound of the i-th loop.
|
||||
int64_t Stride = 0; // The stride of the i-th loop.
|
||||
|
||||
bool operator==(const TgtLoopDescTy &other) const {
|
||||
return Lb == other.Lb && Ub == other.Ub && Stride == other.Stride;
|
||||
}
|
||||
};
|
||||
|
||||
struct TgtNDRangeDescTy {
|
||||
int32_t NumLoops = 0; // Number of loops/dimensions.
|
||||
int32_t DistributeDim = 0; // Dimensions lower than this one
|
||||
// must end up in one WG.
|
||||
TgtLoopDescTy Levels[3]; // Up to 3 loops.
|
||||
|
||||
bool operator==(const TgtNDRangeDescTy &other) const {
|
||||
return NumLoops == other.NumLoops && DistributeDim == other.DistributeDim &&
|
||||
std::equal(Levels, Levels + 3, other.Levels);
|
||||
}
|
||||
bool operator!=(const TgtNDRangeDescTy &other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
};
|
||||
|
||||
/// Forward declaration.
|
||||
struct L0LaunchEnvTy;
|
||||
|
||||
/// Kernel properties.
|
||||
struct KernelPropertiesTy {
|
||||
uint32_t Width = 0;
|
||||
uint32_t SIMDWidth = 0;
|
||||
uint32_t MaxThreadGroupSize = 0;
|
||||
|
||||
/// Cached input parameters used in the previous launch.
|
||||
int32_t NumTeams = -1;
|
||||
int32_t ThreadLimit = -1;
|
||||
|
||||
/// Cached parameters used in the previous launch.
|
||||
ze_kernel_indirect_access_flags_t IndirectAccessFlags =
|
||||
std::numeric_limits<decltype(IndirectAccessFlags)>::max();
|
||||
uint32_t GroupSizes[3] = {0, 0, 0};
|
||||
ze_group_count_t GroupCounts{0, 0, 0};
|
||||
|
||||
std::mutex Mtx;
|
||||
|
||||
/// Check if we can reuse group parameters.
|
||||
bool reuseGroupParams(const int32_t NumTeamsIn, const int32_t ThreadLimitIn,
|
||||
uint32_t *GroupSizesOut, L0LaunchEnvTy &KEnv) const;
|
||||
|
||||
/// Update cached group parameters.
|
||||
void cacheGroupParams(const int32_t NumTeamsIn, const int32_t ThreadLimitIn,
|
||||
const uint32_t *GroupSizesIn, L0LaunchEnvTy &KEnv);
|
||||
};
|
||||
|
||||
struct L0LaunchEnvTy {
|
||||
bool IsAsync;
|
||||
AsyncQueueTy *AsyncQueue;
|
||||
ze_group_count_t GroupCounts = {0, 0, 0};
|
||||
KernelPropertiesTy &KernelPR;
|
||||
bool HalfNumThreads = false;
|
||||
bool IsTeamsNDRange = false;
|
||||
|
||||
L0LaunchEnvTy(bool IsAsync, AsyncQueueTy *AsyncQueue,
|
||||
KernelPropertiesTy &KernelPR)
|
||||
: IsAsync(IsAsync), AsyncQueue(AsyncQueue), KernelPR(KernelPR) {}
|
||||
};
|
||||
|
||||
class L0KernelTy : public GenericKernelTy {
|
||||
// L0 Kernel Handle.
|
||||
ze_kernel_handle_t zeKernel;
|
||||
// Kernel Properties.
|
||||
mutable KernelPropertiesTy Properties;
|
||||
|
||||
void decideKernelGroupArguments(L0DeviceTy &Device, uint32_t NumTeams,
|
||||
uint32_t ThreadLimit, uint32_t *GroupSizes,
|
||||
L0LaunchEnvTy &KEnv) const;
|
||||
|
||||
Error buildKernel(L0ProgramTy &Program);
|
||||
Error readKernelProperties(L0ProgramTy &Program);
|
||||
|
||||
Error setKernelGroups(L0DeviceTy &l0Device, L0LaunchEnvTy &KEnv,
|
||||
uint32_t NumThreads[3], uint32_t NumBlocks[3]) const;
|
||||
Error setIndirectFlags(L0DeviceTy &l0Device, L0LaunchEnvTy &KEnv) const;
|
||||
|
||||
public:
|
||||
/// Create a L0 kernel with a name and an execution mode.
|
||||
L0KernelTy(const char *Name) : GenericKernelTy(Name), zeKernel(nullptr) {}
|
||||
~L0KernelTy() = default;
|
||||
L0KernelTy(const L0KernelTy &) = delete;
|
||||
L0KernelTy(L0KernelTy &&) = delete;
|
||||
L0KernelTy &operator=(const L0KernelTy &) = delete;
|
||||
L0KernelTy &operator=(const L0KernelTy &&) = delete;
|
||||
|
||||
KernelPropertiesTy &getProperties() const { return Properties; }
|
||||
|
||||
/// Initialize the L0 kernel.
|
||||
Error initImpl(GenericDeviceTy &GenericDevice, DeviceImageTy &Image) override;
|
||||
/// Launch the L0 kernel function.
|
||||
Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
|
||||
uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,
|
||||
KernelLaunchParamsTy LaunchParams,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
|
||||
Error deinit() {
|
||||
CALL_ZE_RET_ERROR(zeKernelDestroy, zeKernel);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Expected<uint64_t> maxGroupSize(GenericDeviceTy &GenericDevice,
|
||||
uint64_t DynamicMemSize) const override {
|
||||
return Plugin::error(ErrorCode::UNIMPLEMENTED,
|
||||
"maxGroupSize not implemented yet");
|
||||
}
|
||||
|
||||
ze_kernel_handle_t getZeKernel() const { return zeKernel; }
|
||||
|
||||
Error getGroupsShape(L0DeviceTy &Device, int32_t NumTeams,
|
||||
int32_t ThreadLimit, uint32_t *GroupSizes,
|
||||
L0LaunchEnvTy &KEnv) const;
|
||||
};
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0KERNEL_H
|
||||
604
offload/plugins-nextgen/level_zero/include/L0Memory.h
Normal file
604
offload/plugins-nextgen/level_zero/include/L0Memory.h
Normal file
|
|
@ -0,0 +1,604 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Memory related support for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0MEMORY_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0MEMORY_H
|
||||
|
||||
#include <cassert>
|
||||
#include <level_zero/ze_api.h>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
#include "L0Defs.h"
|
||||
#include "L0Trace.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
class L0DeviceTy;
|
||||
|
||||
// Forward declarations.
|
||||
struct L0OptionsTy;
|
||||
class L0DeviceTy;
|
||||
class L0ContextTy;
|
||||
|
||||
constexpr static int32_t MaxMemKind = TARGET_ALLOC_LAST + 1;
|
||||
|
||||
struct DynamicMemHeapTy {
|
||||
/// Base address memory is allocated from.
|
||||
uintptr_t AllocBase = 0;
|
||||
/// Minimal size served by the current heap.
|
||||
size_t BlockSize = 0;
|
||||
/// Max size served by the current heap.
|
||||
size_t MaxSize = 0;
|
||||
/// Available memory blocks.
|
||||
uint32_t NumBlocks = 0;
|
||||
/// Number of block descriptors.
|
||||
uint32_t NumBlockDesc = 0;
|
||||
/// Number of block counters.
|
||||
uint32_t NumBlockCounter = 0;
|
||||
/// List of memory block descriptors.
|
||||
uint64_t *BlockDesc = nullptr;
|
||||
/// List of memory block counters.
|
||||
uint32_t *BlockCounter = nullptr;
|
||||
};
|
||||
|
||||
struct DynamicMemPoolTy {
|
||||
/// Location of device memory blocks.
|
||||
void *PoolBase = nullptr;
|
||||
/// Heap size common to all heaps.
|
||||
size_t HeapSize = 0;
|
||||
/// Number of heaps available.
|
||||
uint32_t NumHeaps = 0;
|
||||
/// Heap descriptors (using fixed-size array to simplify memory allocation).
|
||||
DynamicMemHeapTy HeapDesc[8];
|
||||
};
|
||||
|
||||
/// Memory allocation information used in memory allocation/deallocation.
|
||||
struct MemAllocInfoTy {
|
||||
/// Base address allocated from compute runtime.
|
||||
void *Base = nullptr;
|
||||
/// Allocation size known to users/libomptarget.
|
||||
size_t ReqSize = 0;
|
||||
/// Allocation size known to the plugin (can be larger than ReqSize).
|
||||
size_t AllocSize = 0;
|
||||
/// TARGET_ALLOC kind.
|
||||
int32_t Kind = TARGET_ALLOC_DEFAULT;
|
||||
/// Is the allocation from a pool?
|
||||
bool InPool = false;
|
||||
/// Is an implicit argument?
|
||||
bool ImplicitArg = false;
|
||||
|
||||
MemAllocInfoTy() = default;
|
||||
|
||||
MemAllocInfoTy(void *Base, size_t ReqSize, size_t AllocSize, int32_t Kind,
|
||||
bool InPool, bool ImplicitArg)
|
||||
: Base(Base), ReqSize(ReqSize), AllocSize(AllocSize), Kind(Kind),
|
||||
InPool(InPool), ImplicitArg(ImplicitArg) {}
|
||||
};
|
||||
|
||||
/// Responsible for all activities involving memory allocation/deallocation.
|
||||
/// It contains memory pool management, memory allocation bookkeeping.
|
||||
class MemAllocatorTy {
|
||||
|
||||
/// Simple memory allocation statistics. Maintains numbers for pool allocation
|
||||
/// and GPU RT allocation.
|
||||
struct MemStatTy {
|
||||
size_t Requested[2] = {0, 0}; // Requested bytes.
|
||||
size_t Allocated[2] = {0, 0}; // Allocated bytes.
|
||||
size_t Freed[2] = {0, 0}; // Freed bytes.
|
||||
size_t InUse[2] = {0, 0}; // Current memory in use.
|
||||
size_t PeakUse[2] = {0, 0}; // Peak bytes used.
|
||||
size_t NumAllocs[2] = {0, 0}; // Number of allocations.
|
||||
};
|
||||
|
||||
/// Memory pool which enables reuse of already allocated blocks:
|
||||
/// -- Pool maintains a list of buckets each of which can allocate fixed-size
|
||||
/// memory.
|
||||
/// -- Each bucket maintains a list of memory blocks allocated by GPU RT.
|
||||
/// -- Each memory block can allocate multiple fixed-size memory requested by
|
||||
/// offload RT or user.
|
||||
/// -- Memory allocation falls back to GPU RT allocation when the pool size
|
||||
/// (total memory used by pool) reaches a threshold.
|
||||
class MemPoolTy {
|
||||
|
||||
/// Memory block maintained in each bucket.
|
||||
struct BlockTy {
|
||||
/// Base address of this block.
|
||||
uintptr_t Base = 0;
|
||||
/// Size of the block.
|
||||
size_t Size = 0;
|
||||
/// Supported allocation size by this block.
|
||||
size_t ChunkSize = 0;
|
||||
/// Total number of slots.
|
||||
uint32_t NumSlots = 0;
|
||||
/// Maximum slot value.
|
||||
static constexpr uint32_t MaxSlots =
|
||||
std::numeric_limits<decltype(NumSlots)>::max();
|
||||
/// Number of slots in use.
|
||||
uint32_t NumUsedSlots = 0;
|
||||
/// Cached available slot returned by the last dealloc() call.
|
||||
uint32_t FreeSlot = MaxSlots;
|
||||
/// Marker for the currently used slots.
|
||||
std::vector<bool> UsedSlots;
|
||||
|
||||
BlockTy(void *_Base, size_t _Size, size_t _ChunkSize) {
|
||||
Base = reinterpret_cast<uintptr_t>(_Base);
|
||||
Size = _Size;
|
||||
ChunkSize = _ChunkSize;
|
||||
NumSlots = Size / ChunkSize;
|
||||
NumUsedSlots = 0;
|
||||
UsedSlots.resize(NumSlots, /*InitValue=*/false);
|
||||
}
|
||||
|
||||
/// Check if the current block is fully used.
|
||||
bool isFull() const { return NumUsedSlots == NumSlots; }
|
||||
|
||||
/// Check if the given address belongs to the current block.
|
||||
bool contains(void *Mem) const {
|
||||
auto M = reinterpret_cast<uintptr_t>(Mem);
|
||||
return M >= Base && M < Base + Size;
|
||||
}
|
||||
|
||||
/// Allocate a single chunk from the block.
|
||||
void *alloc();
|
||||
|
||||
/// Deallocate the given memory.
|
||||
void dealloc(void *Mem);
|
||||
}; // BlockTy
|
||||
|
||||
/// Allocation kind for the current pool.
|
||||
int32_t AllocKind = TARGET_ALLOC_DEFAULT;
|
||||
/// Access to the allocator.
|
||||
MemAllocatorTy *Allocator = nullptr;
|
||||
/// Minimum supported memory allocation size from pool.
|
||||
size_t AllocMin = 1 << 6; // 64B
|
||||
/// Maximum supported memory allocation size from pool.
|
||||
size_t AllocMax = 0;
|
||||
/// Allocation size when the pool needs to allocate a block.
|
||||
size_t AllocUnit = 1 << 16; // 64KB
|
||||
/// Capacity of each block in the buckets which decides number of
|
||||
/// allocatable chunks from the block. Each block in the bucket can serve
|
||||
/// at least BlockCapacity chunks.
|
||||
/// If ChunkSize * BlockCapacity <= AllocUnit
|
||||
/// BlockSize = AllocUnit
|
||||
/// Otherwise,
|
||||
/// BlockSize = ChunkSize * BlockCapacity
|
||||
/// This simply means how much memory is over-allocated.
|
||||
uint32_t BlockCapacity = 0;
|
||||
/// Total memory allocated from GPU RT for this pool.
|
||||
size_t PoolSize = 0;
|
||||
/// Maximum allowed pool size. Allocation falls back to GPU RT allocation if
|
||||
/// when PoolSize reaches PoolSizeMax.
|
||||
size_t PoolSizeMax = 0;
|
||||
/// Small allocation size allowed in the pool even if pool size is over the
|
||||
/// pool size limit.
|
||||
size_t SmallAllocMax = 1024;
|
||||
/// Small allocation pool size.
|
||||
size_t SmallPoolSize = 0;
|
||||
/// Small allocation pool size max (4MB).
|
||||
size_t SmallPoolSizeMax = (4 << 20);
|
||||
/// List of buckets.
|
||||
std::vector<std::vector<BlockTy *>> Buckets;
|
||||
/// List of bucket parameters.
|
||||
std::vector<std::pair<size_t, size_t>> BucketParams;
|
||||
/// Map from allocated pointer to corresponding block.
|
||||
llvm::DenseMap<void *, BlockTy *> PtrToBlock;
|
||||
/// Simple stats counting miss/hit in each bucket.
|
||||
std::vector<std::pair<uint64_t, uint64_t>> BucketStats;
|
||||
/// Need to zero-initialize after L0 allocation.
|
||||
bool ZeroInit = false;
|
||||
|
||||
/// Get bucket ID from the specified allocation size.
|
||||
uint32_t getBucketId(size_t Size) {
|
||||
uint32_t Count = 0;
|
||||
for (size_t SZ = AllocMin; SZ < Size; Count++)
|
||||
SZ <<= 1;
|
||||
return Count;
|
||||
}
|
||||
|
||||
public:
|
||||
MemPoolTy() = default;
|
||||
MemPoolTy(const MemPoolTy &) = delete;
|
||||
MemPoolTy(MemPoolTy &&) = delete;
|
||||
MemPoolTy &operator=(const MemPoolTy &) = delete;
|
||||
MemPoolTy &operator=(const MemPoolTy &&) = delete;
|
||||
~MemPoolTy() = default;
|
||||
|
||||
void printUsage();
|
||||
|
||||
/// Initialize pool with allocation kind, allocator, and user options.
|
||||
Error init(int32_t Kind, MemAllocatorTy *Allocator,
|
||||
const L0OptionsTy &Option);
|
||||
// Initialize pool used for reduction pool.
|
||||
Error init(MemAllocatorTy *Allocator, const L0OptionsTy &Option);
|
||||
// Initialize pool used for small memory pool with fixed parameters.
|
||||
Error init(MemAllocatorTy *Allocator);
|
||||
|
||||
/// Release resources used in the pool.
|
||||
Error deinit();
|
||||
|
||||
/// Allocate the requested size of memory from this pool.
|
||||
/// AllocSize is the chunk size internally used for the returned memory.
|
||||
Expected<void *> alloc(size_t Size, size_t &AllocSize);
|
||||
/// Deallocate the specified memory and returns block size deallocated.
|
||||
size_t dealloc(void *Ptr);
|
||||
}; // MemPoolTy
|
||||
|
||||
/// Allocation information maintained in the plugin.
|
||||
class MemAllocInfoMapTy {
|
||||
/// Map from allocated pointer to allocation information.
|
||||
std::map<void *, MemAllocInfoTy> Map;
|
||||
/// Map from target alloc kind to number of implicit arguments.
|
||||
std::array<uint32_t, MaxMemKind> NumImplicitArgs;
|
||||
|
||||
public:
|
||||
/// Add allocation information to the map.
|
||||
void add(void *Ptr, void *Base, size_t ReqSize, size_t AllocSize,
|
||||
int32_t Kind, bool InPool = false, bool ImplicitArg = false);
|
||||
|
||||
/// Remove allocation information for the given memory location.
|
||||
bool remove(void *Ptr, MemAllocInfoTy *Removed = nullptr);
|
||||
|
||||
/// Finds allocation information for the given memory location.
|
||||
const MemAllocInfoTy *find(void *Ptr) const {
|
||||
auto AllocInfo = Map.find(Ptr);
|
||||
if (AllocInfo == Map.end())
|
||||
return nullptr;
|
||||
else
|
||||
return &AllocInfo->second;
|
||||
}
|
||||
|
||||
/// Check if the map contains the given pointer and offset.
|
||||
bool contains(const void *Ptr, size_t Size) const {
|
||||
if (Map.size() == 0)
|
||||
return false;
|
||||
auto I = Map.upper_bound(const_cast<void *>(Ptr));
|
||||
if (I == Map.begin())
|
||||
return false;
|
||||
--I;
|
||||
|
||||
uintptr_t PtrAsInt = reinterpret_cast<uintptr_t>(Ptr);
|
||||
uintptr_t MapBase = reinterpret_cast<uintptr_t>(I->first);
|
||||
uintptr_t MapSize = static_cast<uintptr_t>(I->second.ReqSize);
|
||||
|
||||
bool Ret = MapBase <= PtrAsInt && PtrAsInt + Size <= MapBase + MapSize;
|
||||
return Ret;
|
||||
}
|
||||
|
||||
/// Returns the number of implicit arguments for the specified allocation
|
||||
/// kind.
|
||||
size_t getNumImplicitArgs(int32_t Kind) {
|
||||
assert(Kind >= 0 && Kind < MaxMemKind &&
|
||||
"Invalid target allocation kind");
|
||||
return NumImplicitArgs[Kind];
|
||||
}
|
||||
}; // MemAllocInfoMapTy
|
||||
|
||||
/// L0 context to use.
|
||||
const L0ContextTy *L0Context = nullptr;
|
||||
/// L0 device to use.
|
||||
L0DeviceTy *Device = nullptr;
|
||||
/// Whether the device supports large memory allocation.
|
||||
bool SupportsLargeMem = false;
|
||||
/// Cached max alloc size supported by device.
|
||||
uint64_t MaxAllocSize;
|
||||
/// Map from allocation kind to memory statistics.
|
||||
std::array<MemStatTy, MaxMemKind> Stats;
|
||||
/// Map from allocation kind to memory pool.
|
||||
std::array<std::unique_ptr<MemPoolTy>, MaxMemKind> Pools;
|
||||
|
||||
/// Memory pool dedicated to reduction scratch space.
|
||||
std::unique_ptr<MemPoolTy> ReductionPool;
|
||||
/// Memory pool dedicated to reduction counters.
|
||||
std::unique_ptr<MemPoolTy> CounterPool;
|
||||
/// Allocation information map.
|
||||
MemAllocInfoMapTy AllocInfo;
|
||||
/// RTL-owned memory that needs to be freed automatically.
|
||||
std::vector<void *> MemOwned;
|
||||
/// Lock protection.
|
||||
std::mutex Mtx;
|
||||
/// Allocator only supports host memory.
|
||||
bool IsHostMem = false;
|
||||
// Internal deallocation function to be called when already
|
||||
// hondling the Mtx lock.
|
||||
Error deallocLocked(void *Ptr);
|
||||
|
||||
/// Allocate memory from L0 GPU RT.
|
||||
Expected<void *> allocFromL0(size_t Size, size_t Align, int32_t Kind);
|
||||
/// Deallocate memory from L0 GPU RT.
|
||||
Error deallocFromL0(void *Ptr);
|
||||
|
||||
/// We use over-allocation workaround to support target pointer with
|
||||
/// offset, and positive "ActiveSize" is specified in such cases to
|
||||
/// correct debug logging.
|
||||
Expected<void *> allocFromL0AndLog(size_t Size, size_t Align, int32_t Kind,
|
||||
size_t ActiveSize = 0) {
|
||||
auto MemOrErr = allocFromL0(Size, Align, Kind);
|
||||
if (!MemOrErr)
|
||||
return MemOrErr;
|
||||
size_t LoggedSize = ActiveSize ? ActiveSize : Size;
|
||||
log(LoggedSize, Size, Kind);
|
||||
return MemOrErr;
|
||||
}
|
||||
|
||||
/// Log memory allocation/deallocation.
|
||||
void log(size_t ReqSize, size_t Size, int32_t Kind, bool Pool = false) {
|
||||
if (Kind < 0 || Kind >= MaxMemKind)
|
||||
return; // Stat is disabled.
|
||||
|
||||
auto &ST = Stats[Kind];
|
||||
int32_t I = Pool ? 1 : 0;
|
||||
if (ReqSize > 0) {
|
||||
ST.Requested[I] += ReqSize;
|
||||
ST.Allocated[I] += Size;
|
||||
ST.InUse[I] += Size;
|
||||
ST.NumAllocs[I]++;
|
||||
} else {
|
||||
ST.Freed[I] += Size;
|
||||
ST.InUse[I] -= Size;
|
||||
}
|
||||
ST.PeakUse[I] = (std::max)(ST.PeakUse[I], ST.InUse[I]);
|
||||
}
|
||||
|
||||
/// Perform copy operation.
|
||||
Error enqueueMemCopy(void *Dst, const void *Src, size_t Size);
|
||||
/// Perform memory fill operation.
|
||||
Error enqueueMemSet(void *Dst, int8_t Value, size_t Size);
|
||||
|
||||
/// Allocate memory with the specified information from a memory pool.
|
||||
Expected<void *> allocFromPool(size_t Size, size_t Align, int32_t Kind,
|
||||
intptr_t Offset, bool UserAlloc,
|
||||
bool DevMalloc, uint32_t MemAdvice,
|
||||
AllocOptionTy AllocOpt);
|
||||
/// Deallocate memory from memory pool.
|
||||
Error deallocFromPool(void *Ptr) {
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
return deallocLocked(Ptr);
|
||||
}
|
||||
|
||||
public:
|
||||
MemAllocatorTy()
|
||||
: MaxAllocSize(std::numeric_limits<decltype(MaxAllocSize)>::max()) {}
|
||||
|
||||
MemAllocatorTy(const MemAllocatorTy &) = delete;
|
||||
MemAllocatorTy(MemAllocatorTy &&) = delete;
|
||||
MemAllocatorTy &operator=(const MemAllocatorTy &) = delete;
|
||||
MemAllocatorTy &operator=(const MemAllocatorTy &&) = delete;
|
||||
~MemAllocatorTy() = default;
|
||||
|
||||
Error initDevicePools(L0DeviceTy &L0Device, const L0OptionsTy &Option);
|
||||
Error initHostPool(L0ContextTy &Driver, const L0OptionsTy &Option);
|
||||
void updateMaxAllocSize(L0DeviceTy &L0Device);
|
||||
|
||||
/// Release resources and report statistics if requested.
|
||||
Error deinit();
|
||||
|
||||
/// Allocate memory with the specified information from a memory pool.
|
||||
Expected<void *> alloc(size_t Size, size_t Align, int32_t Kind,
|
||||
intptr_t Offset, bool UserAlloc, bool DevMalloc,
|
||||
uint32_t MemAdvice, AllocOptionTy AllocOpt) {
|
||||
return allocFromPool(Size, Align, Kind, Offset, UserAlloc, DevMalloc,
|
||||
MemAdvice, AllocOpt);
|
||||
}
|
||||
|
||||
/// Deallocate memory.
|
||||
Error dealloc(void *Ptr) { return deallocFromPool(Ptr); }
|
||||
|
||||
/// Check if the given memory location and offset belongs to any allocated
|
||||
/// memory.
|
||||
bool contains(const void *Ptr, size_t Size) {
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
return AllocInfo.contains(Ptr, Size);
|
||||
}
|
||||
|
||||
/// Get allocation information for the specified memory location.
|
||||
const MemAllocInfoTy *getAllocInfo(void *Ptr) {
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
return AllocInfo.find(Ptr);
|
||||
}
|
||||
|
||||
/// Get kernel indirect access flags using implicit argument info.
|
||||
ze_kernel_indirect_access_flags_t getIndirectFlags() {
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
ze_kernel_indirect_access_flags_t Ret = 0;
|
||||
if (AllocInfo.getNumImplicitArgs(TARGET_ALLOC_DEVICE) > 0)
|
||||
Ret |= ZE_KERNEL_INDIRECT_ACCESS_FLAG_DEVICE;
|
||||
if (AllocInfo.getNumImplicitArgs(TARGET_ALLOC_HOST) > 0)
|
||||
Ret |= ZE_KERNEL_INDIRECT_ACCESS_FLAG_HOST;
|
||||
if (AllocInfo.getNumImplicitArgs(TARGET_ALLOC_SHARED) > 0)
|
||||
Ret |= ZE_KERNEL_INDIRECT_ACCESS_FLAG_SHARED;
|
||||
return Ret;
|
||||
}
|
||||
}; /// MemAllocatorTy
|
||||
|
||||
// Simple generic wrapper to reuse objects
|
||||
// objects must have zero argument accessible constructor.
|
||||
template <class ObjTy> class ObjPool {
|
||||
// Protection.
|
||||
std::unique_ptr<std::mutex> Mtx;
|
||||
// List of Objects.
|
||||
std::list<ObjTy *> Objects;
|
||||
|
||||
public:
|
||||
ObjPool() { Mtx.reset(new std::mutex); }
|
||||
|
||||
ObjPool(const ObjPool &) = delete;
|
||||
ObjPool(ObjPool &) = delete;
|
||||
ObjPool &operator=(const ObjPool &) = delete;
|
||||
ObjPool &operator=(const ObjPool &&) = delete;
|
||||
|
||||
ObjTy *get() {
|
||||
if (!Objects.empty()) {
|
||||
std::lock_guard<std::mutex> Lock(*Mtx);
|
||||
if (!Objects.empty()) {
|
||||
const auto Ret = Objects.back();
|
||||
Objects.pop_back();
|
||||
return Ret;
|
||||
}
|
||||
}
|
||||
return new ObjTy();
|
||||
}
|
||||
|
||||
void release(ObjTy *obj) {
|
||||
std::lock_guard<std::mutex> Lock(*Mtx);
|
||||
Objects.push_back(obj);
|
||||
}
|
||||
|
||||
~ObjPool() {
|
||||
for (auto Object : Objects)
|
||||
delete Object;
|
||||
}
|
||||
};
|
||||
|
||||
/// Common event pool used in the plugin. This event pool assumes all events
|
||||
/// from the pool are host-visible and use the same event pool flag.
|
||||
class EventPoolTy {
|
||||
/// Size of L0 event pool created on demand.
|
||||
size_t PoolSize = 64;
|
||||
|
||||
/// Context of the events.
|
||||
ze_context_handle_t Context = nullptr;
|
||||
|
||||
/// Additional event pool flags common to this pull.
|
||||
uint32_t Flags = 0;
|
||||
|
||||
/// Protection.
|
||||
std::unique_ptr<std::mutex> Mtx;
|
||||
|
||||
/// List of created L0 event pools.
|
||||
std::list<ze_event_pool_handle_t> Pools;
|
||||
|
||||
/// List of free L0 events.
|
||||
std::list<ze_event_handle_t> Events;
|
||||
|
||||
#ifdef OMPT_SUPPORT
|
||||
/// Event to OMPT record map. The timestamp information is recorded to the
|
||||
/// OMPT record before the event is recycled.
|
||||
std::unordered_map<ze_event_handle_t, ompt_record_ompt_t *> EventToRecord;
|
||||
#endif // OMPT_SUPPORT
|
||||
|
||||
public:
|
||||
/// Initialize context, flags, and mutex.
|
||||
Error init(ze_context_handle_t ContextIn, uint32_t FlagsIn) {
|
||||
Context = ContextIn;
|
||||
Flags = FlagsIn;
|
||||
Mtx.reset(new std::mutex);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
/// Destroys L0 resources.
|
||||
Error deinit() {
|
||||
for (auto E : Events)
|
||||
CALL_ZE_RET_ERROR(zeEventDestroy, E);
|
||||
for (auto P : Pools)
|
||||
CALL_ZE_RET_ERROR(zeEventPoolDestroy, P);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
/// Get a free event from the pool.
|
||||
Expected<ze_event_handle_t> getEvent();
|
||||
|
||||
/// Return an event to the pool.
|
||||
Error releaseEvent(ze_event_handle_t Event, L0DeviceTy &Device);
|
||||
};
|
||||
|
||||
/// Staging buffer.
|
||||
/// A single staging buffer is not enough when batching is enabled since there
|
||||
/// can be multiple pending copy operations.
|
||||
class StagingBufferTy {
|
||||
/// Context for L0 calls.
|
||||
ze_context_handle_t Context = nullptr;
|
||||
/// Max allowed size for staging buffer.
|
||||
size_t Size = L0StagingBufferSize;
|
||||
/// Number of buffers allocated together.
|
||||
size_t Count = L0StagingBufferCount;
|
||||
/// Buffers increasing by Count if a new buffer is required.
|
||||
llvm::SmallVector<void *> Buffers;
|
||||
/// Next buffer location in the buffers.
|
||||
size_t Offset = 0;
|
||||
|
||||
Expected<void *> addBuffers() {
|
||||
ze_host_mem_alloc_desc_t AllocDesc{ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC,
|
||||
nullptr, 0};
|
||||
void *Ret = nullptr;
|
||||
size_t AllocSize = Size * Count;
|
||||
CALL_ZE_RET_ERROR(zeMemAllocHost, Context, &AllocDesc, AllocSize,
|
||||
L0DefaultAlignment, &Ret);
|
||||
Buffers.push_back(Ret);
|
||||
return Ret;
|
||||
}
|
||||
|
||||
public:
|
||||
StagingBufferTy() = default;
|
||||
StagingBufferTy(const StagingBufferTy &) = delete;
|
||||
StagingBufferTy(StagingBufferTy &&) = delete;
|
||||
StagingBufferTy &operator=(const StagingBufferTy &) = delete;
|
||||
StagingBufferTy &operator=(const StagingBufferTy &&) = delete;
|
||||
~StagingBufferTy() = default;
|
||||
|
||||
Error clear() {
|
||||
for (auto Ptr : Buffers)
|
||||
CALL_ZE_RET_ERROR(zeMemFree, Context, Ptr);
|
||||
Context = nullptr;
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
bool initialized() const { return Context != nullptr; }
|
||||
|
||||
void init(ze_context_handle_t ContextIn, size_t SizeIn, size_t CountIn) {
|
||||
Context = ContextIn;
|
||||
Size = SizeIn;
|
||||
Count = CountIn;
|
||||
}
|
||||
|
||||
void reset() { Offset = 0; }
|
||||
|
||||
/// Always return the first buffer.
|
||||
Expected<void *> get() {
|
||||
if (Size == 0 || Count == 0)
|
||||
return nullptr;
|
||||
return Buffers.empty() ? addBuffers() : Buffers.front();
|
||||
}
|
||||
|
||||
/// Return the next available buffer.
|
||||
Expected<void *> getNext() {
|
||||
void *Ret = nullptr;
|
||||
if (Size == 0 || Count == 0)
|
||||
return Ret;
|
||||
|
||||
size_t AllocSize = Size * Count;
|
||||
bool NeedToGrow = Buffers.empty() || Offset >= Buffers.size() * AllocSize;
|
||||
if (NeedToGrow) {
|
||||
auto PtrOrErr = addBuffers();
|
||||
if (!PtrOrErr)
|
||||
return PtrOrErr.takeError();
|
||||
Ret = *PtrOrErr;
|
||||
} else
|
||||
Ret = reinterpret_cast<void *>(
|
||||
reinterpret_cast<uintptr_t>(Buffers.back()) + (Offset % AllocSize));
|
||||
|
||||
if (!Ret)
|
||||
return nullptr;
|
||||
|
||||
Offset += Size;
|
||||
return Ret;
|
||||
}
|
||||
|
||||
/// Return either a fixed buffer or next buffer.
|
||||
Expected<void *> get(bool Next) { return Next ? getNext() : get(); }
|
||||
};
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0MEMORY_H
|
||||
166
offload/plugins-nextgen/level_zero/include/L0Options.h
Normal file
166
offload/plugins-nextgen/level_zero/include/L0Options.h
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Level Zero RTL Options support.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0OPTIONS_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0OPTIONS_H
|
||||
|
||||
#include <level_zero/ze_api.h>
|
||||
|
||||
#include "Shared/EnvironmentVar.h"
|
||||
|
||||
#include "L0Defs.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
/// Command submission mode.
|
||||
enum class CommandModeTy { Sync = 0, Async, AsyncOrdered };
|
||||
|
||||
/// Specialization constants used for a module compilation.
|
||||
class SpecConstantsTy {
|
||||
std::vector<uint32_t> ConstantIds;
|
||||
std::vector<const void *> ConstantValues;
|
||||
BumpPtrAllocator &Allocator;
|
||||
|
||||
public:
|
||||
SpecConstantsTy(BumpPtrAllocator &Allocator) : Allocator(Allocator) {}
|
||||
SpecConstantsTy(const SpecConstantsTy &) = delete;
|
||||
SpecConstantsTy(SpecConstantsTy &&) = delete;
|
||||
SpecConstantsTy &operator=(const SpecConstantsTy &) = delete;
|
||||
SpecConstantsTy &operator=(const SpecConstantsTy &&) = delete;
|
||||
SpecConstantsTy(const SpecConstantsTy &&Other)
|
||||
: ConstantIds(std::move(Other.ConstantIds)),
|
||||
ConstantValues(std::move(Other.ConstantValues)),
|
||||
Allocator(Other.Allocator) {}
|
||||
~SpecConstantsTy() = default;
|
||||
|
||||
template <typename T> void addConstant(uint32_t Id, T Val) {
|
||||
T *ValuePtr =
|
||||
reinterpret_cast<T *>(Allocator.Allocate(sizeof(T), alignof(T)));
|
||||
*ValuePtr = Val;
|
||||
|
||||
ConstantIds.push_back(Id);
|
||||
ConstantValues.push_back(reinterpret_cast<void *>(ValuePtr));
|
||||
}
|
||||
|
||||
ze_module_constants_t getModuleConstants() const {
|
||||
ze_module_constants_t Tmp{static_cast<uint32_t>(ConstantValues.size()),
|
||||
ConstantIds.data(),
|
||||
// Unfortunately we have to const_cast it.
|
||||
// L0 data type should probably be fixed.
|
||||
const_cast<const void **>(ConstantValues.data())};
|
||||
return Tmp;
|
||||
}
|
||||
};
|
||||
|
||||
/// L0 Plugin flags.
|
||||
struct L0OptionFlagsTy {
|
||||
uint64_t UseMemoryPool : 1;
|
||||
uint64_t Reserved : 63;
|
||||
L0OptionFlagsTy() : UseMemoryPool(1), Reserved(0) {}
|
||||
};
|
||||
|
||||
struct L0OptionsTy {
|
||||
/// Binary flags.
|
||||
L0OptionFlagsTy Flags;
|
||||
|
||||
/// Staging buffer size.
|
||||
size_t StagingBufferSize = L0StagingBufferSize;
|
||||
|
||||
/// Staging buffer count.
|
||||
size_t StagingBufferCount = L0StagingBufferCount;
|
||||
|
||||
struct MemPoolConfigTy {
|
||||
bool Use;
|
||||
int32_t AllocMax;
|
||||
int32_t Capacity;
|
||||
int32_t PoolSize;
|
||||
};
|
||||
/// Memory pool default parameters for each allocation kind:
|
||||
/// {UseByDefault, AllocMax(MB), Capacity, PoolSize(MB)}
|
||||
std::array<MemPoolConfigTy, 3> MemPoolConfig{
|
||||
MemPoolConfigTy{true, 1, 4, 256}, // TARGET_ALLOC_DEVICE
|
||||
MemPoolConfigTy{true, 1, 4, 256}, // TARGET_ALLOC_HOST
|
||||
MemPoolConfigTy{true, 8, 4, 256}}; // TARGET_ALLOC_SHARED
|
||||
|
||||
/// Parameters for memory pools dedicated to reduction scratch space.
|
||||
std::array<int32_t, 3> ReductionPoolInfo{256, 8, 8192};
|
||||
|
||||
/// Oversubscription rate for normal kernels.
|
||||
uint32_t SubscriptionRate = 4;
|
||||
|
||||
/// Loop kernels with known ND-range may be known to have
|
||||
/// few iterations and they may not exploit the offload device
|
||||
/// to the fullest extent.
|
||||
/// Let's assume a device has N total HW threads available,
|
||||
/// and the kernel requires M hardware threads with LWS set to L.
|
||||
/// If (M < N * ThinThreadsThreshold), then we will try
|
||||
/// to iteratively divide L by 2 to increase the number of HW
|
||||
/// threads used for executing the kernel. Effectively, we will
|
||||
/// end up with L less than the kernel's SIMD width, so the HW
|
||||
/// threads will not use all their SIMD lanes. This (presumably) should
|
||||
/// allow more parallelism, because the stalls in the SIMD lanes
|
||||
/// will be distributed across more HW threads, and the probability
|
||||
/// of having a stall (or a sequence of stalls) on a critical path
|
||||
/// in the kernel should decrease.
|
||||
/// Anyway, this is just a heuristics that seems to work well for some
|
||||
/// kernels (which poorly expose parallelism in the first place).
|
||||
double ThinThreadsThreshold = 0.1;
|
||||
|
||||
// Compilation options for IGC.
|
||||
// OpenCL 2.0 builtins (like atomic_load_explicit and etc.) are used by
|
||||
// runtime, so we have to explicitly specify the "-cl-std=CL2.0" compilation
|
||||
// option. With it, the SPIR-V will be converted to LLVM IR with OpenCL 2.0
|
||||
// builtins. Otherwise, SPIR-V will be converted to LLVM IR with OpenCL 1.2
|
||||
// builtins.
|
||||
static constexpr std::string_view CompilationOptions = "-cl-std=CL2.0 ";
|
||||
static constexpr std::string_view InternalCompilationOptions =
|
||||
"-cl-take-global-address";
|
||||
std::string UserCompilationOptions;
|
||||
|
||||
/// Spec constants used for all modules.
|
||||
SpecConstantsTy CommonSpecConstants;
|
||||
|
||||
/// Command execution mode.
|
||||
/// Whether the runtime uses asynchronous mode or not depends on the type of
|
||||
/// devices and whether immediate command list is fully enabled.
|
||||
CommandModeTy CommandMode = CommandModeTy::Async;
|
||||
|
||||
/// Controls if we need to reduce available HW threads. We need this
|
||||
/// adjustment on XeHPG when Level Zero debug is enabled
|
||||
/// (ZET_ENABLE_PROGRAM_DEBUGGING=1).
|
||||
bool ZeDebugEnabled = false;
|
||||
|
||||
bool Init = false; // Have the options already been processed.
|
||||
|
||||
// Allocator for long-lived allocations (e.g. spec constants).
|
||||
BumpPtrAllocator Allocator;
|
||||
|
||||
L0OptionsTy() : CommonSpecConstants(Allocator) {}
|
||||
|
||||
/// Read environment variables.
|
||||
void processEnvironmentVars();
|
||||
|
||||
void init() {
|
||||
if (!Init) {
|
||||
processEnvironmentVars();
|
||||
Init = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool match(const StringEnvar &Var, const llvm::StringRef Matched) {
|
||||
return Matched.equals_insensitive(Var.get());
|
||||
}
|
||||
|
||||
}; // L0OptionsTy
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0OPTIONS_H
|
||||
114
offload/plugins-nextgen/level_zero/include/L0Plugin.h
Normal file
114
offload/plugins-nextgen/level_zero/include/L0Plugin.h
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Plugin interface for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0PLUGIN_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0PLUGIN_H
|
||||
|
||||
#include "AsyncQueue.h"
|
||||
#include "L0Defs.h"
|
||||
#include "L0Device.h"
|
||||
#include "L0Memory.h"
|
||||
#include "L0Options.h"
|
||||
#include "L0Program.h"
|
||||
#include "TLS.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
/// Class implementing the LevelZero specific functionalities of the plugin.
|
||||
class LevelZeroPluginTy final : public GenericPluginTy {
|
||||
private:
|
||||
struct DeviceInfoTy {
|
||||
L0DeviceIdTy Id;
|
||||
L0ContextTy *Driver;
|
||||
bool isRoot() const { return Id.SubId < 0 && Id.CCSId < 0; }
|
||||
};
|
||||
llvm::SmallVector<DeviceInfoTy> DetectedDevices;
|
||||
|
||||
/// Context (and Driver) specific data.
|
||||
std::list<L0ContextTy> ContextList;
|
||||
|
||||
// Table containing per-thread information using TLS.
|
||||
L0ThreadTblTy ThreadTLSTable;
|
||||
// Table containing per-thread information for each device using TLS.
|
||||
L0DeviceTLSTableTy DeviceTLSTable;
|
||||
// Table containing per-thread information for each Context using TLS.
|
||||
L0ContextTLSTableTy ContextTLSTable;
|
||||
|
||||
/// L0 plugin global options.
|
||||
static L0OptionsTy Options;
|
||||
|
||||
/// Common pool of AsyncQueue.
|
||||
AsyncQueuePoolTy AsyncQueuePool;
|
||||
|
||||
L0ThreadTLSTy &getTLS() { return ThreadTLSTable.get(); }
|
||||
|
||||
/// Find L0 devices and initialize device properties.
|
||||
/// Returns number of devices reported to omptarget.
|
||||
Expected<int32_t> findDevices();
|
||||
|
||||
public:
|
||||
LevelZeroPluginTy() : GenericPluginTy(getTripleArch()) {}
|
||||
virtual ~LevelZeroPluginTy() = default;
|
||||
|
||||
L0DeviceTLSTy &getDeviceTLS(int32_t DeviceId) {
|
||||
return DeviceTLSTable.get(DeviceId);
|
||||
}
|
||||
L0ContextTLSTy &getContextTLS(ze_context_handle_t Context) {
|
||||
return ContextTLSTable.get(Context);
|
||||
}
|
||||
|
||||
static const L0OptionsTy &getOptions() { return Options; }
|
||||
|
||||
const L0DeviceTy &getDeviceFromId(int32_t DeviceId) const {
|
||||
return static_cast<const L0DeviceTy &>(getDevice(DeviceId));
|
||||
}
|
||||
L0DeviceTy &getDeviceFromId(int32_t DeviceId) {
|
||||
return static_cast<L0DeviceTy &>(getDevice(DeviceId));
|
||||
}
|
||||
|
||||
AsyncQueueTy *getAsyncQueue() {
|
||||
auto *Queue = getTLS().getAsyncQueue();
|
||||
if (!Queue)
|
||||
Queue = AsyncQueuePool.get();
|
||||
return Queue;
|
||||
}
|
||||
|
||||
void releaseAsyncQueue(AsyncQueueTy *Queue) {
|
||||
if (!Queue)
|
||||
return;
|
||||
Queue->reset();
|
||||
if (!getTLS().releaseAsyncQueue(Queue))
|
||||
AsyncQueuePool.release(Queue);
|
||||
}
|
||||
|
||||
// Plugin interface.
|
||||
Expected<int32_t> initImpl() override;
|
||||
Error deinitImpl() override;
|
||||
GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,
|
||||
int32_t NumDevices) override;
|
||||
GenericGlobalHandlerTy *createGlobalHandler() override;
|
||||
|
||||
uint16_t getMagicElfBits() const override { return ELF::EM_INTELGT; }
|
||||
Triple::ArchType getTripleArch() const override { return Triple::spirv64; }
|
||||
const char *getName() const override { return GETNAME(TARGET_NAME); }
|
||||
|
||||
Expected<bool> isELFCompatible(uint32_t DeviceId,
|
||||
StringRef Image) const override;
|
||||
|
||||
Error flushQueueImpl(omp_interop_val_t *Interop) override;
|
||||
Error syncBarrierImpl(omp_interop_val_t *Interop) override;
|
||||
Error asyncBarrierImpl(omp_interop_val_t *Interop) override;
|
||||
};
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0PLUGIN_H
|
||||
135
offload/plugins-nextgen/level_zero/include/L0Program.h
Normal file
135
offload/plugins-nextgen/level_zero/include/L0Program.h
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Level Zero Program abstraction.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0PROGRAM_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0PROGRAM_H
|
||||
|
||||
#include "L0Kernel.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
class L0DeviceTy;
|
||||
|
||||
/// Program data to be initialized by plugin.
|
||||
struct ProgramDataTy {
|
||||
int Initialized = 0;
|
||||
int NumDevices = 0;
|
||||
int DeviceNum = -1;
|
||||
uint32_t TotalEUs = 0;
|
||||
uint32_t HWThreadsPerEU = 0;
|
||||
uintptr_t DynamicMemoryLB = 0;
|
||||
uintptr_t DynamicMemoryUB = 0;
|
||||
int DeviceType = 0;
|
||||
void *DynamicMemPool = nullptr;
|
||||
int TeamsThreadLimit = 0;
|
||||
};
|
||||
|
||||
/// Level Zero program that can contain multiple modules.
|
||||
class L0ProgramTy : public DeviceImageTy {
|
||||
/// Handle multiple modules within a single target image.
|
||||
llvm::SmallVector<ze_module_handle_t> Modules;
|
||||
|
||||
/// Map of kernel names to Modules
|
||||
std::unordered_map<std::string, ze_module_handle_t> KernelsToModuleMap;
|
||||
|
||||
/// List of kernels built for this image.
|
||||
/// We need to delete them ourselves as the main library is not doing
|
||||
/// that right now.
|
||||
std::list<L0KernelTy *> Kernels;
|
||||
|
||||
/// Module that contains global data including device RTL.
|
||||
ze_module_handle_t GlobalModule = nullptr;
|
||||
|
||||
/// Requires module link.
|
||||
bool RequiresModuleLink = false;
|
||||
|
||||
/// Is this module library.
|
||||
bool IsLibModule = false;
|
||||
|
||||
/// Build a single module with the given image, build option, and format.
|
||||
Error addModule(const size_t Size, const uint8_t *Image,
|
||||
const std::string_view BuildOption,
|
||||
ze_module_format_t Format);
|
||||
/// Read file and return the size of the binary if successful.
|
||||
size_t readFile(const char *FileName, std::vector<uint8_t> &OutFile) const;
|
||||
void replaceDriverOptsWithBackendOpts(const L0DeviceTy &Device,
|
||||
std::string &Options) const;
|
||||
|
||||
/// Check if the image should be handled as a library module.
|
||||
void setLibModule();
|
||||
|
||||
L0DeviceTy &getL0Device() const;
|
||||
|
||||
public:
|
||||
L0ProgramTy() = delete;
|
||||
|
||||
L0ProgramTy(int32_t ImageId, GenericDeviceTy &Device,
|
||||
std::unique_ptr<MemoryBuffer> Image)
|
||||
: DeviceImageTy(ImageId, Device, std::move(Image)) {}
|
||||
~L0ProgramTy() = default;
|
||||
L0ProgramTy(const L0ProgramTy &other) = delete;
|
||||
L0ProgramTy(L0ProgramTy &&) = delete;
|
||||
L0ProgramTy &operator=(const L0ProgramTy &) = delete;
|
||||
L0ProgramTy &operator=(const L0ProgramTy &&) = delete;
|
||||
|
||||
Error deinit();
|
||||
|
||||
static L0ProgramTy &makeL0Program(DeviceImageTy &Device) {
|
||||
return static_cast<L0ProgramTy &>(Device);
|
||||
}
|
||||
|
||||
/// Build modules from the target image description.
|
||||
Error buildModules(const std::string_view BuildOptions);
|
||||
|
||||
/// Link modules stored in \p Modules.
|
||||
Error linkModules();
|
||||
|
||||
/// Loads the kernels names from all modules.
|
||||
Error loadModuleKernels();
|
||||
|
||||
/// Read data from the location in the device image which corresponds to the
|
||||
/// specified global variable name.
|
||||
Error readGlobalVariable(const char *Name, size_t Size, void *HostPtr);
|
||||
|
||||
/// Write data to the location in the device image which corresponds to the
|
||||
/// specified global variable name.
|
||||
Error writeGlobalVariable(const char *Name, size_t Size, const void *HostPtr);
|
||||
|
||||
/// Looks up an OpenMP declare target global variable with the given
|
||||
/// \p Name and \p Size in the device environment for the current device.
|
||||
/// The lookup is first done via the device offload table. If it fails,
|
||||
/// then the lookup falls back to non-OpenMP specific lookup on the device.
|
||||
Expected<void *> getOffloadVarDeviceAddr(const char *Name) const;
|
||||
|
||||
/// Returns the handle of a module that contains a given Kernel name.
|
||||
ze_module_handle_t findModuleFromKernelName(const char *KernelName) const {
|
||||
auto K = KernelsToModuleMap.find(std::string(KernelName));
|
||||
if (K == KernelsToModuleMap.end())
|
||||
return nullptr;
|
||||
|
||||
return K->second;
|
||||
}
|
||||
|
||||
void addKernel(L0KernelTy *Kernel) { Kernels.push_back(Kernel); }
|
||||
};
|
||||
|
||||
struct L0GlobalHandlerTy final : public GenericGlobalHandlerTy {
|
||||
Error getGlobalMetadataFromDevice(GenericDeviceTy &Device,
|
||||
DeviceImageTy &Image,
|
||||
GlobalTy &DeviceGlobal) override;
|
||||
};
|
||||
|
||||
bool isValidOneOmpImage(StringRef Image, uint64_t &MajorVer,
|
||||
uint64_t &MinorVer);
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0PROGRAM_H
|
||||
127
offload/plugins-nextgen/level_zero/include/L0Trace.h
Normal file
127
offload/plugins-nextgen/level_zero/include/L0Trace.h
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Code for tracing L0.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
// clang-format off
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0TRACE_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0TRACE_H
|
||||
|
||||
#include "Shared/Debug.h"
|
||||
#include "omptarget.h"
|
||||
#include <string>
|
||||
#include <level_zero/ze_api.h>
|
||||
|
||||
#define CALL_ZE(Rc, Fn, ...) \
|
||||
do { \
|
||||
Rc = Fn(__VA_ARGS__); \
|
||||
} while (0)
|
||||
|
||||
/// For non-thread-safe functions.
|
||||
#define CALL_ZE_RET_MTX(Ret, Fn, Mtx, ...) \
|
||||
do { \
|
||||
Mtx.lock(); \
|
||||
ze_result_t rc; \
|
||||
CALL_ZE(rc, Fn, __VA_ARGS__); \
|
||||
Mtx.unlock(); \
|
||||
if (rc != ZE_RESULT_SUCCESS) { \
|
||||
DP("Error: %s:%s failed with error code %d, %s\n", __func__, #Fn, rc, \
|
||||
getZeErrorName(rc)); \
|
||||
return Ret; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CALL_ZE_RET_ERROR_MTX(Fn, Mtx, ...) \
|
||||
CALL_ZE_RET_MTX( \
|
||||
Plugin::error(ErrorCode::UNKNOWN, "%s failed with error %d, %s", \
|
||||
#Fn, rc, getZeErrorName(rc)), Fn, Mtx, __VA_ARGS__)
|
||||
|
||||
/// For thread-safe functions.
|
||||
#define CALL_ZE_RET(Ret, Fn, ...) \
|
||||
do { \
|
||||
ze_result_t rc; \
|
||||
CALL_ZE(rc, Fn, __VA_ARGS__); \
|
||||
if (rc != ZE_RESULT_SUCCESS) { \
|
||||
DP("Error: %s:%s failed with error code %d, %s\n", __func__, #Fn, rc, \
|
||||
getZeErrorName(rc)); \
|
||||
return Ret; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CALL_ZE_RET_ERROR(Fn, ...) \
|
||||
CALL_ZE_RET( \
|
||||
Plugin::error(ErrorCode::UNKNOWN, "%s failed with error %d, %s", \
|
||||
#Fn, rc, getZeErrorName(rc)), Fn, __VA_ARGS__)
|
||||
|
||||
#define CALL_ZE_EXT_SILENT_RET(Device, Ret, Name, ...) \
|
||||
do { \
|
||||
ze_result_t rc; \
|
||||
CALL_ZE_EXT_SILENT(Device, rc, Name, __VA_ARGS__); \
|
||||
if (rc != ZE_RESULT_SUCCESS) \
|
||||
return Ret; \
|
||||
} while (0)
|
||||
|
||||
#define CALL_ZE_EXT_RET_ERROR(Device, Name, ...) \
|
||||
CALL_ZE_EXT_SILENT_RET(Device, \
|
||||
Plugin::error(ErrorCode::UNKNOWN, "%s failed with code %d, %s", \
|
||||
#Name, rc, getZeErrorName(rc)), Name, __VA_ARGS__)
|
||||
|
||||
#define FOREACH_ZE_ERROR_CODE(Fn) \
|
||||
Fn(ZE_RESULT_SUCCESS) \
|
||||
Fn(ZE_RESULT_NOT_READY) \
|
||||
Fn(ZE_RESULT_ERROR_DEVICE_LOST) \
|
||||
Fn(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY) \
|
||||
Fn(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY) \
|
||||
Fn(ZE_RESULT_ERROR_MODULE_BUILD_FAILURE) \
|
||||
Fn(ZE_RESULT_ERROR_MODULE_LINK_FAILURE) \
|
||||
Fn(ZE_RESULT_ERROR_DEVICE_REQUIRES_RESET) \
|
||||
Fn(ZE_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE) \
|
||||
Fn(ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS) \
|
||||
Fn(ZE_RESULT_ERROR_NOT_AVAILABLE) \
|
||||
Fn(ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE) \
|
||||
Fn(ZE_RESULT_WARNING_DROPPED_DATA) \
|
||||
Fn(ZE_RESULT_ERROR_UNINITIALIZED) \
|
||||
Fn(ZE_RESULT_ERROR_UNSUPPORTED_VERSION) \
|
||||
Fn(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_ARGUMENT) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_NULL_HANDLE) \
|
||||
Fn(ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_NULL_POINTER) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_SIZE) \
|
||||
Fn(ZE_RESULT_ERROR_UNSUPPORTED_SIZE) \
|
||||
Fn(ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_ENUMERATION) \
|
||||
Fn(ZE_RESULT_ERROR_UNSUPPORTED_ENUMERATION) \
|
||||
Fn(ZE_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_NATIVE_BINARY) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_GLOBAL_NAME) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_KERNEL_NAME) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_FUNCTION_NAME) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_MODULE_UNLINKED) \
|
||||
Fn(ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE) \
|
||||
Fn(ZE_RESULT_ERROR_OVERLAPPING_REGIONS) \
|
||||
Fn(ZE_RESULT_WARNING_ACTION_REQUIRED) \
|
||||
Fn(ZE_RESULT_ERROR_UNKNOWN)
|
||||
|
||||
#define CASE_TO_STRING(Num) case Num: return #Num;
|
||||
inline const char *getZeErrorName(int32_t Error) {
|
||||
switch (Error) {
|
||||
FOREACH_ZE_ERROR_CODE(CASE_TO_STRING)
|
||||
default:
|
||||
return "ZE_RESULT_ERROR_UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_L0TRACE_H
|
||||
71
offload/plugins-nextgen/level_zero/include/TLS.h
Normal file
71
offload/plugins-nextgen/level_zero/include/TLS.h
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Thread Level Storage abstraction.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_TLS_H
|
||||
#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_TLS_H
|
||||
|
||||
#include <bitset>
|
||||
|
||||
#include "AsyncQueue.h"
|
||||
#include "L0Memory.h"
|
||||
#include "L0Trace.h"
|
||||
#include "PerThreadTable.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
/// All thread-local data used by the Plugin.
|
||||
class L0ThreadTLSTy {
|
||||
/// Async info tracking.
|
||||
static constexpr int32_t PerThreadQueues = 10;
|
||||
std::bitset<PerThreadQueues> InUseQueues{0};
|
||||
AsyncQueueTy AsyncQueues[PerThreadQueues];
|
||||
|
||||
public:
|
||||
L0ThreadTLSTy() = default;
|
||||
L0ThreadTLSTy(const L0ThreadTLSTy &) = delete;
|
||||
L0ThreadTLSTy(L0ThreadTLSTy &&) = delete;
|
||||
L0ThreadTLSTy &operator=(const L0ThreadTLSTy &) = delete;
|
||||
L0ThreadTLSTy &operator=(const L0ThreadTLSTy &&) = delete;
|
||||
~L0ThreadTLSTy() = default;
|
||||
|
||||
AsyncQueueTy *getAsyncQueue() {
|
||||
AsyncQueueTy *Ret = nullptr;
|
||||
if (!InUseQueues.all()) {
|
||||
// there's a free queue in this thread, find it.
|
||||
for (size_t Queue = 0; Queue < PerThreadQueues; Queue++) {
|
||||
if (!InUseQueues.test(Queue)) {
|
||||
InUseQueues.set(Queue);
|
||||
Ret = &AsyncQueues[Queue];
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert(Ret && "A queue should have been found!");
|
||||
}
|
||||
return Ret;
|
||||
}
|
||||
|
||||
bool releaseAsyncQueue(AsyncQueueTy *Queue) {
|
||||
if (Queue >= &AsyncQueues[0] && Queue < &AsyncQueues[PerThreadQueues]) {
|
||||
// it's a local queue
|
||||
size_t QueueId = Queue - &AsyncQueues[0];
|
||||
InUseQueues.reset(QueueId);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
using L0ThreadTblTy = PerThread<L0ThreadTLSTy>;
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_LEVEL_ZERO_TLS_H
|
||||
51
offload/plugins-nextgen/level_zero/src/L0Context.cpp
Normal file
51
offload/plugins-nextgen/level_zero/src/L0Context.cpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Level Zero Context abstraction.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "L0Context.h"
|
||||
#include "L0Plugin.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
Error L0ContextTy::init() {
|
||||
CALL_ZE_RET_ERROR(zeDriverGetApiVersion, zeDriver, &APIVersion);
|
||||
DP("Driver API version is %" PRIx32 "\n", APIVersion);
|
||||
|
||||
ze_context_desc_t Desc{ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
|
||||
CALL_ZE_RET_ERROR(zeContextCreate, zeDriver, &Desc, &zeContext);
|
||||
if (auto Err = EventPool.init(zeContext, 0))
|
||||
return Err;
|
||||
if (auto Err = HostMemAllocator.initHostPool(*this, Plugin.getOptions()))
|
||||
return Err;
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0ContextTy::deinit() {
|
||||
if (auto Err = EventPool.deinit())
|
||||
return Err;
|
||||
if (auto Err = HostMemAllocator.deinit())
|
||||
return Err;
|
||||
if (zeContext)
|
||||
CALL_ZE_RET_ERROR(zeContextDestroy, zeContext);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
StagingBufferTy &L0ContextTy::getStagingBuffer() {
|
||||
auto &TLS = Plugin.getContextTLS(getZeContext());
|
||||
auto &Buffer = TLS.getStagingBuffer();
|
||||
const auto &Options = Plugin.getOptions();
|
||||
if (!Buffer.initialized())
|
||||
Buffer.init(getZeContext(), Options.StagingBufferSize,
|
||||
Options.StagingBufferCount);
|
||||
return Buffer;
|
||||
}
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
1098
offload/plugins-nextgen/level_zero/src/L0Device.cpp
Normal file
1098
offload/plugins-nextgen/level_zero/src/L0Device.cpp
Normal file
File diff suppressed because it is too large
Load diff
137
offload/plugins-nextgen/level_zero/src/L0DynWrapper.cpp
Normal file
137
offload/plugins-nextgen/level_zero/src/L0DynWrapper.cpp
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
//===--- level_zero/dynamic_level_zero/level_zero.cpp ------------- C++ -*-===//
|
||||
//
|
||||
// Implement wrapper for level_zero API calls through dlopen
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <level_zero/ze_api.h>
|
||||
#include <level_zero/zes_api.h>
|
||||
#include <memory>
|
||||
|
||||
#include "DLWrap.h"
|
||||
#include "Shared/Debug.h"
|
||||
#include "llvm/Support/DynamicLibrary.h"
|
||||
|
||||
DLWRAP_INITIALIZE()
|
||||
|
||||
DLWRAP_INTERNAL(zeInit, 1)
|
||||
DLWRAP(zeDriverGet, 2)
|
||||
DLWRAP(zeDeviceGet, 3)
|
||||
DLWRAP(zeDeviceGetSubDevices, 3)
|
||||
DLWRAP(zeModuleCreate, 5)
|
||||
DLWRAP(zeModuleGetProperties, 2)
|
||||
DLWRAP(zeModuleBuildLogDestroy, 1)
|
||||
DLWRAP(zeModuleBuildLogGetString, 3)
|
||||
DLWRAP(zeModuleGetKernelNames, 3)
|
||||
DLWRAP(zeModuleDestroy, 1)
|
||||
DLWRAP(zeCommandListAppendBarrier, 4)
|
||||
DLWRAP(zeCommandListAppendLaunchKernel, 6)
|
||||
DLWRAP(zeCommandListAppendLaunchCooperativeKernel, 6)
|
||||
DLWRAP(zeCommandListAppendMemoryCopy, 7)
|
||||
DLWRAP(zeCommandListAppendMemoryCopyRegion, 12)
|
||||
DLWRAP(zeCommandListAppendMemoryFill, 8)
|
||||
DLWRAP(zeCommandListAppendMemoryPrefetch, 3)
|
||||
DLWRAP(zeCommandListAppendMemAdvise, 5)
|
||||
DLWRAP(zeCommandListClose, 1)
|
||||
DLWRAP(zeCommandListCreate, 4)
|
||||
DLWRAP(zeCommandListCreateImmediate, 4)
|
||||
DLWRAP(zeCommandListDestroy, 1)
|
||||
DLWRAP(zeCommandListReset, 1)
|
||||
DLWRAP(zeCommandQueueCreate, 4)
|
||||
DLWRAP(zeCommandQueueDestroy, 1)
|
||||
DLWRAP(zeCommandQueueExecuteCommandLists, 4)
|
||||
DLWRAP(zeCommandQueueSynchronize, 2)
|
||||
DLWRAP(zeContextCreate, 3)
|
||||
DLWRAP(zeContextDestroy, 1)
|
||||
DLWRAP(zeContextMakeMemoryResident, 4)
|
||||
DLWRAP(zeDeviceCanAccessPeer, 3)
|
||||
DLWRAP(zeDeviceGetProperties, 2)
|
||||
DLWRAP(zeDeviceGetCommandQueueGroupProperties, 3)
|
||||
DLWRAP(zeDeviceGetComputeProperties, 2)
|
||||
DLWRAP(zeDeviceGetMemoryProperties, 3)
|
||||
DLWRAP(zeDeviceGetCacheProperties, 3)
|
||||
DLWRAP(zeDeviceGetGlobalTimestamps, 3)
|
||||
DLWRAP(zeDriverGetApiVersion, 2)
|
||||
DLWRAP(zeDriverGetExtensionFunctionAddress, 3)
|
||||
DLWRAP(zeDriverGetExtensionProperties, 3)
|
||||
DLWRAP(zeEventCreate, 3)
|
||||
DLWRAP(zeEventDestroy, 1)
|
||||
DLWRAP(zeEventHostReset, 1)
|
||||
DLWRAP(zeEventHostSynchronize, 2)
|
||||
DLWRAP(zeEventPoolCreate, 5)
|
||||
DLWRAP(zeEventPoolDestroy, 1)
|
||||
DLWRAP(zeEventQueryKernelTimestamp, 2)
|
||||
DLWRAP(zeFenceCreate, 3)
|
||||
DLWRAP(zeFenceDestroy, 1)
|
||||
DLWRAP(zeFenceHostSynchronize, 2)
|
||||
DLWRAP(zeKernelCreate, 3)
|
||||
DLWRAP(zeKernelDestroy, 1)
|
||||
DLWRAP(zeKernelGetName, 3)
|
||||
DLWRAP(zeKernelGetProperties, 2)
|
||||
DLWRAP(zeKernelSetArgumentValue, 4)
|
||||
DLWRAP(zeKernelSetGroupSize, 4)
|
||||
DLWRAP(zeKernelSetIndirectAccess, 2)
|
||||
DLWRAP(zeKernelSuggestGroupSize, 7)
|
||||
DLWRAP(zeKernelSuggestMaxCooperativeGroupCount, 2)
|
||||
DLWRAP(zeMemAllocDevice, 6)
|
||||
DLWRAP(zeMemAllocHost, 5)
|
||||
DLWRAP(zeMemAllocShared, 7)
|
||||
DLWRAP(zeMemFree, 2)
|
||||
DLWRAP(zeMemGetAddressRange, 4)
|
||||
DLWRAP(zeMemGetAllocProperties, 4)
|
||||
DLWRAP(zeModuleDynamicLink, 3)
|
||||
DLWRAP(zeModuleGetGlobalPointer, 4)
|
||||
DLWRAP(zesDeviceEnumMemoryModules, 3)
|
||||
DLWRAP(zesMemoryGetState, 2)
|
||||
DLWRAP(zeCommandListHostSynchronize, 2)
|
||||
|
||||
DLWRAP_FINALIZE()
|
||||
|
||||
#ifdef _WIN32
|
||||
#define LEVEL_ZERO_LIBRARY "ze_loader.dll"
|
||||
#else
|
||||
#define LEVEL_ZERO_LIBRARY "libze_loader.so"
|
||||
#endif // _WIN32
|
||||
|
||||
#ifndef TARGET_NAME
|
||||
#error "Missing TARGET_NAME macro"
|
||||
#endif
|
||||
#ifndef DEBUG_PREFIX
|
||||
#define DEBUG_PREFIX "TARGET " GETNAME(TARGET_NAME) " RTL"
|
||||
#endif
|
||||
|
||||
static bool loadLevelZero() {
|
||||
const char *L0Library = LEVEL_ZERO_LIBRARY;
|
||||
std::string ErrMsg;
|
||||
|
||||
DP("Trying to load %s\n", L0Library);
|
||||
auto DynlibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
|
||||
llvm::sys::DynamicLibrary::getPermanentLibrary(L0Library, &ErrMsg));
|
||||
if (!DynlibHandle->isValid()) {
|
||||
if (ErrMsg.empty())
|
||||
ErrMsg = "unknown error";
|
||||
DP("Unable to load library '%s': %s!\n", L0Library, ErrMsg.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
for (size_t I = 0; I < dlwrap::size(); I++) {
|
||||
const char *Sym = dlwrap::symbol(I);
|
||||
|
||||
void *P = DynlibHandle->getAddressOfSymbol(Sym);
|
||||
if (P == nullptr) {
|
||||
DP("Unable to find '%s' in '%s'!\n", Sym, L0Library);
|
||||
return false;
|
||||
}
|
||||
DP("Implementing %s with dlsym(%s) -> %p\n", Sym, Sym, P);
|
||||
|
||||
*dlwrap::pointer(I) = P;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
ze_result_t ZE_APICALL zeInit(ze_init_flags_t flags) {
|
||||
if (!loadLevelZero())
|
||||
return ZE_RESULT_ERROR_UNKNOWN;
|
||||
return dlwrap_zeInit(flags);
|
||||
}
|
||||
474
offload/plugins-nextgen/level_zero/src/L0Kernel.cpp
Normal file
474
offload/plugins-nextgen/level_zero/src/L0Kernel.cpp
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// GenericKernel implementation for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "L0Kernel.h"
|
||||
#include "L0Device.h"
|
||||
#include "L0Plugin.h"
|
||||
#include "L0Program.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
bool KernelPropertiesTy::reuseGroupParams(const int32_t NumTeamsIn,
|
||||
const int32_t ThreadLimitIn,
|
||||
uint32_t *GroupSizesOut,
|
||||
L0LaunchEnvTy &KEnv) const {
|
||||
if (NumTeamsIn != NumTeams || ThreadLimitIn != ThreadLimit)
|
||||
return false;
|
||||
// Found matching input parameters.
|
||||
std::copy_n(GroupSizes, 3, GroupSizesOut);
|
||||
KEnv.GroupCounts = GroupCounts;
|
||||
return true;
|
||||
}
|
||||
|
||||
void KernelPropertiesTy::cacheGroupParams(const int32_t NumTeamsIn,
|
||||
const int32_t ThreadLimitIn,
|
||||
const uint32_t *GroupSizesIn,
|
||||
L0LaunchEnvTy &KEnv) {
|
||||
NumTeams = NumTeamsIn;
|
||||
ThreadLimit = ThreadLimitIn;
|
||||
std::copy_n(GroupSizesIn, 3, GroupSizes);
|
||||
GroupCounts = KEnv.GroupCounts;
|
||||
}
|
||||
|
||||
Error L0KernelTy::readKernelProperties(L0ProgramTy &Program) {
|
||||
const auto &l0Device = L0DeviceTy::makeL0Device(Program.getDevice());
|
||||
auto &KernelPR = getProperties();
|
||||
ze_kernel_properties_t KP = {};
|
||||
KP.stype = ZE_STRUCTURE_TYPE_KERNEL_PROPERTIES;
|
||||
KP.pNext = nullptr;
|
||||
ze_kernel_preferred_group_size_properties_t KPrefGRPSize = {};
|
||||
KPrefGRPSize.stype = ZE_STRUCTURE_TYPE_KERNEL_PREFERRED_GROUP_SIZE_PROPERTIES;
|
||||
KPrefGRPSize.pNext = nullptr;
|
||||
if (l0Device.getDriverAPIVersion() >= ZE_API_VERSION_1_2)
|
||||
KP.pNext = &KPrefGRPSize;
|
||||
|
||||
CALL_ZE_RET_ERROR(zeKernelGetProperties, zeKernel, &KP);
|
||||
KernelPR.SIMDWidth = KP.maxSubgroupSize;
|
||||
KernelPR.Width = KP.maxSubgroupSize;
|
||||
|
||||
if (KP.pNext)
|
||||
KernelPR.Width = KPrefGRPSize.preferredMultiple;
|
||||
|
||||
if (!l0Device.isDeviceArch(DeviceArchTy::DeviceArch_Gen)) {
|
||||
KernelPR.Width = (std::max)(KernelPR.Width, 2 * KernelPR.SIMDWidth);
|
||||
}
|
||||
KernelPR.MaxThreadGroupSize = KP.maxSubgroupSize * KP.maxNumSubgroups;
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0KernelTy::buildKernel(L0ProgramTy &Program) {
|
||||
const auto *KernelName = getName();
|
||||
|
||||
auto Module = Program.findModuleFromKernelName(KernelName);
|
||||
ze_kernel_desc_t KernelDesc = {ZE_STRUCTURE_TYPE_KERNEL_DESC, nullptr, 0,
|
||||
KernelName};
|
||||
CALL_ZE_RET_ERROR(zeKernelCreate, Module, &KernelDesc, &zeKernel);
|
||||
if (auto Err = readKernelProperties(Program))
|
||||
return Err;
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0KernelTy::initImpl(GenericDeviceTy &GenericDevice,
|
||||
DeviceImageTy &Image) {
|
||||
auto &Program = L0ProgramTy::makeL0Program(Image);
|
||||
|
||||
if (auto Err = buildKernel(Program))
|
||||
return Err;
|
||||
Program.addKernel(this);
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
void L0KernelTy::decideKernelGroupArguments(L0DeviceTy &Device,
|
||||
uint32_t NumTeams,
|
||||
uint32_t ThreadLimit,
|
||||
uint32_t *GroupSizes,
|
||||
L0LaunchEnvTy &KEnv) const {
|
||||
|
||||
const KernelPropertiesTy &KernelPR = getProperties();
|
||||
|
||||
const auto DeviceId = Device.getDeviceId();
|
||||
bool MaxGroupSizeForced = false;
|
||||
bool MaxGroupCountForced = false;
|
||||
uint32_t MaxGroupSize = Device.getMaxGroupSize();
|
||||
const auto &Option = LevelZeroPluginTy::getOptions();
|
||||
const auto OptSubscRate = Option.SubscriptionRate;
|
||||
auto &GroupCounts = KEnv.GroupCounts;
|
||||
|
||||
uint32_t SIMDWidth = KernelPR.SIMDWidth;
|
||||
uint32_t KernelWidth = KernelPR.Width;
|
||||
uint32_t KernelMaxThreadGroupSize = KernelPR.MaxThreadGroupSize;
|
||||
|
||||
if (KernelMaxThreadGroupSize < MaxGroupSize) {
|
||||
MaxGroupSize = KernelMaxThreadGroupSize;
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Capping maximum team size to %" PRIu32
|
||||
" due to kernel constraints.\n",
|
||||
MaxGroupSize);
|
||||
}
|
||||
|
||||
if (ThreadLimit > 0) {
|
||||
MaxGroupSizeForced = true;
|
||||
MaxGroupSize = ThreadLimit;
|
||||
}
|
||||
|
||||
uint32_t MaxGroupCount = 0;
|
||||
if (NumTeams > 0) {
|
||||
MaxGroupCount = NumTeams;
|
||||
MaxGroupCountForced = true;
|
||||
}
|
||||
|
||||
if (MaxGroupCountForced) {
|
||||
// If number of teams is specified by the user, then use KernelWidth.
|
||||
// WIs per WG by default, so that it matches
|
||||
// decideLoopKernelGroupArguments() behavior.
|
||||
if (!MaxGroupSizeForced) {
|
||||
MaxGroupSize = KernelWidth;
|
||||
}
|
||||
} else {
|
||||
const uint32_t NumSubslices = Device.getNumSubslices();
|
||||
uint32_t NumThreadsPerSubslice = Device.getNumThreadsPerSubslice();
|
||||
if (KEnv.HalfNumThreads)
|
||||
NumThreadsPerSubslice /= 2;
|
||||
|
||||
MaxGroupCount = NumSubslices * NumThreadsPerSubslice;
|
||||
if (MaxGroupSizeForced) {
|
||||
// Set group size for the HW capacity.
|
||||
uint32_t NumThreadsPerGroup = (MaxGroupSize + SIMDWidth - 1) / SIMDWidth;
|
||||
uint32_t NumGroupsPerSubslice =
|
||||
(NumThreadsPerSubslice + NumThreadsPerGroup - 1) / NumThreadsPerGroup;
|
||||
MaxGroupCount = NumGroupsPerSubslice * NumSubslices;
|
||||
} else {
|
||||
assert(!MaxGroupSizeForced && !MaxGroupCountForced);
|
||||
assert((MaxGroupSize <= KernelWidth || MaxGroupSize % KernelWidth == 0) &&
|
||||
"Invalid maxGroupSize");
|
||||
// Maximize group size.
|
||||
while (MaxGroupSize >= KernelWidth) {
|
||||
uint32_t NumThreadsPerGroup =
|
||||
(MaxGroupSize + SIMDWidth - 1) / SIMDWidth;
|
||||
|
||||
if (NumThreadsPerSubslice % NumThreadsPerGroup == 0) {
|
||||
uint32_t NumGroupsPerSubslice =
|
||||
NumThreadsPerSubslice / NumThreadsPerGroup;
|
||||
MaxGroupCount = NumGroupsPerSubslice * NumSubslices;
|
||||
break;
|
||||
}
|
||||
MaxGroupSize -= KernelWidth;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t GRPCounts[3] = {MaxGroupCount, 1, 1};
|
||||
uint32_t GRPSizes[3] = {MaxGroupSize, 1, 1};
|
||||
if (!MaxGroupCountForced) {
|
||||
GRPCounts[0] *= OptSubscRate;
|
||||
}
|
||||
GroupCounts.groupCountX = GRPCounts[0];
|
||||
GroupCounts.groupCountY = GRPCounts[1];
|
||||
GroupCounts.groupCountZ = GRPCounts[2];
|
||||
std::copy(GRPSizes, GRPSizes + 3, GroupSizes);
|
||||
}
|
||||
|
||||
Error L0KernelTy::getGroupsShape(L0DeviceTy &Device, int32_t NumTeams,
|
||||
int32_t ThreadLimit, uint32_t *GroupSizes,
|
||||
L0LaunchEnvTy &KEnv) const {
|
||||
|
||||
const auto DeviceId = Device.getDeviceId();
|
||||
const auto &KernelPR = getProperties();
|
||||
|
||||
// Read the most recent global thread limit and max teams.
|
||||
const int32_t NumTeamsICV = 0;
|
||||
const int32_t ThreadLimitICV = 0;
|
||||
|
||||
bool IsXeHPG = Device.isDeviceArch(DeviceArchTy::DeviceArch_XeHPG);
|
||||
KEnv.HalfNumThreads =
|
||||
LevelZeroPluginTy::getOptions().ZeDebugEnabled && IsXeHPG;
|
||||
uint32_t KernelWidth = KernelPR.Width;
|
||||
uint32_t SIMDWidth = KernelPR.SIMDWidth;
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Assumed kernel SIMD width is %" PRIu32 "\n", SIMDWidth);
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Preferred team size is multiple of %" PRIu32 "\n", KernelWidth);
|
||||
assert(SIMDWidth <= KernelWidth && "Invalid SIMD width.");
|
||||
|
||||
if (ThreadLimit > 0) {
|
||||
// use thread_limit clause value default.
|
||||
DP("Max team size is set to %" PRId32 " (thread_limit clause)\n",
|
||||
ThreadLimit);
|
||||
} else if (ThreadLimitICV > 0) {
|
||||
// else use thread-limit-var ICV.
|
||||
ThreadLimit = ThreadLimitICV;
|
||||
DP("Max team size is set to %" PRId32 " (thread-limit-icv)\n", ThreadLimit);
|
||||
}
|
||||
|
||||
size_t MaxThreadLimit = Device.getMaxGroupSize();
|
||||
// Set correct max group size if the kernel was compiled with explicit SIMD.
|
||||
if (SIMDWidth == 1)
|
||||
MaxThreadLimit = Device.getNumThreadsPerSubslice();
|
||||
|
||||
if (KernelPR.MaxThreadGroupSize < MaxThreadLimit) {
|
||||
MaxThreadLimit = KernelPR.MaxThreadGroupSize;
|
||||
DP("Capping maximum team size to %zu due to kernel constraints.\n",
|
||||
MaxThreadLimit);
|
||||
}
|
||||
|
||||
if (ThreadLimit > static_cast<int32_t>(MaxThreadLimit)) {
|
||||
ThreadLimit = MaxThreadLimit;
|
||||
DP("Max team size execceds current maximum %zu. Adjusted\n",
|
||||
MaxThreadLimit);
|
||||
}
|
||||
// scope code to ease integration with downstream custom code.
|
||||
{
|
||||
if (NumTeams > 0) {
|
||||
DP("Number of teams is set to %" PRId32
|
||||
"(num_teams clause or no teams construct)\n",
|
||||
NumTeams);
|
||||
} else if (NumTeamsICV > 0) {
|
||||
// OMP_NUM_TEAMS only matters, if num_teams() clause is absent.
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"OMP_NUM_TEAMS(%" PRId32 ") is ignored\n", NumTeamsICV);
|
||||
|
||||
NumTeams = NumTeamsICV;
|
||||
DP("Max number of teams is set to %" PRId32 " (OMP_NUM_TEAMS)\n",
|
||||
NumTeams);
|
||||
}
|
||||
|
||||
decideKernelGroupArguments(Device, (uint32_t)NumTeams,
|
||||
(uint32_t)ThreadLimit, GroupSizes, KEnv);
|
||||
}
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
static Error launchKernelWithImmCmdList(L0DeviceTy &l0Device,
|
||||
ze_kernel_handle_t zeKernel,
|
||||
L0LaunchEnvTy &KEnv,
|
||||
CommandModeTy CommandMode) {
|
||||
const auto DeviceId = l0Device.getDeviceId();
|
||||
auto *IdStr = l0Device.getZeIdCStr();
|
||||
auto CmdListOrErr = l0Device.getImmCmdList();
|
||||
if (!CmdListOrErr)
|
||||
return CmdListOrErr.takeError();
|
||||
const ze_command_list_handle_t CmdList = *CmdListOrErr;
|
||||
// Command queue is not used with immediate command list.
|
||||
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Using immediate command list for kernel submission.\n");
|
||||
auto EventOrError = l0Device.getEvent();
|
||||
if (!EventOrError)
|
||||
return EventOrError.takeError();
|
||||
ze_event_handle_t Event = *EventOrError;
|
||||
size_t NumWaitEvents = 0;
|
||||
ze_event_handle_t *WaitEvents = nullptr;
|
||||
auto *AsyncQueue = KEnv.AsyncQueue;
|
||||
if (KEnv.IsAsync && !AsyncQueue->WaitEvents.empty()) {
|
||||
if (CommandMode == CommandModeTy::AsyncOrdered) {
|
||||
NumWaitEvents = 1;
|
||||
WaitEvents = &AsyncQueue->WaitEvents.back();
|
||||
} else {
|
||||
NumWaitEvents = AsyncQueue->WaitEvents.size();
|
||||
WaitEvents = AsyncQueue->WaitEvents.data();
|
||||
}
|
||||
}
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Kernel depends on %zu data copying events.\n", NumWaitEvents);
|
||||
CALL_ZE_RET_ERROR(zeCommandListAppendLaunchKernel, CmdList, zeKernel,
|
||||
&KEnv.GroupCounts, Event, NumWaitEvents, WaitEvents);
|
||||
KEnv.KernelPR.Mtx.unlock();
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Submitted kernel " DPxMOD " to device %s\n", DPxPTR(zeKernel), IdStr);
|
||||
|
||||
if (KEnv.IsAsync) {
|
||||
AsyncQueue->WaitEvents.push_back(Event);
|
||||
AsyncQueue->KernelEvent = Event;
|
||||
} else {
|
||||
CALL_ZE_RET_ERROR(zeEventHostSynchronize, Event, L0DefaultTimeout);
|
||||
if (auto Err = l0Device.releaseEvent(Event))
|
||||
return Err;
|
||||
}
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Executed kernel entry " DPxMOD " on device %s\n", DPxPTR(zeKernel),
|
||||
IdStr);
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
static Error launchKernelWithCmdQueue(L0DeviceTy &l0Device,
|
||||
ze_kernel_handle_t zeKernel,
|
||||
L0LaunchEnvTy &KEnv) {
|
||||
const auto DeviceId = l0Device.getDeviceId();
|
||||
const auto *IdStr = l0Device.getZeIdCStr();
|
||||
|
||||
auto CmdListOrErr = l0Device.getCmdList();
|
||||
if (!CmdListOrErr)
|
||||
return CmdListOrErr.takeError();
|
||||
ze_command_list_handle_t CmdList = *CmdListOrErr;
|
||||
auto CmdQueueOrErr = l0Device.getCmdQueue();
|
||||
if (!CmdQueueOrErr)
|
||||
return CmdQueueOrErr.takeError();
|
||||
const ze_command_queue_handle_t CmdQueue = *CmdQueueOrErr;
|
||||
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Using regular command list for kernel submission.\n");
|
||||
|
||||
ze_event_handle_t Event = nullptr;
|
||||
CALL_ZE_RET_ERROR(zeCommandListAppendLaunchKernel, CmdList, zeKernel,
|
||||
&KEnv.GroupCounts, Event, 0, nullptr);
|
||||
KEnv.KernelPR.Mtx.unlock();
|
||||
CALL_ZE_RET_ERROR(zeCommandListClose, CmdList);
|
||||
CALL_ZE_RET_ERROR_MTX(zeCommandQueueExecuteCommandLists, l0Device.getMutex(),
|
||||
CmdQueue, 1, &CmdList, nullptr);
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Submitted kernel " DPxMOD " to device %s\n", DPxPTR(zeKernel), IdStr);
|
||||
CALL_ZE_RET_ERROR(zeCommandQueueSynchronize, CmdQueue, L0DefaultTimeout);
|
||||
CALL_ZE_RET_ERROR(zeCommandListReset, CmdList);
|
||||
if (Event) {
|
||||
if (auto Err = l0Device.releaseEvent(Event))
|
||||
return Err;
|
||||
}
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Executed kernel entry " DPxMOD " on device %s\n", DPxPTR(zeKernel),
|
||||
IdStr);
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0KernelTy::setKernelGroups(L0DeviceTy &l0Device, L0LaunchEnvTy &KEnv,
|
||||
uint32_t NumThreads[3],
|
||||
uint32_t NumBlocks[3]) const {
|
||||
|
||||
if (KernelEnvironment.Configuration.ExecMode != OMP_TGT_EXEC_MODE_BARE) {
|
||||
// For non-bare mode, the groups are already set in the launch.
|
||||
KEnv.GroupCounts = {NumBlocks[0], NumBlocks[1], NumBlocks[2]};
|
||||
CALL_ZE_RET_ERROR(zeKernelSetGroupSize, getZeKernel(), NumThreads[0],
|
||||
NumThreads[1], NumThreads[2]);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
int32_t NumTeams = NumBlocks[0];
|
||||
int32_t ThreadLimit = NumThreads[0];
|
||||
if (NumTeams < 0)
|
||||
NumTeams = 0;
|
||||
if (ThreadLimit < 0)
|
||||
ThreadLimit = 0;
|
||||
|
||||
uint32_t GroupSizes[3];
|
||||
auto DeviceId = l0Device.getDeviceId();
|
||||
auto &KernelPR = KEnv.KernelPR;
|
||||
// Check if we can reuse previous group parameters.
|
||||
bool GroupParamsReused =
|
||||
KernelPR.reuseGroupParams(NumTeams, ThreadLimit, GroupSizes, KEnv);
|
||||
|
||||
if (!GroupParamsReused) {
|
||||
if (auto Err =
|
||||
getGroupsShape(l0Device, NumTeams, ThreadLimit, GroupSizes, KEnv))
|
||||
return Err;
|
||||
KernelPR.cacheGroupParams(NumTeams, ThreadLimit, GroupSizes, KEnv);
|
||||
}
|
||||
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Team sizes = {%" PRIu32 ", %" PRIu32 ", %" PRIu32 "}\n", GroupSizes[0],
|
||||
GroupSizes[1], GroupSizes[2]);
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Number of teams = {%" PRIu32 ", %" PRIu32 ", %" PRIu32 "}\n",
|
||||
KEnv.GroupCounts.groupCountX, KEnv.GroupCounts.groupCountY,
|
||||
KEnv.GroupCounts.groupCountZ);
|
||||
|
||||
if (!GroupParamsReused) {
|
||||
CALL_ZE_RET_ERROR(zeKernelSetGroupSize, getZeKernel(), GroupSizes[0],
|
||||
GroupSizes[1], GroupSizes[2]);
|
||||
}
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0KernelTy::setIndirectFlags(L0DeviceTy &l0Device,
|
||||
L0LaunchEnvTy &KEnv) const {
|
||||
// Set Kernel Indirect flags.
|
||||
ze_kernel_indirect_access_flags_t Flags = 0;
|
||||
Flags |= l0Device.getMemAllocator(TARGET_ALLOC_HOST).getIndirectFlags();
|
||||
Flags |= l0Device.getMemAllocator(TARGET_ALLOC_DEVICE).getIndirectFlags();
|
||||
|
||||
if (KEnv.KernelPR.IndirectAccessFlags != Flags) {
|
||||
// Combine with common access flags.
|
||||
const auto FinalFlags = l0Device.getIndirectFlags() | Flags;
|
||||
CALL_ZE_RET_ERROR(zeKernelSetIndirectAccess, zeKernel, FinalFlags);
|
||||
DP("Setting indirect access flags " DPxMOD "\n", DPxPTR(FinalFlags));
|
||||
KEnv.KernelPR.IndirectAccessFlags = Flags;
|
||||
}
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0KernelTy::launchImpl(GenericDeviceTy &GenericDevice,
|
||||
uint32_t NumThreads[3], uint32_t NumBlocks[3],
|
||||
KernelArgsTy &KernelArgs,
|
||||
KernelLaunchParamsTy LaunchParams,
|
||||
AsyncInfoWrapperTy &AsyncInfoWrapper) const {
|
||||
auto &l0Device = L0DeviceTy::makeL0Device(GenericDevice);
|
||||
__tgt_async_info *AsyncInfo = AsyncInfoWrapper;
|
||||
|
||||
auto zeKernel = getZeKernel();
|
||||
auto DeviceId = l0Device.getDeviceId();
|
||||
int32_t NumArgs = KernelArgs.NumArgs;
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId, "Launching kernel " DPxMOD "...\n",
|
||||
DPxPTR(zeKernel));
|
||||
|
||||
auto &Plugin = l0Device.getPlugin();
|
||||
auto *IdStr = l0Device.getZeIdCStr();
|
||||
auto &Options = LevelZeroPluginTy::getOptions();
|
||||
bool IsAsync = AsyncInfo && l0Device.asyncEnabled();
|
||||
if (IsAsync && !AsyncInfo->Queue) {
|
||||
AsyncInfo->Queue = reinterpret_cast<void *>(Plugin.getAsyncQueue());
|
||||
if (!AsyncInfo->Queue)
|
||||
IsAsync = false; // Couldn't get a queue, revert to sync.
|
||||
}
|
||||
auto *AsyncQueue =
|
||||
IsAsync ? static_cast<AsyncQueueTy *>(AsyncInfo->Queue) : nullptr;
|
||||
auto &KernelPR = getProperties();
|
||||
|
||||
L0LaunchEnvTy KEnv(IsAsync, AsyncQueue, KernelPR);
|
||||
|
||||
// Protect from kernel preparation to submission as kernels are shared.
|
||||
KernelPR.Mtx.lock();
|
||||
|
||||
if (auto Err = setKernelGroups(l0Device, KEnv, NumThreads, NumBlocks))
|
||||
return Err;
|
||||
|
||||
// Set kernel arguments.
|
||||
for (int32_t I = 0; I < NumArgs; I++) {
|
||||
// Scope code to ease integration with downstream custom code.
|
||||
{
|
||||
void *Arg = (static_cast<void **>(LaunchParams.Data))[I];
|
||||
CALL_ZE_RET_ERROR(zeKernelSetArgumentValue, zeKernel, I, sizeof(Arg),
|
||||
Arg == nullptr ? nullptr : &Arg);
|
||||
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
|
||||
"Kernel Pointer argument %" PRId32 " (value: " DPxMOD
|
||||
") was set successfully for device %s.\n",
|
||||
I, DPxPTR(Arg), IdStr);
|
||||
}
|
||||
}
|
||||
|
||||
if (auto Err = setIndirectFlags(l0Device, KEnv))
|
||||
return Err;
|
||||
|
||||
// The next calls should unlock the KernelLock internally.
|
||||
const bool UseImmCmdList = l0Device.useImmForCompute();
|
||||
if (UseImmCmdList)
|
||||
return launchKernelWithImmCmdList(l0Device, zeKernel, KEnv,
|
||||
Options.CommandMode);
|
||||
|
||||
return launchKernelWithCmdQueue(l0Device, zeKernel, KEnv);
|
||||
}
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
720
offload/plugins-nextgen/level_zero/src/L0Memory.cpp
Normal file
720
offload/plugins-nextgen/level_zero/src/L0Memory.cpp
Normal file
|
|
@ -0,0 +1,720 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Memory related support for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "L0Memory.h"
|
||||
#include "L0Device.h"
|
||||
#include "L0Plugin.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
#if OMPTARGET_DEBUG
|
||||
static const char *AllocKindToStr(int32_t Kind) {
|
||||
switch (Kind) {
|
||||
case TARGET_ALLOC_DEVICE:
|
||||
return "DEVICE";
|
||||
case TARGET_ALLOC_HOST:
|
||||
return "HOST";
|
||||
case TARGET_ALLOC_SHARED:
|
||||
return "SHARED";
|
||||
default:
|
||||
return "DEFAULT";
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void *MemAllocatorTy::MemPoolTy::BlockTy::alloc() {
|
||||
if (isFull())
|
||||
return nullptr;
|
||||
|
||||
if (FreeSlot != MaxSlots) {
|
||||
const uint32_t Slot = FreeSlot;
|
||||
FreeSlot = MaxSlots;
|
||||
UsedSlots[Slot] = true;
|
||||
NumUsedSlots++;
|
||||
return reinterpret_cast<void *>(Base + Slot * ChunkSize);
|
||||
}
|
||||
for (uint32_t I = 0; I < NumSlots; I++) {
|
||||
if (UsedSlots[I])
|
||||
continue;
|
||||
UsedSlots[I] = true;
|
||||
NumUsedSlots++;
|
||||
return reinterpret_cast<void *>(Base + I * ChunkSize);
|
||||
}
|
||||
// Should not reach here.
|
||||
assert(false && "Inconsistent memory pool state");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/// Deallocate the given memory.
|
||||
void MemAllocatorTy::MemPoolTy::BlockTy::dealloc(void *Mem) {
|
||||
if (!contains(Mem))
|
||||
assert(0 && "Inconsistent memory pool state");
|
||||
const uint32_t Slot = (reinterpret_cast<uintptr_t>(Mem) - Base) / ChunkSize;
|
||||
UsedSlots[Slot] = false;
|
||||
NumUsedSlots--;
|
||||
FreeSlot = Slot;
|
||||
}
|
||||
|
||||
Error MemAllocatorTy::MemPoolTy::init(int32_t Kind, MemAllocatorTy *AllocatorIn,
|
||||
const L0OptionsTy &Option) {
|
||||
AllocKind = Kind;
|
||||
Allocator = AllocatorIn;
|
||||
|
||||
// Read user-defined options.
|
||||
const auto &UserOptions = Option.MemPoolConfig[AllocKind];
|
||||
const size_t UserAllocMax = UserOptions.AllocMax;
|
||||
const size_t UserCapacity = UserOptions.Capacity;
|
||||
const size_t UserPoolSize = UserOptions.PoolSize;
|
||||
|
||||
BlockCapacity = UserCapacity;
|
||||
PoolSizeMax = UserPoolSize << 20; // Covert MB to B.
|
||||
PoolSize = 0;
|
||||
|
||||
auto Context = Allocator->L0Context->getZeContext();
|
||||
const auto Device = Allocator->Device;
|
||||
|
||||
// Check page size used for this allocation kind to decide minimum.
|
||||
// allocation size when allocating from L0.
|
||||
auto MemOrErr = Allocator->allocFromL0(8, 0, AllocKind);
|
||||
if (!MemOrErr)
|
||||
return MemOrErr.takeError();
|
||||
void *Mem = *MemOrErr;
|
||||
ze_memory_allocation_properties_t AP{
|
||||
ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES, nullptr,
|
||||
ZE_MEMORY_TYPE_UNKNOWN, 0, 0};
|
||||
CALL_ZE_RET_ERROR(zeMemGetAllocProperties, Context, Mem, &AP, nullptr);
|
||||
AllocUnit = (std::max)(AP.pageSize, AllocUnit);
|
||||
if (auto Err = Allocator->deallocFromL0(Mem))
|
||||
return Err;
|
||||
|
||||
bool IsDiscrete = false;
|
||||
if (Device) {
|
||||
ze_device_properties_t Properties{};
|
||||
Properties.deviceId = 0;
|
||||
Properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
Properties.pNext = nullptr;
|
||||
CALL_ZE_RET_ERROR(zeDeviceGetProperties, Device->getZeDevice(),
|
||||
&Properties);
|
||||
IsDiscrete = Device->isDiscreteDevice();
|
||||
|
||||
if (AllocKind == TARGET_ALLOC_SHARED && IsDiscrete) {
|
||||
// Use page size as minimum chunk size for USM shared on discrete.
|
||||
// device.
|
||||
// FIXME: pageSize is not returned correctly (=0) on some new devices,
|
||||
// so use fallback value for now.
|
||||
AllocMin = (std::max)(AP.pageSize, AllocUnit);
|
||||
AllocUnit = AllocMin * BlockCapacity;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert MB to B and round up to power of 2.
|
||||
AllocMax = AllocMin << getBucketId(UserAllocMax * (1 << 20));
|
||||
if (AllocMin >= AllocMax) {
|
||||
AllocMax = 2 * AllocMin;
|
||||
DP("Warning: Adjusting pool's AllocMax to %zu for %s due to device "
|
||||
"requirements.\n",
|
||||
AllocMax, AllocKindToStr(AllocKind));
|
||||
}
|
||||
assert(AllocMin < AllocMax &&
|
||||
"Invalid parameters while initializing memory pool");
|
||||
const auto MinSize = getBucketId(AllocMin);
|
||||
const auto MaxSize = getBucketId(AllocMax);
|
||||
Buckets.resize(MaxSize - MinSize + 1);
|
||||
BucketStats.resize(Buckets.size(), {0, 0});
|
||||
|
||||
// Set bucket parameters
|
||||
for (size_t I = 0; I < Buckets.size(); I++) {
|
||||
const size_t ChunkSize = AllocMin << I;
|
||||
size_t BlockSize = ChunkSize * BlockCapacity;
|
||||
// On discrete device, the cost of native L0 invocation doubles when the
|
||||
// the requested size doubles after certain threshold, so allocating
|
||||
// larger block does not pay off at all. It is better to keep a single
|
||||
// chunk in a single block in such cases.
|
||||
if (BlockSize <= AllocUnit) {
|
||||
BlockSize = AllocUnit; // Allocation unit is already large enough.
|
||||
} else if (IsDiscrete) {
|
||||
// Do not preallocate if it does not pay off.
|
||||
if (ChunkSize >= L0UsmPreAllocThreshold ||
|
||||
(AllocKind == TARGET_ALLOC_HOST &&
|
||||
ChunkSize >= L0HostUsmPreAllocThreshold))
|
||||
BlockSize = ChunkSize;
|
||||
}
|
||||
BucketParams.emplace_back(ChunkSize, BlockSize);
|
||||
}
|
||||
|
||||
DP("Initialized %s pool for device " DPxMOD ": AllocUnit = %zu, "
|
||||
"AllocMax = %zu, "
|
||||
"Capacity = %" PRIu32 ", PoolSizeMax = %zu\n",
|
||||
AllocKindToStr(AllocKind), DPxPTR(Device), AllocUnit, AllocMax,
|
||||
BlockCapacity, PoolSizeMax);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
// Used for reduction pool.
|
||||
Error MemAllocatorTy::MemPoolTy::init(MemAllocatorTy *AllocatorIn,
|
||||
const L0OptionsTy &Option) {
|
||||
AllocKind = TARGET_ALLOC_DEVICE;
|
||||
Allocator = AllocatorIn;
|
||||
AllocMin = AllocUnit = 1024 << 6; // 64KB.
|
||||
AllocMax = Option.ReductionPoolInfo[0] << 20;
|
||||
BlockCapacity = Option.ReductionPoolInfo[1];
|
||||
PoolSize = 0;
|
||||
PoolSizeMax = (size_t)Option.ReductionPoolInfo[2] << 20;
|
||||
|
||||
const auto MinSize = getBucketId(AllocMin);
|
||||
const auto MaxSize = getBucketId(AllocMax);
|
||||
Buckets.resize(MaxSize - MinSize + 1);
|
||||
BucketStats.resize(Buckets.size(), {0, 0});
|
||||
for (size_t I = 0; I < Buckets.size(); I++) {
|
||||
const size_t ChunkSize = AllocMin << I;
|
||||
BucketParams.emplace_back(ChunkSize, ChunkSize * BlockCapacity);
|
||||
}
|
||||
|
||||
DP("Initialized reduction scratch pool for device " DPxMOD
|
||||
": AllocMin = %zu, AllocMax = %zu, PoolSizeMax = %zu\n",
|
||||
DPxPTR(Allocator->Device), AllocMin, AllocMax, PoolSizeMax);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
// Used for small memory pool with fixed parameters.
|
||||
Error MemAllocatorTy::MemPoolTy::init(MemAllocatorTy *AllocatorIn) {
|
||||
AllocKind = TARGET_ALLOC_DEVICE;
|
||||
Allocator = AllocatorIn;
|
||||
AllocMax = AllocMin;
|
||||
BlockCapacity = AllocUnit / AllocMax;
|
||||
PoolSize = 0;
|
||||
PoolSizeMax = (1 << 20); // This should be sufficiently large.
|
||||
Buckets.resize(1);
|
||||
BucketStats.resize(1, {0, 0});
|
||||
BucketParams.emplace_back(AllocMax, AllocUnit);
|
||||
ZeroInit = true;
|
||||
DP("Initialized zero-initialized reduction counter pool for "
|
||||
"device " DPxMOD ": AllocMin = %zu, AllocMax = %zu, PoolSizeMax = %zu\n",
|
||||
DPxPTR(Allocator->Device), AllocMin, AllocMax, PoolSizeMax);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
void MemAllocatorTy::MemPoolTy::printUsage() {
|
||||
auto PrintNum = [](uint64_t Num) {
|
||||
if (Num > 1e9)
|
||||
fprintf(stderr, "%11.2e", float(Num));
|
||||
else
|
||||
fprintf(stderr, "%11" PRIu64, Num);
|
||||
};
|
||||
|
||||
bool HasPoolAlloc = false;
|
||||
for (auto &Stat : BucketStats) {
|
||||
if (Stat.first > 0 || Stat.second > 0) {
|
||||
HasPoolAlloc = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
DP("MemPool usage for %s, device " DPxMOD "\n", AllocKindToStr(AllocKind),
|
||||
DPxPTR(Allocator->Device));
|
||||
|
||||
if (HasPoolAlloc) {
|
||||
DP("-- AllocMax=%zu(MB), Capacity=%" PRIu32 ", PoolSizeMax=%zu(MB)\n",
|
||||
AllocMax >> 20, BlockCapacity, PoolSizeMax >> 20);
|
||||
DP("-- %18s:%11s%11s%11s\n", "", "NewAlloc", "Reuse", "Hit(%)");
|
||||
for (size_t I = 0; I < Buckets.size(); I++) {
|
||||
const auto &Stat = BucketStats[I];
|
||||
if (Stat.first > 0 || Stat.second > 0) {
|
||||
DP("-- Bucket[%10zu]:", BucketParams[I].first);
|
||||
PrintNum(Stat.first);
|
||||
PrintNum(Stat.second);
|
||||
fprintf(stderr, "%11.2f\n",
|
||||
float(Stat.second) / float(Stat.first + Stat.second) * 100);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DP("-- Not used\n");
|
||||
}
|
||||
}
|
||||
|
||||
/// Release resources used in the pool.
|
||||
Error MemAllocatorTy::MemPoolTy::deinit() {
|
||||
const int DebugLevel = getDebugLevel();
|
||||
if (DebugLevel > 0)
|
||||
printUsage();
|
||||
for (auto &Bucket : Buckets) {
|
||||
for (auto *Block : Bucket) {
|
||||
if (DebugLevel > 0)
|
||||
Allocator->log(0, Block->Size, AllocKind);
|
||||
auto Err =
|
||||
Allocator->deallocFromL0(reinterpret_cast<void *>(Block->Base));
|
||||
delete Block;
|
||||
if (Err)
|
||||
return Err;
|
||||
}
|
||||
}
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
/// Allocate the requested size of memory from this pool.
|
||||
/// AllocSize is the chunk size internally used for the returned memory.
|
||||
Expected<void *> MemAllocatorTy::MemPoolTy::alloc(size_t Size,
|
||||
size_t &AllocSize) {
|
||||
if (Size == 0 || Size > AllocMax)
|
||||
return nullptr;
|
||||
|
||||
const uint32_t BucketId = getBucketId(Size);
|
||||
auto &Blocks = Buckets[BucketId];
|
||||
void *Mem = nullptr;
|
||||
|
||||
for (auto *Block : Blocks) {
|
||||
if (Block->isFull())
|
||||
continue;
|
||||
Mem = Block->alloc();
|
||||
assert(Mem && "Inconsistent state while allocating memory from pool");
|
||||
PtrToBlock.try_emplace(Mem, Block);
|
||||
break;
|
||||
}
|
||||
|
||||
if (Mem == nullptr) {
|
||||
const bool IsSmallAllocatable =
|
||||
(Size <= SmallAllocMax && SmallPoolSize <= SmallPoolSizeMax);
|
||||
const bool IsFull = (PoolSize > PoolSizeMax);
|
||||
if (IsFull && !IsSmallAllocatable)
|
||||
return nullptr;
|
||||
// Bucket is empty or all blocks in the bucket are full.
|
||||
const auto ChunkSize = BucketParams[BucketId].first;
|
||||
const auto BlockSize = BucketParams[BucketId].second;
|
||||
auto BaseOrErr = Allocator->allocFromL0AndLog(BlockSize, 0, AllocKind);
|
||||
if (!BaseOrErr)
|
||||
return BaseOrErr.takeError();
|
||||
|
||||
void *Base = *BaseOrErr;
|
||||
if (ZeroInit) {
|
||||
auto Err = Allocator->enqueueMemSet(Base, 0, BlockSize);
|
||||
if (Err)
|
||||
return Err;
|
||||
}
|
||||
|
||||
BlockTy *Block = new BlockTy(Base, BlockSize, ChunkSize);
|
||||
Blocks.push_back(Block);
|
||||
Mem = Block->alloc();
|
||||
PtrToBlock.try_emplace(Mem, Block);
|
||||
if (IsFull)
|
||||
SmallPoolSize += BlockSize;
|
||||
else
|
||||
PoolSize += BlockSize;
|
||||
DP("New block allocation for %s pool: base = " DPxMOD
|
||||
", size = %zu, pool size = %zu\n",
|
||||
AllocKindToStr(AllocKind), DPxPTR(Base), BlockSize, PoolSize);
|
||||
BucketStats[BucketId].first++;
|
||||
} else {
|
||||
BucketStats[BucketId].second++;
|
||||
}
|
||||
|
||||
AllocSize = (AllocMin << BucketId);
|
||||
|
||||
return Mem;
|
||||
}
|
||||
|
||||
/// Deallocate the specified memory and returns block size deallocated.
|
||||
size_t MemAllocatorTy::MemPoolTy::dealloc(void *Ptr) {
|
||||
if (PtrToBlock.count(Ptr) == 0)
|
||||
return 0;
|
||||
PtrToBlock[Ptr]->dealloc(Ptr);
|
||||
const size_t Deallocated = PtrToBlock[Ptr]->ChunkSize;
|
||||
PtrToBlock.erase(Ptr);
|
||||
return Deallocated;
|
||||
}
|
||||
|
||||
void MemAllocatorTy::MemAllocInfoMapTy::add(void *Ptr, void *Base,
|
||||
size_t ReqSize, size_t AllocSize,
|
||||
int32_t Kind, bool InPool,
|
||||
bool ImplicitArg) {
|
||||
const auto Inserted = Map.emplace(
|
||||
Ptr, MemAllocInfoTy{Base, ReqSize, AllocSize, Kind, InPool, ImplicitArg});
|
||||
// Check if we keep valid disjoint memory ranges.
|
||||
[[maybe_unused]] bool Valid = Inserted.second;
|
||||
if (Valid) {
|
||||
if (Inserted.first != Map.begin()) {
|
||||
const auto I = std::prev(Inserted.first, 1);
|
||||
Valid =
|
||||
Valid && (uintptr_t)I->first + I->second.ReqSize <= (uintptr_t)Ptr;
|
||||
}
|
||||
if (Valid) {
|
||||
const auto I = std::next(Inserted.first, 1);
|
||||
if (I != Map.end())
|
||||
Valid = Valid && (uintptr_t)Ptr + ReqSize <= (uintptr_t)I->first;
|
||||
}
|
||||
}
|
||||
assert(Valid && "Invalid overlapping memory allocation");
|
||||
assert(Kind >= 0 && Kind < MaxMemKind && "Invalid target allocation kind");
|
||||
if (ImplicitArg)
|
||||
NumImplicitArgs[Kind]++;
|
||||
}
|
||||
|
||||
/// Remove allocation information for the given memory location.
|
||||
bool MemAllocatorTy::MemAllocInfoMapTy::remove(void *Ptr,
|
||||
MemAllocInfoTy *Removed) {
|
||||
const auto AllocInfo = Map.find(Ptr);
|
||||
if (AllocInfo == Map.end())
|
||||
return false;
|
||||
if (AllocInfo->second.ImplicitArg)
|
||||
NumImplicitArgs[AllocInfo->second.Kind]--;
|
||||
if (Removed)
|
||||
*Removed = AllocInfo->second;
|
||||
Map.erase(AllocInfo);
|
||||
return true;
|
||||
}
|
||||
|
||||
Error MemAllocatorTy::initDevicePools(L0DeviceTy &L0Device,
|
||||
const L0OptionsTy &Options) {
|
||||
SupportsLargeMem = L0Device.supportsLargeMem();
|
||||
IsHostMem = false;
|
||||
Device = &L0Device;
|
||||
L0Context = &L0Device.getL0Context();
|
||||
for (auto Kind : {TARGET_ALLOC_DEVICE, TARGET_ALLOC_SHARED}) {
|
||||
if (Options.MemPoolConfig[Kind].Use) {
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
Pools[Kind] = std::make_unique<MemPoolTy>();
|
||||
if (auto Err = Pools[Kind]->init(Kind, this, Options))
|
||||
return Err;
|
||||
}
|
||||
}
|
||||
ReductionPool = std::make_unique<MemPoolTy>();
|
||||
if (auto Err = ReductionPool->init(this, Options))
|
||||
return Err;
|
||||
CounterPool = std::make_unique<MemPoolTy>();
|
||||
if (auto Err = CounterPool->init(this))
|
||||
return Err;
|
||||
updateMaxAllocSize(L0Device);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error MemAllocatorTy::initHostPool(L0ContextTy &Driver,
|
||||
const L0OptionsTy &Option) {
|
||||
SupportsLargeMem = Driver.supportsLargeMem();
|
||||
IsHostMem = true;
|
||||
L0Context = &Driver;
|
||||
if (Option.MemPoolConfig[TARGET_ALLOC_HOST].Use) {
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
Pools[TARGET_ALLOC_HOST] = std::make_unique<MemPoolTy>();
|
||||
if (auto Err =
|
||||
Pools[TARGET_ALLOC_HOST]->init(TARGET_ALLOC_HOST, this, Option))
|
||||
return Err;
|
||||
}
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
void MemAllocatorTy::updateMaxAllocSize(L0DeviceTy &L0Device) {
|
||||
// Update the maximum allocation size for this Allocator.
|
||||
auto maxMemAllocSize = L0Device.getMaxMemAllocSize();
|
||||
|
||||
if (IsHostMem) {
|
||||
// MaxAllocSize should be the minimum of all devices from the driver.
|
||||
if (MaxAllocSize > maxMemAllocSize) {
|
||||
MaxAllocSize = maxMemAllocSize;
|
||||
DP("Updated MaxAllocSize for driver " DPxMOD " to %zu\n",
|
||||
DPxPTR(L0Context), MaxAllocSize);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
MaxAllocSize = maxMemAllocSize;
|
||||
DP("Updated MaxAllocSize for device " DPxMOD " to %zu\n", DPxPTR(Device),
|
||||
MaxAllocSize);
|
||||
}
|
||||
|
||||
/// Release resources and report statistics if requested.
|
||||
Error MemAllocatorTy::deinit() {
|
||||
if (!L0Context)
|
||||
return Plugin::success();
|
||||
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
if (!L0Context)
|
||||
return Plugin::success();
|
||||
// Release RTL-owned memory.
|
||||
for (auto *M : MemOwned) {
|
||||
auto Err = deallocLocked(M);
|
||||
if (Err)
|
||||
return Err;
|
||||
}
|
||||
for (auto &Pool : Pools) {
|
||||
if (Pool) {
|
||||
if (auto Err = Pool->deinit())
|
||||
return Err;
|
||||
Pool.reset(nullptr);
|
||||
}
|
||||
}
|
||||
if (ReductionPool) {
|
||||
if (auto Err = ReductionPool->deinit())
|
||||
return Err;
|
||||
ReductionPool.reset(nullptr);
|
||||
}
|
||||
if (CounterPool) {
|
||||
if (auto Err = CounterPool->deinit())
|
||||
return Err;
|
||||
CounterPool.reset(nullptr);
|
||||
}
|
||||
// Report memory usage if requested.
|
||||
if (getDebugLevel() > 0) {
|
||||
for (size_t Kind = 0; Kind < MaxMemKind; Kind++) {
|
||||
auto &Stat = Stats[Kind];
|
||||
DP("Memory usage for %s, device " DPxMOD "\n", AllocKindToStr(Kind),
|
||||
DPxPTR(Device));
|
||||
if (Stat.NumAllocs[0] == 0 && Stat.NumAllocs[1] == 0) {
|
||||
DP("-- Not used\n");
|
||||
continue;
|
||||
}
|
||||
DP("-- Allocator: %12s, %12s\n", "Native", "Pool");
|
||||
DP("-- Requested: %12zu, %12zu\n", Stat.Requested[0], Stat.Requested[1]);
|
||||
DP("-- Allocated: %12zu, %12zu\n", Stat.Allocated[0], Stat.Allocated[1]);
|
||||
DP("-- Freed : %12zu, %12zu\n", Stat.Freed[0], Stat.Freed[1]);
|
||||
DP("-- InUse : %12zu, %12zu\n", Stat.InUse[0], Stat.InUse[1]);
|
||||
DP("-- PeakUse : %12zu, %12zu\n", Stat.PeakUse[0], Stat.PeakUse[1]);
|
||||
DP("-- NumAllocs: %12zu, %12zu\n", Stat.NumAllocs[0], Stat.NumAllocs[1]);
|
||||
}
|
||||
}
|
||||
|
||||
// Mark as deinitialized.
|
||||
L0Context = nullptr;
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
/// Allocate memory with the specified information.
|
||||
Expected<void *> MemAllocatorTy::allocFromPool(size_t Size, size_t Align,
|
||||
int32_t Kind, intptr_t Offset,
|
||||
bool UserAlloc, bool DevMalloc,
|
||||
uint32_t MemAdvice,
|
||||
AllocOptionTy AllocOpt) {
|
||||
assert((Kind == TARGET_ALLOC_DEVICE || Kind == TARGET_ALLOC_HOST ||
|
||||
Kind == TARGET_ALLOC_SHARED) &&
|
||||
"Unknown memory kind while allocating target memory");
|
||||
|
||||
std::lock_guard<std::mutex> Lock(Mtx);
|
||||
|
||||
// We do not expect meaningful Align parameter when Offset > 0, so the
|
||||
// following code does not handle such case.
|
||||
|
||||
size_t AllocSize = Size + Offset;
|
||||
void *Mem = nullptr;
|
||||
void *AllocBase = nullptr;
|
||||
const bool UseScratchPool =
|
||||
(AllocOpt == AllocOptionTy::ALLOC_OPT_REDUCTION_SCRATCH);
|
||||
const bool UseZeroInitPool =
|
||||
(AllocOpt == AllocOptionTy::ALLOC_OPT_REDUCTION_COUNTER);
|
||||
const bool UseDedicatedPool = UseScratchPool || UseZeroInitPool;
|
||||
|
||||
if ((Pools[Kind] &&
|
||||
MemAdvice == std::numeric_limits<decltype(MemAdvice)>::max()) ||
|
||||
UseDedicatedPool) {
|
||||
// Pool is enabled for the allocation kind, and we do not use any memory
|
||||
// advice. We should avoid using pool if there is any meaningful memory
|
||||
// advice not to affect sibling allocation in the same block.
|
||||
if (Align > 0)
|
||||
AllocSize += (Align - 1);
|
||||
size_t PoolAllocSize = 0;
|
||||
MemPoolTy *Pool = nullptr;
|
||||
|
||||
if (UseScratchPool)
|
||||
Pool = ReductionPool.get();
|
||||
else if (UseZeroInitPool)
|
||||
Pool = CounterPool.get();
|
||||
else
|
||||
Pool = Pools[Kind].get();
|
||||
|
||||
auto PtrOrErr = Pool->alloc(AllocSize, PoolAllocSize);
|
||||
if (!PtrOrErr)
|
||||
return PtrOrErr.takeError();
|
||||
AllocBase = *PtrOrErr;
|
||||
if (AllocBase) {
|
||||
uintptr_t Base = (uintptr_t)AllocBase;
|
||||
if (Align > 0)
|
||||
Base = (Base + Align) & ~(Align - 1);
|
||||
Mem = (void *)(Base + Offset);
|
||||
AllocInfo.add(Mem, AllocBase, Size, PoolAllocSize, Kind, true, UserAlloc);
|
||||
log(Size, PoolAllocSize, Kind, true /* Pool */);
|
||||
if (DevMalloc)
|
||||
MemOwned.push_back(AllocBase);
|
||||
if (UseDedicatedPool) {
|
||||
DP("Allocated %zu bytes from %s pool\n", Size,
|
||||
UseScratchPool ? "scratch" : "zero-initialized");
|
||||
}
|
||||
return Mem;
|
||||
}
|
||||
}
|
||||
|
||||
auto AllocBaseOrErr =
|
||||
allocFromL0AndLog(AllocSize, Align, Kind, /*ActiveSize=*/Size);
|
||||
if (!AllocBaseOrErr)
|
||||
return AllocBaseOrErr.takeError();
|
||||
AllocBase = *AllocBaseOrErr;
|
||||
if (AllocBase) {
|
||||
Mem = (void *)((uintptr_t)AllocBase + Offset);
|
||||
AllocInfo.add(Mem, AllocBase, Size, AllocSize, Kind, false, UserAlloc);
|
||||
if (DevMalloc)
|
||||
MemOwned.push_back(AllocBase);
|
||||
if (UseDedicatedPool) {
|
||||
// We do not want this happen in general.
|
||||
DP("Allocated %zu bytes from L0 for %s pool\n", Size,
|
||||
UseScratchPool ? "scratch" : "zero-initialized");
|
||||
}
|
||||
}
|
||||
return Mem;
|
||||
}
|
||||
|
||||
/// Deallocate memory.
|
||||
Error MemAllocatorTy::deallocLocked(void *Ptr) {
|
||||
MemAllocInfoTy Info;
|
||||
if (!AllocInfo.remove(Ptr, &Info)) {
|
||||
return Plugin::error(ErrorCode::BACKEND_FAILURE,
|
||||
"Cannot find memory allocation information for " DPxMOD
|
||||
"\n",
|
||||
DPxPTR(Ptr));
|
||||
}
|
||||
if (Info.InPool) {
|
||||
size_t DeallocSize = 0;
|
||||
if (Pools[Info.Kind])
|
||||
DeallocSize = Pools[Info.Kind]->dealloc(Info.Base);
|
||||
if (DeallocSize == 0) {
|
||||
// Try reduction scratch pool.
|
||||
DeallocSize = ReductionPool->dealloc(Info.Base);
|
||||
// Try reduction counter pool.
|
||||
if (DeallocSize == 0)
|
||||
DeallocSize = CounterPool->dealloc(Info.Base);
|
||||
if (DeallocSize == 0) {
|
||||
return Plugin::error(ErrorCode::BACKEND_FAILURE,
|
||||
"Cannot return memory " DPxMOD " to pool\n",
|
||||
DPxPTR(Ptr));
|
||||
}
|
||||
}
|
||||
log(0, DeallocSize, Info.Kind, true /* Pool */);
|
||||
return Plugin::success();
|
||||
}
|
||||
if (!Info.Base) {
|
||||
DP("Error: Cannot find base address of " DPxMOD "\n", DPxPTR(Ptr));
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Cannot find base address of " DPxMOD "\n",
|
||||
DPxPTR(Ptr));
|
||||
}
|
||||
log(/*NoReqSize*/ 0, Info.AllocSize, Info.Kind);
|
||||
|
||||
if (auto Err = deallocFromL0(Info.Base))
|
||||
return Err;
|
||||
DP("Deleted device memory " DPxMOD " (Base: " DPxMOD ", Size: %zu)\n",
|
||||
DPxPTR(Ptr), DPxPTR(Info.Base), Info.AllocSize);
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error MemAllocatorTy::enqueueMemSet(void *Dst, int8_t Value, size_t Size) {
|
||||
return Device->enqueueMemFill(Dst, &Value, sizeof(int8_t), Size);
|
||||
}
|
||||
|
||||
Error MemAllocatorTy::enqueueMemCopy(void *Dst, const void *Src, size_t Size) {
|
||||
return Device->enqueueMemCopy(Dst, Src, Size);
|
||||
}
|
||||
|
||||
Expected<void *> MemAllocatorTy::allocFromL0(size_t Size, size_t Align,
|
||||
int32_t Kind) {
|
||||
void *Mem = nullptr;
|
||||
ze_device_mem_alloc_desc_t DeviceDesc{ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC,
|
||||
nullptr, 0, 0};
|
||||
ze_host_mem_alloc_desc_t HostDesc{ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC,
|
||||
nullptr, 0};
|
||||
|
||||
// Use relaxed allocation limit if driver supports.
|
||||
ze_relaxed_allocation_limits_exp_desc_t RelaxedDesc{
|
||||
ZE_STRUCTURE_TYPE_RELAXED_ALLOCATION_LIMITS_EXP_DESC, nullptr,
|
||||
ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE};
|
||||
if (Size > MaxAllocSize && SupportsLargeMem) {
|
||||
DeviceDesc.pNext = &RelaxedDesc;
|
||||
HostDesc.pNext = &RelaxedDesc;
|
||||
}
|
||||
|
||||
auto ZeDevice = Device ? Device->getZeDevice() : nullptr;
|
||||
auto ZeContext = L0Context->getZeContext();
|
||||
bool MakeResident = false;
|
||||
switch (Kind) {
|
||||
case TARGET_ALLOC_DEVICE:
|
||||
MakeResident = true;
|
||||
CALL_ZE_RET_ERROR(zeMemAllocDevice, ZeContext, &DeviceDesc, Size, Align,
|
||||
ZeDevice, &Mem);
|
||||
DP("Allocated %" PRId64 " bytes of device memory " DPxMOD "\n", Size,
|
||||
DPxPTR(Mem));
|
||||
break;
|
||||
case TARGET_ALLOC_HOST:
|
||||
CALL_ZE_RET_ERROR(zeMemAllocHost, ZeContext, &HostDesc, Size, Align, &Mem);
|
||||
DP("Allocated %" PRId64 " bytes of host memory " DPxMOD "\n", Size,
|
||||
DPxPTR(Mem));
|
||||
break;
|
||||
case TARGET_ALLOC_SHARED:
|
||||
CALL_ZE_RET_ERROR(zeMemAllocShared, ZeContext, &DeviceDesc, &HostDesc, Size,
|
||||
Align, ZeDevice, &Mem);
|
||||
DP("Allocated %" PRId64 " bytes of shared memory " DPxMOD "\n", Size,
|
||||
DPxPTR(Mem));
|
||||
break;
|
||||
default:
|
||||
assert(0 && "Invalid target data allocation kind");
|
||||
}
|
||||
|
||||
if (MakeResident) {
|
||||
assert(Device &&
|
||||
"Device is not set for memory allocation. Is this a Device Pool?");
|
||||
if (auto Err = Device->makeMemoryResident(Mem, Size)) {
|
||||
Mem = nullptr;
|
||||
return std::move(Err);
|
||||
}
|
||||
}
|
||||
return Mem;
|
||||
}
|
||||
|
||||
Error MemAllocatorTy::deallocFromL0(void *Ptr) {
|
||||
CALL_ZE_RET_ERROR(zeMemFree, L0Context->getZeContext(), Ptr);
|
||||
DP("Freed device pointer " DPxMOD "\n", DPxPTR(Ptr));
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Expected<ze_event_handle_t> EventPoolTy::getEvent() {
|
||||
std::lock_guard<std::mutex> Lock(*Mtx);
|
||||
|
||||
if (Events.empty()) {
|
||||
// Need to create a new L0 pool.
|
||||
ze_event_pool_desc_t Desc{ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, nullptr, 0, 0};
|
||||
Desc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE | Flags;
|
||||
Desc.count = PoolSize;
|
||||
ze_event_pool_handle_t Pool;
|
||||
CALL_ZE_RET_ERROR(zeEventPoolCreate, Context, &Desc, 0, nullptr, &Pool);
|
||||
Pools.push_back(Pool);
|
||||
|
||||
// Create events.
|
||||
ze_event_desc_t EventDesc{ZE_STRUCTURE_TYPE_EVENT_DESC, nullptr, 0, 0, 0};
|
||||
EventDesc.wait = 0;
|
||||
EventDesc.signal = ZE_EVENT_SCOPE_FLAG_HOST;
|
||||
for (uint32_t I = 0; I < PoolSize; I++) {
|
||||
EventDesc.index = I;
|
||||
ze_event_handle_t Event;
|
||||
CALL_ZE_RET_ERROR(zeEventCreate, Pool, &EventDesc, &Event);
|
||||
Events.push_back(Event);
|
||||
}
|
||||
}
|
||||
|
||||
auto Ret = Events.back();
|
||||
Events.pop_back();
|
||||
|
||||
return Ret;
|
||||
}
|
||||
|
||||
/// Return an event to the pool.
|
||||
Error EventPoolTy::releaseEvent(ze_event_handle_t Event, L0DeviceTy &Device) {
|
||||
std::lock_guard<std::mutex> Lock(*Mtx);
|
||||
CALL_ZE_RET_ERROR(zeEventHostReset, Event);
|
||||
Events.push_back(Event);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
195
offload/plugins-nextgen/level_zero/src/L0Options.cpp
Normal file
195
offload/plugins-nextgen/level_zero/src/L0Options.cpp
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Level Zero RTL Options support.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "omptarget.h"
|
||||
|
||||
#include "L0Defs.h"
|
||||
#include "L0Options.h"
|
||||
#include "L0Trace.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
/// Read environment variables.
|
||||
void L0OptionsTy::processEnvironmentVars() {
|
||||
// Compilation options for IGC.
|
||||
UserCompilationOptions +=
|
||||
std::string(" ") +
|
||||
StringEnvar("LIBOMPTARGET_LEVEL_ZERO_COMPILATION_OPTIONS", "").get();
|
||||
|
||||
// Memory pool syntax:
|
||||
// LIBOMPTARGET_LEVEL_ZERO_MEMORY_POOL=<Option>
|
||||
// <Option> := 0 | <PoolInfoList>
|
||||
// <PoolInfoList> := <PoolInfo>[,<PoolInfoList>]
|
||||
// <PoolInfo> := <MemType>[,<AllocMax>[,<Capacity>[,<PoolSize>]]]
|
||||
// <MemType> := all | device | host | shared
|
||||
// <AllocMax> := non-negative integer or empty, max allocation size in
|
||||
// MB (default: 1).
|
||||
// <Capacity> := positive integer or empty, number of allocations from
|
||||
// a single block (default: 4).
|
||||
// <PoolSize> := positive integer or empty, max pool size in MB
|
||||
// (default: 256).
|
||||
const StringEnvar MemoryPoolVar("LIBOMPTARGET_LEVEL_ZERO_MEMORY_POOL", "");
|
||||
if (MemoryPoolVar.isPresent()) {
|
||||
if (MemoryPoolVar.get() == "0") {
|
||||
Flags.UseMemoryPool = 0;
|
||||
MemPoolConfig.fill({false, 0, 0, 0});
|
||||
} else {
|
||||
std::istringstream Str(MemoryPoolVar.get());
|
||||
int32_t MemType = -1;
|
||||
int32_t Offset = 0;
|
||||
int32_t Valid = 1;
|
||||
constexpr std::array<int32_t, 3> DefaultValue{1, 4, 256};
|
||||
constexpr int32_t AllMemType =
|
||||
std::numeric_limits<decltype(AllMemType)>::max();
|
||||
std::array<int32_t, 3> AllInfo{1, 4, 256};
|
||||
std::array<std::array<int32_t, 3>, 3> PoolInfo;
|
||||
PoolInfo.fill({-1, 0, 0});
|
||||
for (std::string Token; std::getline(Str, Token, ',') && Valid > 0;) {
|
||||
if (Token == "device") {
|
||||
MemType = TARGET_ALLOC_DEVICE;
|
||||
PoolInfo[TARGET_ALLOC_DEVICE] = DefaultValue;
|
||||
Offset = 0;
|
||||
} else if (Token == "host") {
|
||||
MemType = TARGET_ALLOC_HOST;
|
||||
PoolInfo[TARGET_ALLOC_HOST] = DefaultValue;
|
||||
Offset = 0;
|
||||
} else if (Token == "shared") {
|
||||
MemType = TARGET_ALLOC_SHARED;
|
||||
PoolInfo[TARGET_ALLOC_SHARED] = DefaultValue;
|
||||
Offset = 0;
|
||||
} else if (Token == "all") {
|
||||
MemType = AllMemType;
|
||||
Offset = 0;
|
||||
Valid = 2;
|
||||
} else if (Offset < 3 && MemType >= 0) {
|
||||
int32_t Num = std::atoi(Token.c_str());
|
||||
bool ValidNum = (Num >= 0 && Offset == 0) || (Num > 0 && Offset > 0);
|
||||
if (ValidNum && MemType == AllMemType)
|
||||
AllInfo[Offset++] = Num;
|
||||
else if (ValidNum)
|
||||
PoolInfo[MemType][Offset++] = Num;
|
||||
else if (Token.size() == 0)
|
||||
Offset++;
|
||||
else
|
||||
Valid = 0;
|
||||
} else {
|
||||
Valid = 0;
|
||||
}
|
||||
}
|
||||
if (Valid > 0) {
|
||||
if (Valid == 2) {
|
||||
// "all" is specified -- ignore other inputs.
|
||||
if (AllInfo[0] > 0) {
|
||||
MemPoolConfig[TARGET_ALLOC_DEVICE] = {true, AllInfo[0], AllInfo[1],
|
||||
AllInfo[2]};
|
||||
MemPoolConfig[TARGET_ALLOC_HOST] = {true, AllInfo[0], AllInfo[1],
|
||||
AllInfo[2]};
|
||||
MemPoolConfig[TARGET_ALLOC_SHARED] = {true, AllInfo[0], AllInfo[1],
|
||||
AllInfo[2]};
|
||||
} else {
|
||||
MemPoolConfig.fill({false, 0, 0, 0});
|
||||
}
|
||||
} else {
|
||||
for (size_t Pool = 0; Pool < PoolInfo.size(); ++Pool) {
|
||||
switch (PoolInfo[Pool][0]) {
|
||||
case -1:
|
||||
// No value was specified, keep the default.
|
||||
break;
|
||||
case 0:
|
||||
// Pool was disabled.
|
||||
MemPoolConfig[Pool] = {false, 0, 0, 0};
|
||||
break;
|
||||
default:
|
||||
// Use the user specified values.
|
||||
MemPoolConfig[Pool] = {true, PoolInfo[Pool][0], PoolInfo[Pool][1],
|
||||
PoolInfo[Pool][2]};
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DP("Ignoring incorrect memory pool configuration "
|
||||
"LIBOMPTARGET_LEVEL_ZERO_MEMORY_POOL=%s\n",
|
||||
MemoryPoolVar.get().c_str());
|
||||
DP("LIBOMPTARGET_LEVEL_ZERO_MEMORY_POOL=<Option>\n");
|
||||
DP(" <Option> := 0 | <PoolInfoList>\n");
|
||||
DP(" <PoolInfoList> := <PoolInfo>[,<PoolInfoList>]\n");
|
||||
DP(" <PoolInfo> := "
|
||||
"<MemType>[,<AllocMax>[,<Capacity>[,<PoolSize>]]]\n");
|
||||
DP(" <MemType> := all | device | host | shared\n");
|
||||
DP(" <AllocMax> := non-negative integer or empty, "
|
||||
"max allocation size in MB (default: 1)\n");
|
||||
DP(" <Capacity> := positive integer or empty, "
|
||||
"number of allocations from a single block (default: 4)\n");
|
||||
DP(" <PoolSize> := positive integer or empty, "
|
||||
"max pool size in MB (default: 256)\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (StringEnvar("INTEL_ENABLE_OFFLOAD_ANNOTATIONS").isPresent()) {
|
||||
// To match SYCL RT behavior, we just need to check whether
|
||||
// INTEL_ENABLE_OFFLOAD_ANNOTATIONS is set. The actual value
|
||||
// does not matter.
|
||||
CommonSpecConstants.addConstant<char>(0xFF747469, 1);
|
||||
}
|
||||
|
||||
// LIBOMPTARGET_LEVEL_ZERO_STAGING_BUFFER_SIZE=<SizeInKB>.
|
||||
const Envar<size_t> StagingBufferSizeVar(
|
||||
"LIBOMPTARGET_LEVEL_ZERO_STAGING_BUFFER_SIZE");
|
||||
if (StagingBufferSizeVar.isPresent()) {
|
||||
size_t SizeInKB = StagingBufferSizeVar;
|
||||
if (SizeInKB > (16 << 10)) {
|
||||
SizeInKB = (16 << 10);
|
||||
DP("Staging buffer size is capped at %zu KB\n", SizeInKB);
|
||||
}
|
||||
StagingBufferSize = SizeInKB << 10;
|
||||
}
|
||||
|
||||
// LIBOMPTARGET_LEVEL_ZERO_COMMAND_MODE=<Fmt>.
|
||||
// <Fmt> := sync | async | async_ordered
|
||||
// sync: perform synchronization after each command.
|
||||
// async: perform synchronization when it is required.
|
||||
// async_ordered: same as "async", but command is ordered.
|
||||
// This option is ignored unless IMM is fully enabled on compute and copy.
|
||||
// On Intel PVC GPU, when used with immediate command lists over Level Zero
|
||||
// backend, a target region may involve multiple command submissions to the
|
||||
// L0 copy queue and compute queue. L0 events are used for each submission
|
||||
// (data transfer of a single item or kernel execution). When "async" is
|
||||
// specified, a) each data transfer to device is submitted with an event.
|
||||
// b) The kernel is submitted next with a dependence on all the previous
|
||||
// data transfer events. The kernel also has an event associated with it.
|
||||
// c) The data transfer from device will be submitted with a dependence on
|
||||
// the kernel event. d) Finally wait on the host for all the events
|
||||
// associated with the data transfer from device.
|
||||
// The env-var also affects any "target update" constructs as well.
|
||||
// The env-var only affects the L0 copy/ compute commands issued from a
|
||||
// single target construct execution, not across multiple invocations.
|
||||
const StringEnvar CommandModeVar("LIBOMPTARGET_LEVEL_ZERO_COMMAND_MODE");
|
||||
if (CommandModeVar.isPresent()) {
|
||||
if (match(CommandModeVar, "sync"))
|
||||
CommandMode = CommandModeTy::Sync;
|
||||
else if (match(CommandModeVar, "async"))
|
||||
CommandMode = CommandModeTy::Async;
|
||||
else if (match(CommandModeVar, "async_ordered"))
|
||||
CommandMode = CommandModeTy::AsyncOrdered;
|
||||
else
|
||||
MESSAGE("Warning: Ignoring invalid value for "
|
||||
"LIBOMPTARGET_LEVEL_ZERO_COMMAND_MODE=%s\n",
|
||||
CommandModeVar.get().c_str());
|
||||
}
|
||||
|
||||
// Detect if we need to enable compatibility with Level Zero debug mode.
|
||||
ZeDebugEnabled = BoolEnvar("ZET_ENABLE_PROGRAM_DEBUGGING", false);
|
||||
}
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
248
offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
Normal file
248
offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
//===--- Target RTLs Implementation ---------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// RTL for SPIR-V/Xe machine.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <level_zero/zes_api.h>
|
||||
|
||||
#include "L0Device.h"
|
||||
#include "L0Interop.h"
|
||||
#include "L0Kernel.h"
|
||||
#include "L0Plugin.h"
|
||||
#include "L0Trace.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
using namespace llvm::omp::target;
|
||||
using namespace error;
|
||||
|
||||
#pragma clang diagnostic ignored "-Wglobal-constructors"
|
||||
// Common data across all possible plugin instantiations.
|
||||
L0OptionsTy LevelZeroPluginTy::Options;
|
||||
|
||||
Expected<int32_t> LevelZeroPluginTy::findDevices() {
|
||||
CALL_ZE_RET_ERROR(zeInit, ZE_INIT_FLAG_GPU_ONLY);
|
||||
uint32_t NumDrivers = 0;
|
||||
CALL_ZE_RET_ERROR(zeDriverGet, &NumDrivers, nullptr);
|
||||
if (NumDrivers == 0) {
|
||||
DP("Cannot find any drivers.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// We expect multiple drivers on Windows to support different device types,
|
||||
// so we need to maintain multiple drivers and contexts in general.
|
||||
llvm::SmallVector<ze_driver_handle_t> FoundDrivers(NumDrivers);
|
||||
CALL_ZE_RET_ERROR(zeDriverGet, &NumDrivers, FoundDrivers.data());
|
||||
|
||||
struct RootInfoTy {
|
||||
uint32_t OrderId;
|
||||
ze_device_handle_t ZeDevice;
|
||||
L0ContextTy *Driver;
|
||||
bool IsDiscrete;
|
||||
};
|
||||
llvm::SmallVector<RootInfoTy> RootDevices;
|
||||
|
||||
uint32_t OrderId = 0;
|
||||
for (uint32_t DriverId = 0; DriverId < NumDrivers; DriverId++) {
|
||||
const auto &Driver = FoundDrivers[DriverId];
|
||||
uint32_t DeviceCount = 0;
|
||||
ze_result_t RC;
|
||||
CALL_ZE(RC, zeDeviceGet, Driver, &DeviceCount, nullptr);
|
||||
if (RC != ZE_RESULT_SUCCESS || DeviceCount == 0) {
|
||||
DP("Cannot find any devices from driver " DPxMOD ".\n", DPxPTR(Driver));
|
||||
continue;
|
||||
}
|
||||
// We have a driver that supports at least one device.
|
||||
ContextList.emplace_back(*this, Driver, DriverId);
|
||||
auto &DrvInfo = ContextList.back();
|
||||
if (auto Err = DrvInfo.init())
|
||||
return std::move(Err);
|
||||
llvm::SmallVector<ze_device_handle_t> FoundDevices(DeviceCount);
|
||||
CALL_ZE_RET_ERROR(zeDeviceGet, Driver, &DeviceCount, FoundDevices.data());
|
||||
|
||||
for (auto &zeDevice : FoundDevices)
|
||||
RootDevices.push_back(
|
||||
{OrderId++, zeDevice, &DrvInfo, L0DeviceTy::isDiscrete(zeDevice)});
|
||||
}
|
||||
|
||||
// Move discrete devices to the front.
|
||||
std::sort(RootDevices.begin(), RootDevices.end(),
|
||||
[](const RootInfoTy &A, const RootInfoTy &B) {
|
||||
// If both are discrete, order by OrderId.
|
||||
// If both are not discrete, order by OrderId.
|
||||
// Otherwise, discrete goes first.
|
||||
|
||||
if (A.IsDiscrete && B.IsDiscrete)
|
||||
return A.OrderId < B.OrderId;
|
||||
if (!A.IsDiscrete && !B.IsDiscrete)
|
||||
return A.OrderId < B.OrderId;
|
||||
return A.IsDiscrete;
|
||||
});
|
||||
|
||||
for (size_t RootId = 0; RootId < RootDevices.size(); RootId++) {
|
||||
const auto ZeDevice = RootDevices[RootId].ZeDevice;
|
||||
auto *RootDriver = RootDevices[RootId].Driver;
|
||||
DetectedDevices.push_back(DeviceInfoTy{
|
||||
{ZeDevice, static_cast<int32_t>(RootId), -1, -1}, RootDriver});
|
||||
}
|
||||
int32_t NumDevices = DetectedDevices.size();
|
||||
|
||||
DP("Found %" PRIu32 " devices.\n", NumDevices);
|
||||
DP("List of devices (DeviceID[.SubID[.CCSID]])\n");
|
||||
for (auto &DeviceInfo : DetectedDevices) {
|
||||
(void)DeviceInfo; // Avoid unused variable warning in non-debug builds.
|
||||
DP("-- Device %" PRIu32 "%s%s\n", DeviceInfo.Id.RootId,
|
||||
(DeviceInfo.Id.SubId < 0
|
||||
? ""
|
||||
: ("." + std::to_string(DeviceInfo.Id.SubId)).c_str()),
|
||||
(DeviceInfo.Id.CCSId < 0
|
||||
? ""
|
||||
: ("." + std::to_string(DeviceInfo.Id.CCSId)).c_str()));
|
||||
}
|
||||
return NumDevices;
|
||||
}
|
||||
|
||||
Expected<int32_t> LevelZeroPluginTy::initImpl() {
|
||||
DP("Level0 NG plugin initialization\n");
|
||||
// Process options before anything else.
|
||||
Options.init();
|
||||
return findDevices();
|
||||
}
|
||||
|
||||
Error LevelZeroPluginTy::deinitImpl() {
|
||||
DP("Deinit Level0 plugin!\n");
|
||||
if (auto Err = ContextTLSTable.deinit())
|
||||
return Err;
|
||||
if (auto Err = DeviceTLSTable.deinit())
|
||||
return Err;
|
||||
for (auto &Context : ContextList)
|
||||
if (auto Err = Context.deinit())
|
||||
return Err;
|
||||
ContextList.clear();
|
||||
DP("Level0 plugin deinitialized successfully\n");
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
GenericDeviceTy *LevelZeroPluginTy::createDevice(GenericPluginTy &Plugin,
|
||||
int32_t DeviceId,
|
||||
int32_t NumDevices) {
|
||||
auto &DeviceInfo = DetectedDevices[DeviceId];
|
||||
auto RootId = DeviceInfo.Id.RootId;
|
||||
auto SubId = DeviceInfo.Id.SubId;
|
||||
auto CCSId = DeviceInfo.Id.CCSId;
|
||||
auto zeDevice = DeviceInfo.Id.zeId;
|
||||
auto *zeDriver = DeviceInfo.Driver;
|
||||
|
||||
std::string IdStr = std::to_string(RootId) +
|
||||
(SubId < 0 ? "" : "." + std::to_string(SubId)) +
|
||||
(CCSId < 0 ? "" : "." + std::to_string(CCSId));
|
||||
|
||||
return new L0DeviceTy(static_cast<LevelZeroPluginTy &>(Plugin), DeviceId,
|
||||
NumDevices, zeDevice, *zeDriver, std::move(IdStr),
|
||||
CCSId < 0 ? 0 : CCSId /* ComputeIndex */);
|
||||
}
|
||||
|
||||
GenericGlobalHandlerTy *LevelZeroPluginTy::createGlobalHandler() {
|
||||
return new L0GlobalHandlerTy();
|
||||
}
|
||||
|
||||
Error LevelZeroPluginTy::flushQueueImpl(omp_interop_val_t *Interop) {
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Expected<bool> LevelZeroPluginTy::isELFCompatible(uint32_t DeviceId,
|
||||
StringRef Image) const {
|
||||
uint64_t MajorVer, MinorVer;
|
||||
return isValidOneOmpImage(Image, MajorVer, MinorVer);
|
||||
}
|
||||
|
||||
Error LevelZeroPluginTy::syncBarrierImpl(omp_interop_val_t *Interop) {
|
||||
if (!Interop) {
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Invalid/inconsistent OpenMP interop " DPxMOD "\n",
|
||||
DPxPTR(Interop));
|
||||
}
|
||||
if (!Interop->async_info || !Interop->async_info->Queue)
|
||||
return Plugin::success();
|
||||
|
||||
const auto L0 = static_cast<L0Interop::Property *>(Interop->rtl_property);
|
||||
const auto device_id = Interop->device_id;
|
||||
auto &l0Device = getDeviceFromId(device_id);
|
||||
|
||||
// We can synchronize both L0 & SYCL objects with the same ze command.
|
||||
if (l0Device.useImmForInterop()) {
|
||||
DP("LevelZeroPluginTy::sync_barrier: Synchronizing " DPxMOD
|
||||
" with ImmCmdList barrier\n",
|
||||
DPxPTR(Interop));
|
||||
auto ImmCmdList = L0->ImmCmdList;
|
||||
|
||||
CALL_ZE_RET_ERROR(zeCommandListHostSynchronize, ImmCmdList,
|
||||
L0DefaultTimeout);
|
||||
} else {
|
||||
DP("LevelZeroPluginTy::sync_barrier: Synchronizing " DPxMOD
|
||||
" with queue synchronize\n",
|
||||
DPxPTR(Interop));
|
||||
auto CmdQueue = L0->CommandQueue;
|
||||
CALL_ZE_RET_ERROR(zeCommandQueueSynchronize, CmdQueue, L0DefaultTimeout);
|
||||
}
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error LevelZeroPluginTy::asyncBarrierImpl(omp_interop_val_t *Interop) {
|
||||
if (!Interop) {
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Invalid/inconsistent OpenMP interop " DPxMOD "\n",
|
||||
DPxPTR(Interop));
|
||||
}
|
||||
if (!Interop->async_info || !Interop->async_info->Queue)
|
||||
return Plugin::success();
|
||||
|
||||
const auto L0 = static_cast<L0Interop::Property *>(Interop->rtl_property);
|
||||
const auto device_id = Interop->device_id;
|
||||
if (Interop->attrs.inorder)
|
||||
return Plugin::success();
|
||||
|
||||
auto &l0Device = getDeviceFromId(device_id);
|
||||
if (l0Device.useImmForInterop()) {
|
||||
DP("LevelZeroPluginTy::async_barrier: Appending ImmCmdList barrier "
|
||||
"to " DPxMOD "\n",
|
||||
DPxPTR(Interop));
|
||||
auto ImmCmdList = L0->ImmCmdList;
|
||||
CALL_ZE_RET_ERROR(zeCommandListAppendBarrier, ImmCmdList, nullptr, 0,
|
||||
nullptr);
|
||||
} else {
|
||||
#if 0
|
||||
// TODO: re-enable once we have a way to delay the CmdList reset .
|
||||
DP("LevelZeroPluginTy::async_barrier: Appending CmdList barrier to " DPxMOD
|
||||
"\n",
|
||||
DPxPTR(Interop));
|
||||
auto CmdQueue = L0->CommandQueue;
|
||||
ze_command_list_handle_t CmdList = l0Device.getCmdList();
|
||||
CALL_ZE_RET_ERROR(zeCommandListAppendBarrier, CmdList, nullptr, 0, nullptr);
|
||||
CALL_ZE_RET_ERROR(zeCommandListClose, CmdList);
|
||||
CALL_ZE_RET_ERROR(zeCommandQueueExecuteCommandLists, CmdQueue, 1, &CmdList,
|
||||
nullptr);
|
||||
CALL_ZE_RET_ERROR(zeCommandListReset, CmdList);
|
||||
#else
|
||||
return syncBarrierImpl(Interop);
|
||||
#endif
|
||||
}
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
|
||||
extern "C" {
|
||||
llvm::omp::target::plugin::GenericPluginTy *createPlugin_level_zero() {
|
||||
return new llvm::omp::target::plugin::LevelZeroPluginTy();
|
||||
}
|
||||
}
|
||||
550
offload/plugins-nextgen/level_zero/src/L0Program.cpp
Normal file
550
offload/plugins-nextgen/level_zero/src/L0Program.cpp
Normal file
|
|
@ -0,0 +1,550 @@
|
|||
//===--- Level Zero Target RTL Implementation -----------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Level Zero Program abstraction.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <fstream>
|
||||
#ifdef _WIN32
|
||||
#include <fcntl.h>
|
||||
#include <io.h>
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#endif // !_WIN32
|
||||
|
||||
#include "L0Plugin.h"
|
||||
#include "L0Program.h"
|
||||
|
||||
namespace llvm::omp::target::plugin {
|
||||
|
||||
Error L0GlobalHandlerTy::getGlobalMetadataFromDevice(GenericDeviceTy &Device,
|
||||
DeviceImageTy &Image,
|
||||
GlobalTy &DeviceGlobal) {
|
||||
const char *GlobalName = DeviceGlobal.getName().data();
|
||||
|
||||
L0ProgramTy &Program = L0ProgramTy::makeL0Program(Image);
|
||||
auto AddrOrErr = Program.getOffloadVarDeviceAddr(GlobalName);
|
||||
if (!AddrOrErr)
|
||||
return AddrOrErr.takeError();
|
||||
|
||||
// Save the pointer to the symbol allowing nullptr.
|
||||
DeviceGlobal.setPtr(*AddrOrErr);
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
inline L0DeviceTy &L0ProgramTy::getL0Device() const {
|
||||
return L0DeviceTy::makeL0Device(getDevice());
|
||||
}
|
||||
|
||||
Error L0ProgramTy::deinit() {
|
||||
for (auto *Kernel : Kernels) {
|
||||
if (auto Err = Kernel->deinit())
|
||||
return Err;
|
||||
getL0Device().getPlugin().free(Kernel);
|
||||
}
|
||||
for (auto Module : Modules) {
|
||||
CALL_ZE_RET_ERROR(zeModuleDestroy, Module);
|
||||
}
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
void L0ProgramTy::setLibModule() {
|
||||
#if _WIN32
|
||||
return;
|
||||
#else
|
||||
// Check if the image belongs to a dynamic library.
|
||||
Dl_info DLI{nullptr, nullptr, nullptr, nullptr};
|
||||
if (dladdr(getStart(), &DLI) && DLI.dli_fname) {
|
||||
std::vector<uint8_t> FileBin;
|
||||
auto Size = readFile(DLI.dli_fname, FileBin);
|
||||
if (Size) {
|
||||
auto MB = MemoryBuffer::getMemBuffer(
|
||||
StringRef(reinterpret_cast<const char *>(FileBin.data()), Size),
|
||||
/*BufferName=*/"", /*RequiresNullTerminator=*/false);
|
||||
auto ELF = ELFObjectFileBase::createELFObjectFile(MB->getMemBufferRef());
|
||||
if (ELF) {
|
||||
if (auto *Obj = dyn_cast<ELF64LEObjectFile>((*ELF).get())) {
|
||||
const auto Header = Obj->getELFFile().getHeader();
|
||||
if (Header.e_type == ELF::ET_DYN) {
|
||||
DP("Processing current image as library\n");
|
||||
IsLibModule = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // _WIN32
|
||||
}
|
||||
|
||||
Error L0ProgramTy::addModule(size_t Size, const uint8_t *Image,
|
||||
const std::string_view CommonBuildOptions,
|
||||
ze_module_format_t Format) {
|
||||
const ze_module_constants_t SpecConstants =
|
||||
LevelZeroPluginTy::getOptions().CommonSpecConstants.getModuleConstants();
|
||||
auto &l0Device = getL0Device();
|
||||
std::string BuildOptions(CommonBuildOptions);
|
||||
|
||||
// Add required flag to enable dynamic linking.
|
||||
if (IsLibModule)
|
||||
BuildOptions += " -library-compilation ";
|
||||
|
||||
ze_module_desc_t ModuleDesc{};
|
||||
ModuleDesc.stype = ZE_STRUCTURE_TYPE_MODULE_DESC;
|
||||
ModuleDesc.pNext = nullptr;
|
||||
ModuleDesc.format = Format;
|
||||
ze_module_handle_t Module = nullptr;
|
||||
ze_module_build_log_handle_t BuildLog = nullptr;
|
||||
|
||||
// Build a single module from a single image.
|
||||
ModuleDesc.inputSize = Size;
|
||||
ModuleDesc.pInputModule = Image;
|
||||
ModuleDesc.pBuildFlags = BuildOptions.c_str();
|
||||
ModuleDesc.pConstants = &SpecConstants;
|
||||
CALL_ZE_RET_ERROR(zeModuleCreate, l0Device.getZeContext(),
|
||||
l0Device.getZeDevice(), &ModuleDesc, &Module, &BuildLog);
|
||||
|
||||
// Check if module link is required. We do not need this check for
|
||||
// library module.
|
||||
if (!RequiresModuleLink && !IsLibModule) {
|
||||
ze_module_properties_t Properties = {ZE_STRUCTURE_TYPE_MODULE_PROPERTIES,
|
||||
nullptr, 0};
|
||||
CALL_ZE_RET_ERROR(zeModuleGetProperties, Module, &Properties);
|
||||
RequiresModuleLink = Properties.flags & ZE_MODULE_PROPERTY_FLAG_IMPORTS;
|
||||
}
|
||||
// For now, assume the first module contains libraries, globals.
|
||||
if (Modules.empty())
|
||||
GlobalModule = Module;
|
||||
Modules.push_back(Module);
|
||||
l0Device.addGlobalModule(Module);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
Error L0ProgramTy::linkModules() {
|
||||
auto &l0Device = getL0Device();
|
||||
if (!RequiresModuleLink) {
|
||||
DP("Module link is not required\n");
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
if (Modules.empty())
|
||||
return Plugin::error(ErrorCode::UNKNOWN,
|
||||
"Invalid number of modules when linking modules");
|
||||
|
||||
ze_module_build_log_handle_t LinkLog = nullptr;
|
||||
CALL_ZE_RET_ERROR(zeModuleDynamicLink,
|
||||
static_cast<uint32_t>(l0Device.getNumGlobalModules()),
|
||||
l0Device.getGlobalModulesArray(), &LinkLog);
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
size_t L0ProgramTy::readFile(const char *FileName,
|
||||
std::vector<uint8_t> &OutFile) const {
|
||||
std::ifstream IFS(FileName, std::ios::binary);
|
||||
if (!IFS.good())
|
||||
return 0;
|
||||
IFS.seekg(0, IFS.end);
|
||||
auto FileSize = static_cast<size_t>(IFS.tellg());
|
||||
OutFile.resize(FileSize);
|
||||
IFS.seekg(0);
|
||||
if (!IFS.read(reinterpret_cast<char *>(OutFile.data()), FileSize)) {
|
||||
OutFile.clear();
|
||||
return 0;
|
||||
}
|
||||
return FileSize;
|
||||
}
|
||||
|
||||
void L0ProgramTy::replaceDriverOptsWithBackendOpts(const L0DeviceTy &Device,
|
||||
std::string &Options) const {
|
||||
// Options that need to be replaced with backend-specific options.
|
||||
static const struct {
|
||||
std::string Option;
|
||||
std::string BackendOption;
|
||||
} OptionTranslationTable[] = {
|
||||
{"-ftarget-compile-fast",
|
||||
"-igc_opts 'PartitionUnit=1,SubroutineThreshold=50000'"},
|
||||
{"-foffload-fp32-prec-div", "-ze-fp32-correctly-rounded-divide-sqrt"},
|
||||
{"-foffload-fp32-prec-sqrt", "-ze-fp32-correctly-rounded-divide-sqrt"},
|
||||
};
|
||||
|
||||
for (const auto &OptPair : OptionTranslationTable) {
|
||||
const size_t Pos = Options.find(OptPair.Option);
|
||||
if (Pos != std::string::npos)
|
||||
Options.replace(Pos, OptPair.Option.length(), OptPair.BackendOption);
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME: move this to llvm/BinaryFormat/ELF.h and elf.h:
|
||||
#define NT_INTEL_ONEOMP_OFFLOAD_VERSION 1
|
||||
#define NT_INTEL_ONEOMP_OFFLOAD_IMAGE_COUNT 2
|
||||
#define NT_INTEL_ONEOMP_OFFLOAD_IMAGE_AUX 3
|
||||
|
||||
bool isValidOneOmpImage(StringRef Image, uint64_t &MajorVer,
|
||||
uint64_t &MinorVer) {
|
||||
const auto MB = MemoryBuffer::getMemBuffer(Image,
|
||||
/*BufferName=*/"",
|
||||
/*RequiresNullTerminator=*/false);
|
||||
auto ExpectedNewE =
|
||||
ELFObjectFileBase::createELFObjectFile(MB->getMemBufferRef());
|
||||
if (!ExpectedNewE) {
|
||||
DP("Warning: unable to get ELF handle!\n");
|
||||
return false;
|
||||
}
|
||||
bool Res = false;
|
||||
auto processObjF = [&](const auto ELFObjF) {
|
||||
if (!ELFObjF) {
|
||||
DP("Warning: Unexpected ELF type!\n");
|
||||
return false;
|
||||
}
|
||||
const auto &ELFF = ELFObjF->getELFFile();
|
||||
auto Sections = ELFF.sections();
|
||||
if (!Sections) {
|
||||
DP("Warning: unable to get ELF sections!\n");
|
||||
return false;
|
||||
}
|
||||
bool SeenOffloadSection = false;
|
||||
for (auto Sec : *Sections) {
|
||||
if (Sec.sh_type != ELF::SHT_NOTE)
|
||||
continue;
|
||||
Error Err = Plugin::success();
|
||||
for (auto Note : ELFF.notes(Sec, Err)) {
|
||||
if (Err) {
|
||||
DP("Warning: unable to get ELF notes handle!\n");
|
||||
return false;
|
||||
}
|
||||
if (Note.getName() != "INTELONEOMPOFFLOAD")
|
||||
continue;
|
||||
SeenOffloadSection = true;
|
||||
if (Note.getType() != NT_INTEL_ONEOMP_OFFLOAD_VERSION)
|
||||
continue;
|
||||
|
||||
std::string DescStr(std::move(Note.getDescAsStringRef(4).str()));
|
||||
const auto DelimPos = DescStr.find('.');
|
||||
if (DelimPos == std::string::npos) {
|
||||
// The version has to look like "Major#.Minor#".
|
||||
DP("Invalid NT_INTEL_ONEOMP_OFFLOAD_VERSION: '%s'\n",
|
||||
DescStr.c_str());
|
||||
return false;
|
||||
}
|
||||
const std::string MajorVerStr = DescStr.substr(0, DelimPos);
|
||||
DescStr.erase(0, DelimPos + 1);
|
||||
MajorVer = std::stoull(MajorVerStr);
|
||||
MinorVer = std::stoull(DescStr);
|
||||
return (MajorVer == 1 && MinorVer == 0);
|
||||
}
|
||||
}
|
||||
return SeenOffloadSection;
|
||||
};
|
||||
if (const auto *O = dyn_cast<ELF64LEObjectFile>((*ExpectedNewE).get())) {
|
||||
Res = processObjF(O);
|
||||
} else if (const auto *O =
|
||||
dyn_cast<ELF32LEObjectFile>((*ExpectedNewE).get())) {
|
||||
Res = processObjF(O);
|
||||
} else {
|
||||
assert(false && "Unexpected ELF format");
|
||||
}
|
||||
return Res;
|
||||
}
|
||||
|
||||
Error L0ProgramTy::buildModules(const std::string_view BuildOptions) {
|
||||
auto &l0Device = getL0Device();
|
||||
auto Image = getMemoryBuffer();
|
||||
if (identify_magic(Image.getBuffer()) == file_magic::spirv_object) {
|
||||
// Handle legacy plain SPIR-V image.
|
||||
const uint8_t *ImgBegin = reinterpret_cast<const uint8_t *>(getStart());
|
||||
return addModule(getSize(), ImgBegin, BuildOptions,
|
||||
ZE_MODULE_FORMAT_IL_SPIRV);
|
||||
}
|
||||
|
||||
uint64_t MajorVer, MinorVer;
|
||||
if (!isValidOneOmpImage(Image.getBuffer(), MajorVer, MinorVer)) {
|
||||
DP("Warning: image is not a valid oneAPI OpenMP image.\n");
|
||||
return Plugin::error(ErrorCode::UNKNOWN, "Invalid oneAPI OpenMP image");
|
||||
}
|
||||
|
||||
setLibModule();
|
||||
|
||||
// Iterate over the images and pick the first one that fits.
|
||||
uint64_t ImageCount = 0;
|
||||
struct V1ImageInfo {
|
||||
// 0 - native, 1 - SPIR-V.
|
||||
uint64_t Format = std::numeric_limits<uint64_t>::max();
|
||||
std::string CompileOpts;
|
||||
std::string LinkOpts;
|
||||
// We may have multiple sections created from split-kernel mode.
|
||||
std::vector<const uint8_t *> PartBegin;
|
||||
std::vector<uint64_t> PartSize;
|
||||
|
||||
V1ImageInfo(uint64_t Format, std::string CompileOpts, std::string LinkOpts)
|
||||
: Format(Format), CompileOpts(std::move(CompileOpts)),
|
||||
LinkOpts(std::move(LinkOpts)) {}
|
||||
};
|
||||
std::unordered_map<uint64_t, V1ImageInfo> AuxInfo;
|
||||
|
||||
auto ExpectedNewE = ELFObjectFileBase::createELFObjectFile(Image);
|
||||
assert(ExpectedNewE &&
|
||||
"isValidOneOmpImage() returns true for invalid ELF image");
|
||||
auto processELF = [&](auto *EObj) {
|
||||
assert(EObj && "isValidOneOmpImage() returns true for invalid ELF image.");
|
||||
const auto &E = EObj->getELFFile();
|
||||
// Collect auxiliary information.
|
||||
uint64_t MaxImageIdx = 0;
|
||||
|
||||
auto Sections = E.sections();
|
||||
assert(Sections && "isValidOneOmpImage() returns true for ELF image with "
|
||||
"invalid sections.");
|
||||
|
||||
for (auto Sec : *Sections) {
|
||||
if (Sec.sh_type != ELF::SHT_NOTE)
|
||||
continue;
|
||||
Error Err = Plugin::success();
|
||||
for (auto Note : E.notes(Sec, Err)) {
|
||||
assert(!Err && "isValidOneOmpImage() returns true for ELF image with "
|
||||
"invalid notes.");
|
||||
if (Note.getName().str() != "INTELONEOMPOFFLOAD")
|
||||
continue;
|
||||
|
||||
const uint64_t Type = Note.getType();
|
||||
auto DescStrRef = Note.getDescAsStringRef(4);
|
||||
switch (Type) {
|
||||
default:
|
||||
DP("Warning: unrecognized INTELONEOMPOFFLOAD note.\n");
|
||||
break;
|
||||
case NT_INTEL_ONEOMP_OFFLOAD_VERSION:
|
||||
break;
|
||||
case NT_INTEL_ONEOMP_OFFLOAD_IMAGE_COUNT:
|
||||
if (DescStrRef.getAsInteger(10, ImageCount)) {
|
||||
DP("Warning: invalid NT_INTEL_ONEOMP_OFFLOAD_IMAGE_COUNT: '%s'\n",
|
||||
DescStrRef.str().c_str());
|
||||
ImageCount = 0;
|
||||
}
|
||||
break;
|
||||
case NT_INTEL_ONEOMP_OFFLOAD_IMAGE_AUX:
|
||||
llvm::SmallVector<llvm::StringRef, 4> Parts;
|
||||
DescStrRef.split(Parts, '\0', /* MaxSplit = */ 4,
|
||||
/* KeepEmpty = */ true);
|
||||
|
||||
// Ignore records with less than 4 strings.
|
||||
if (Parts.size() != 4) {
|
||||
DP("Warning: short NT_INTEL_ONEOMP_OFFLOAD_IMAGE_AUX "
|
||||
"record is ignored.\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t Idx = 0;
|
||||
if (Parts[0].getAsInteger(10, Idx)) {
|
||||
DP("Warning: ignoring auxiliary information (invalid index "
|
||||
"'%s').\n",
|
||||
Parts[0].str().c_str());
|
||||
continue;
|
||||
}
|
||||
MaxImageIdx = (std::max)(MaxImageIdx, Idx);
|
||||
if (AuxInfo.find(Idx) != AuxInfo.end()) {
|
||||
DP("Warning: duplicate auxiliary information for image %" PRIu64
|
||||
" is ignored.\n",
|
||||
Idx);
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t Part1Id;
|
||||
if (Parts[1].getAsInteger(10, Part1Id)) {
|
||||
DP("Warning: ignoring auxiliary information (invalid part id "
|
||||
"'%s').\n",
|
||||
Parts[1].str().c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
AuxInfo.emplace(
|
||||
std::piecewise_construct, std::forward_as_tuple(Idx),
|
||||
std::forward_as_tuple(Part1Id, Parts[2].str(), Parts[3].str()));
|
||||
// Image pointer and size will be initialized later.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (MaxImageIdx >= ImageCount)
|
||||
DP("Warning: invalid image index found in auxiliary information.\n");
|
||||
|
||||
for (auto Sec : *Sections) {
|
||||
const char *Prefix = "__openmp_offload_spirv_";
|
||||
auto ExpectedSectionName = E.getSectionName(Sec);
|
||||
assert(ExpectedSectionName && "isValidOneOmpImage() returns true for ELF "
|
||||
"image with invalid section names");
|
||||
auto &SectionNameRef = *ExpectedSectionName;
|
||||
if (!SectionNameRef.consume_front(Prefix))
|
||||
continue;
|
||||
|
||||
// Expected section name in split-kernel mode with the following pattern:
|
||||
// __openmp_offload_spirv_<image_id>_<part_id>
|
||||
auto Parts = SectionNameRef.split('_');
|
||||
// It seems that we do not need part ID as long as they are ordered
|
||||
// in the image and we keep the ordering in the runtime.
|
||||
SectionNameRef = Parts.first;
|
||||
if (Parts.second.empty()) {
|
||||
DP("Found a single section in the image\n");
|
||||
} else {
|
||||
DP("Found a split section in the image\n");
|
||||
}
|
||||
|
||||
uint64_t Idx = 0;
|
||||
if (SectionNameRef.getAsInteger(10, Idx)) {
|
||||
DP("Warning: ignoring image section (invalid index '%s').\n",
|
||||
SectionNameRef.str().c_str());
|
||||
continue;
|
||||
}
|
||||
if (Idx >= ImageCount) {
|
||||
DP("Warning: ignoring image section (index %" PRIu64
|
||||
" is out of range).\n",
|
||||
Idx);
|
||||
continue;
|
||||
}
|
||||
|
||||
auto AuxInfoIt = AuxInfo.find(Idx);
|
||||
if (AuxInfoIt == AuxInfo.end()) {
|
||||
DP("Warning: ignoring image section (no aux info).\n");
|
||||
continue;
|
||||
}
|
||||
auto Contents = E.getSectionContents(Sec);
|
||||
assert(Contents);
|
||||
AuxInfoIt->second.PartBegin.push_back((*Contents).data());
|
||||
AuxInfoIt->second.PartSize.push_back(Sec.sh_size);
|
||||
}
|
||||
};
|
||||
|
||||
if (auto *O = dyn_cast<ELF64LEObjectFile>((*ExpectedNewE).get())) {
|
||||
processELF(O);
|
||||
} else if (auto *O = dyn_cast<ELF32LEObjectFile>((*ExpectedNewE).get())) {
|
||||
processELF(O);
|
||||
} else {
|
||||
assert(false && "Unexpected ELF format");
|
||||
}
|
||||
|
||||
for (uint64_t Idx = 0; Idx < ImageCount; ++Idx) {
|
||||
const auto It = AuxInfo.find(Idx);
|
||||
if (It == AuxInfo.end()) {
|
||||
DP("Warning: image %" PRIu64
|
||||
" without auxiliary information is ingored.\n",
|
||||
Idx);
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto NumParts = It->second.PartBegin.size();
|
||||
// Split-kernel is not supported in SPIRV format.
|
||||
if (NumParts > 1 && It->second.Format != 0) {
|
||||
DP("Warning: split-kernel images are not supported in SPIRV format\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip unknown image format.
|
||||
if (It->second.Format != 0 && It->second.Format != 1) {
|
||||
DP("Warning: image %" PRIu64 "is ignored due to unknown format.\n", Idx);
|
||||
continue;
|
||||
}
|
||||
|
||||
const bool IsBinary = (It->second.Format == 0);
|
||||
const auto ModuleFormat =
|
||||
IsBinary ? ZE_MODULE_FORMAT_NATIVE : ZE_MODULE_FORMAT_IL_SPIRV;
|
||||
std::string Options(BuildOptions);
|
||||
{
|
||||
Options += " " + It->second.CompileOpts + " " + It->second.LinkOpts;
|
||||
replaceDriverOptsWithBackendOpts(l0Device, Options);
|
||||
}
|
||||
|
||||
for (size_t I = 0; I < NumParts; I++) {
|
||||
const unsigned char *ImgBegin =
|
||||
reinterpret_cast<const unsigned char *>(It->second.PartBegin[I]);
|
||||
size_t ImgSize = It->second.PartSize[I];
|
||||
|
||||
DP("Creating module from %s image part #%" PRIu64 "-%zu.\n",
|
||||
IsBinary ? "Binary" : "SPIR-V", Idx, I);
|
||||
if (auto Err = addModule(ImgSize, ImgBegin, Options, ModuleFormat))
|
||||
return Err;
|
||||
}
|
||||
DP("Created module from image #%" PRIu64 ".\n", Idx);
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
return Plugin::error(ErrorCode::UNKNOWN, "Failed to create program modules.");
|
||||
}
|
||||
|
||||
Expected<void *> L0ProgramTy::getOffloadVarDeviceAddr(const char *CName) const {
|
||||
DP("Looking up OpenMP global variable '%s'.\n", CName);
|
||||
|
||||
if (!GlobalModule || !CName)
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Invalid arguments to getOffloadVarDeviceAddr");
|
||||
|
||||
std::string Name(CName);
|
||||
size_t SizeDummy = 0;
|
||||
void *DevicePtr = nullptr;
|
||||
ze_result_t RC;
|
||||
for (auto Module : Modules) {
|
||||
CALL_ZE(RC, zeModuleGetGlobalPointer, Module, Name.c_str(), &SizeDummy,
|
||||
&DevicePtr);
|
||||
if (RC == ZE_RESULT_SUCCESS && DevicePtr)
|
||||
return DevicePtr;
|
||||
}
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Global variable '%s' not found on device",
|
||||
Name.c_str());
|
||||
}
|
||||
|
||||
Error L0ProgramTy::readGlobalVariable(const char *Name, size_t Size,
|
||||
void *HostPtr) {
|
||||
size_t SizeDummy = 0;
|
||||
void *DevicePtr = nullptr;
|
||||
ze_result_t RC;
|
||||
CALL_ZE(RC, zeModuleGetGlobalPointer, GlobalModule, Name, &SizeDummy,
|
||||
&DevicePtr);
|
||||
if (RC != ZE_RESULT_SUCCESS || !DevicePtr) {
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Cannot read from device global variable %s", Name);
|
||||
}
|
||||
return getL0Device().enqueueMemCopy(HostPtr, DevicePtr, Size);
|
||||
}
|
||||
|
||||
Error L0ProgramTy::writeGlobalVariable(const char *Name, size_t Size,
|
||||
const void *HostPtr) {
|
||||
size_t SizeDummy = 0;
|
||||
void *DevicePtr = nullptr;
|
||||
ze_result_t RC;
|
||||
CALL_ZE(RC, zeModuleGetGlobalPointer, GlobalModule, Name, &SizeDummy,
|
||||
&DevicePtr);
|
||||
if (RC != ZE_RESULT_SUCCESS || !DevicePtr) {
|
||||
return Plugin::error(ErrorCode::INVALID_ARGUMENT,
|
||||
"Cannot write to device global variable %s", Name);
|
||||
}
|
||||
return getL0Device().enqueueMemCopy(DevicePtr, HostPtr, Size);
|
||||
}
|
||||
|
||||
Error L0ProgramTy::loadModuleKernels() {
|
||||
// We need to build kernels here before filling the offload entries since we
|
||||
// don't know which module contains a specific kernel with a name.
|
||||
for (auto Module : Modules) {
|
||||
uint32_t Count = 0;
|
||||
CALL_ZE_RET_ERROR(zeModuleGetKernelNames, Module, &Count,
|
||||
/*Names=*/nullptr);
|
||||
if (Count == 0)
|
||||
continue;
|
||||
|
||||
llvm::SmallVector<const char *> Names(Count);
|
||||
CALL_ZE_RET_ERROR(zeModuleGetKernelNames, Module, &Count, Names.data());
|
||||
|
||||
for (auto *Name : Names) {
|
||||
KernelsToModuleMap.emplace(Name, Module);
|
||||
}
|
||||
}
|
||||
|
||||
return Plugin::success();
|
||||
}
|
||||
|
||||
} // namespace llvm::omp::target::plugin
|
||||
Loading…
Reference in a new issue