[flang][cuda] Restrict managed data transfers to whole-array assignments (#212593)
Assignments involving managed or unified data are lowered to `cuf.data_transfer`, a synchronous copy. The predicate only excluded scalar left-hand sides, so array sections were transferred too, turning a loop that assigns one section per iteration into a sequence of blocking copies. Key the decision on the managed operand instead: a transfer when the managed variable or array is whole, host code when it is a section or an element. Managed data is host-addressable, so the host assignment is safe. Managed function results and assignments involving device memory are unaffected. This is the behavior specified in the CUDA Fortran Programming Guide 3.4.1: > An assignment statement where the right hand side is a managed array section and > the left hand side is any host or managed variable copies data using generated > host code.
This commit is contained in:
Zhen Wang
GitHub
parent
e12e16f6e0
commit
eb64c21b79
3 changed files with 265 additions and 38 deletions
|
|
@ -1392,6 +1392,15 @@ template <typename A> inline bool HasCUDADeviceAttrs(const A &expr) {
|
|||
return GetNbOfCUDADeviceSymbols(expr) > 0;
|
||||
}
|
||||
|
||||
// True for a whole reference to a managed or unified array: a whole array
|
||||
// variable, or a whole array component that itself has the attribute (a%b where
|
||||
// b is managed). An array section, an array element, a component of a managed
|
||||
// object and a computed value are all false.
|
||||
template <typename A> inline bool IsWholeManagedArray(const A &expr) {
|
||||
const Symbol *sym{UnwrapWholeSymbolOrComponentDataRef(expr)};
|
||||
return expr.Rank() > 0 && sym && IsCUDAManagedOrUnifiedSymbol(*sym);
|
||||
}
|
||||
|
||||
// Check if any of the symbols part of the lhs or rhs expression has a CUDA
|
||||
// device attribute.
|
||||
template <typename A, typename B>
|
||||
|
|
@ -1403,30 +1412,33 @@ inline bool IsCUDADataTransfer(const A &lhs, const B &rhs) {
|
|||
if (HasNonAllocatableModuleCUDAManagedSymbols(lhs))
|
||||
return false;
|
||||
|
||||
if (lhsNbManagedSymbols >= 1 && lhs.Rank() > 0 && rhsNbSymbols == 0 &&
|
||||
rhsNbManagedSymbols == 0 && (IsVariable(rhs) || IsConstantExpr(rhs))) {
|
||||
return true; // Managed arrays initialization is performed on the device.
|
||||
// CUDA Fortran Programming Guide 3.4.1: an assignment involving managed or
|
||||
// unified data is a copy when the managed side is a whole variable or array,
|
||||
// and is done on the host when it is a section. The host can read and write
|
||||
// managed data directly, and copying section by section in a loop is slow.
|
||||
bool wholeLhs{IsWholeManagedArray(lhs)};
|
||||
bool wholeRhs{IsWholeManagedArray(rhs)};
|
||||
|
||||
if (wholeLhs && rhsNbSymbols == 0 && rhsNbManagedSymbols == 0 &&
|
||||
(IsVariable(rhs) || IsConstantExpr(rhs))) {
|
||||
return true; // Initializing a whole managed array is done on the device.
|
||||
}
|
||||
|
||||
// Managed/unified data is host-addressable, so several assignments are
|
||||
// performed on the host and need no explicit data transfer:
|
||||
// - A whole-allocatable left-hand side involving managed/unified data: the
|
||||
// assignment has reallocation semantics and is performed on the host.
|
||||
// - Element-wise (scalar) access to managed/unified data.
|
||||
// - A right-hand side expression involving managed/unified data assigned into
|
||||
// a host-addressable (managed/unified or host) left-hand side: evaluating
|
||||
// it on the host avoids materializing a temporary.
|
||||
// - A managed/unified left-hand side assigned from host-only data.
|
||||
// A whole-array assignment whose right-hand side is a managed/unified
|
||||
// variable is a synchronous data transfer that waits for previously launched
|
||||
// kernels.
|
||||
// Assignments done on the host, with no copy:
|
||||
// - A whole allocatable left-hand side: the assignment may reallocate it,
|
||||
// which is done on the host.
|
||||
// - No managed operand is a whole array: they are all sections or elements,
|
||||
// which the host can read and write in place.
|
||||
// - An expression involving managed data assigned to a managed or host
|
||||
// left-hand side: evaluating it on the host avoids a temporary.
|
||||
// - A managed left-hand side assigned from host-only data.
|
||||
if ((IsAllocatableDesignator(lhs) &&
|
||||
(lhsNbManagedSymbols >= 1 || rhsNbManagedSymbols >= 1)) ||
|
||||
(lhsNbManagedSymbols >= 1 && rhsNbManagedSymbols == rhsNbSymbols &&
|
||||
lhs.Rank() == 0) ||
|
||||
!(wholeLhs || wholeRhs)) ||
|
||||
(lhsNbManagedSymbols == 0 && !HasCUDADeviceAttrs(lhs) &&
|
||||
rhsNbManagedSymbols >= 1 && rhsNbManagedSymbols == rhsNbSymbols &&
|
||||
lhs.Rank() == 0) ||
|
||||
!wholeRhs) ||
|
||||
(rhsNbManagedSymbols >= 1 && !IsVariable(rhs) &&
|
||||
(lhsNbManagedSymbols >= 1 || !HasCUDADeviceAttrs(lhs))) ||
|
||||
(lhsNbManagedSymbols >= 1 && rhsNbSymbols == 0)) {
|
||||
|
|
|
|||
|
|
@ -634,6 +634,9 @@ end subroutine
|
|||
! CHECK-LABEL: func.func @_QPsub33
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A section reference is assigned on the host even when it covers the whole array:
|
||||
! what matters is the form of the reference, not the extent it happens to cover
|
||||
! (CUDA Fortran Programming Guide 3.4.1).
|
||||
subroutine sub34(n)
|
||||
integer :: n
|
||||
real(2), managed, allocatable :: dx(:)
|
||||
|
|
@ -642,7 +645,7 @@ subroutine sub34(n)
|
|||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPsub34
|
||||
! CHECK: cuf.data_transfer %{{.*}} to %{{.*}} {hasManagedOrUnifedSymbols, transfer_kind = #cuf.cuda_transfer<host_device>} : f16, !fir.box<!fir.array<?xf16>>
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
module managed_mod
|
||||
integer, managed :: marray(10)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
|
||||
|
||||
! A whole-array assignment whose right-hand side is a managed variable or a
|
||||
! managed function result is a synchronous data transfer (matching CUDA Fortran
|
||||
! assignment-statement semantics). Element-wise (scalar) accesses and right-hand
|
||||
! side expressions involving managed data are performed on the host and need no
|
||||
! transfer.
|
||||
! CUDA Fortran Programming Guide 3.4.1: an assignment involving managed data is a
|
||||
! copy when the managed side is a whole variable or array, and is done on the host
|
||||
! when it is a section. Sections are still copied when the other side is device
|
||||
! data, which the host cannot access directly. A managed function result may come
|
||||
! from a kernel that is still running, so reading it is also a copy.
|
||||
|
||||
module mfr
|
||||
contains
|
||||
|
|
@ -16,11 +16,13 @@ contains
|
|||
end function
|
||||
end module
|
||||
|
||||
! Assignments that are data transfers.
|
||||
|
||||
subroutine managed_array_assign()
|
||||
integer(4), managed :: ma(16), mb(16)
|
||||
integer(4) :: ha(16)
|
||||
ma = mb ! managed = managed
|
||||
ma = ha ! managed = host
|
||||
ma = ha ! managed = host array
|
||||
ha = ma ! host = managed
|
||||
end subroutine
|
||||
|
||||
|
|
@ -29,37 +31,247 @@ end subroutine
|
|||
! CHECK: cuf.data_transfer %{{.*}} to %{{.*}} {hasManagedOrUnifedSymbols, transfer_kind = #cuf.cuda_transfer<host_device>}
|
||||
! CHECK: cuf.data_transfer %{{.*}} to %{{.*}} {hasManagedOrUnifedSymbols, transfer_kind = #cuf.cuda_transfer<device_host>}
|
||||
|
||||
! A right-hand side expression involving managed data is evaluated on the host
|
||||
! to avoid materializing a temporary, so no data transfer is generated.
|
||||
subroutine managed_array_assign_2d()
|
||||
integer(4), managed :: ma(4,16), mb(4,16)
|
||||
integer(4) :: ha(4,16)
|
||||
ma = mb
|
||||
ha = ma
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_array_assign_2d()
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_host>
|
||||
|
||||
! A whole array component counts as a whole array when the component itself has
|
||||
! the managed attribute.
|
||||
subroutine managed_component_attr_assign()
|
||||
type :: t
|
||||
integer(4), managed :: m(16)
|
||||
end type
|
||||
type(t) :: x, y
|
||||
x%m = y%m
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_component_attr_assign()
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
|
||||
! One whole managed side is enough: the other may be a section.
|
||||
subroutine managed_mixed_whole_section()
|
||||
integer(4), managed :: ma(4), mb(4,16)
|
||||
integer(4) :: ha(4)
|
||||
ma = mb(:,1) ! whole managed = managed section
|
||||
mb(:,1) = ma ! managed section = whole managed
|
||||
ha = mb(:,1) ! whole host = managed section: performed on the host
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_mixed_whole_section()
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A whole managed array is copied from a constant, a host scalar, a host array or
|
||||
! a host section.
|
||||
subroutine managed_whole_from_host(hs)
|
||||
integer(4) :: hs
|
||||
integer(4), managed :: ma(4)
|
||||
integer(4) :: hb(4,16)
|
||||
ma = 0
|
||||
ma = hs
|
||||
ma = hb(:,1)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_whole_from_host
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<host_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<host_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<host_device>
|
||||
|
||||
! The host cannot access device memory directly, so an assignment between managed
|
||||
! and device data is a copy in both directions, sections included.
|
||||
subroutine managed_device_transfer(n)
|
||||
integer :: n, i
|
||||
integer(4), managed :: ma(4,16)
|
||||
integer(4), device :: da(4,16)
|
||||
integer(4) :: ha(4,16)
|
||||
ma = da ! whole managed = whole device
|
||||
da = ma ! whole device = whole managed
|
||||
do i = 1, n
|
||||
ma(:,i) = da(:,i) ! managed section = device section
|
||||
da(:,i) = ma(:,i) ! device section = managed section
|
||||
ha(:,i) = da(:,i) ! host section = device section
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_device_transfer
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_host>
|
||||
|
||||
! Only the managed side has to be whole: a host section can be the target of a
|
||||
! whole managed array copy.
|
||||
subroutine host_section_from_managed(n)
|
||||
integer :: n, i
|
||||
integer(4), managed :: mb(4)
|
||||
integer(4) :: ha(4,16)
|
||||
do i = 1, n
|
||||
ha(:,i) = mb
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPhost_section_from_managed
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_host>
|
||||
|
||||
! A managed function result is copied whatever the shape of the left-hand side.
|
||||
subroutine managed_func_result()
|
||||
use mfr
|
||||
integer(4), managed :: b(4)
|
||||
integer(4) :: hb(4)
|
||||
b = fr(4)
|
||||
hb = fr(4)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_func_result()
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_host>
|
||||
|
||||
subroutine managed_func_result_section()
|
||||
use mfr
|
||||
integer(4), managed :: b(4)
|
||||
b(1:2) = fr(2)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_func_result_section()
|
||||
! CHECK: cuf.data_transfer {{.*}}#cuf.cuda_transfer<device_device>
|
||||
|
||||
! Assignments done on the host.
|
||||
|
||||
! An expression involving managed data is evaluated on the host, which avoids a
|
||||
! temporary.
|
||||
subroutine managed_expr_assign()
|
||||
integer(4), managed :: ma(16), mb(16)
|
||||
integer(4), managed :: ma(16)
|
||||
integer(4) :: ha(16)
|
||||
ma = mb + 1 ! managed = managed expression
|
||||
ha = ma + 1 ! host = managed expression
|
||||
ha = ma + 1
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_expr_assign()
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! Element-wise accesses of managed data are performed on the host.
|
||||
subroutine managed_scalar_access(n)
|
||||
integer :: n, i
|
||||
integer(4), managed :: ma(16)
|
||||
integer(4) :: ha(16)
|
||||
do i = 1, n
|
||||
ha(i) = ma(i) + 1 ! element-wise access is performed on the host
|
||||
ha(i) = ma(i) + 1
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_scalar_access
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A managed function result may be produced by an asynchronous kernel, so
|
||||
! consuming it in an assignment is a synchronizing data transfer.
|
||||
subroutine managed_func_result()
|
||||
use mfr
|
||||
integer(4), managed :: b(4)
|
||||
b = fr(4)
|
||||
subroutine managed_element_assign(n)
|
||||
integer :: n, i
|
||||
integer(4), managed :: ma(16), mb(16)
|
||||
integer(4) :: ha(16)
|
||||
do i = 1, n
|
||||
ma(i) = mb(i) ! managed element = managed element
|
||||
ha(i) = mb(i) ! host element = managed element
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_func_result()
|
||||
! CHECK: cuf.data_transfer
|
||||
! CHECK-LABEL: func.func @_QPmanaged_element_assign
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A managed array section is assigned on the host, whatever the host right-hand
|
||||
! side is.
|
||||
subroutine managed_section_from_host(n, hs)
|
||||
integer :: n, i
|
||||
integer(4) :: hs
|
||||
integer(4), managed :: mb(4,16)
|
||||
integer(4) :: ha(4), hb(4,16)
|
||||
do i = 1, n
|
||||
mb(:,i) = 0
|
||||
mb(:,i) = hs
|
||||
mb(:,i) = ha
|
||||
mb(:,i) = hb(:,i)
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_section_from_host
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A managed array section on either side is assigned on the host: a loop assigning
|
||||
! one section at a time would otherwise become a sequence of blocking copies.
|
||||
subroutine managed_section_assign(n)
|
||||
integer :: n, i
|
||||
integer(4), managed :: ma(4,16), mb(4,16)
|
||||
integer(4) :: ha(4,16)
|
||||
do i = 1, n
|
||||
ma(:,i) = mb(:,i) ! managed section = managed section
|
||||
ha(:,i) = mb(:,i) ! host section = managed section
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_section_assign
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
subroutine managed_section_assign_1d()
|
||||
integer(4), managed :: ma(16), mb(16)
|
||||
integer(4) :: ha(16)
|
||||
ma(1:8) = mb(1:8) ! contiguous section
|
||||
ma(1:16:2) = mb(1:16:2) ! strided section
|
||||
ha(1:8) = mb(1:8)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_section_assign_1d
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A section that covers the whole array is still a section: what matters is the
|
||||
! form of the reference, not the extent it happens to cover.
|
||||
subroutine managed_full_slice_assign()
|
||||
integer(4), managed :: ma(4,16), mb(4,16)
|
||||
integer(4) :: ha(4,16)
|
||||
ma(:,:) = mb(:,:)
|
||||
ha(:,:) = mb(:,:)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_full_slice_assign()
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
! A component of a managed object is not a whole managed array: the attribute is
|
||||
! on the object, not on the component.
|
||||
subroutine managed_component_assign()
|
||||
type :: t
|
||||
integer(4) :: m(16)
|
||||
end type
|
||||
type(t), managed :: ma, mb
|
||||
type(t) :: ha
|
||||
ma%m = mb%m
|
||||
ha%m = mb%m
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_component_assign()
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
subroutine managed_component_section_assign()
|
||||
type :: t
|
||||
integer(4) :: m(16)
|
||||
end type
|
||||
type(t), managed :: ma, mb
|
||||
ma%m(1:8) = mb%m(1:8)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_component_section_assign()
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
||||
subroutine managed_component_attr_section_assign()
|
||||
type :: t
|
||||
integer(4), managed :: m(16)
|
||||
end type
|
||||
type(t) :: x, y
|
||||
x%m(1:8) = y%m(1:8)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPmanaged_component_attr_section_assign()
|
||||
! CHECK-NOT: cuf.data_transfer
|
||||
|
|
|
|||
Loading…
Reference in a new issue