Skip to content

Commit 895d01a

Browse files
LudwigBoessLudwigBoess
authored
Merge branch '1.5.0rc' into dev/hybrid
2 parents 91e3e72 + 99f80f7 commit 895d01a

46 files changed

Lines changed: 5591 additions & 900 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CMakeLists.txt‎

Lines changed: 156 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ set(PROJECT_NAME entity)
77

88
project(
99
${PROJECT_NAME}
10-
VERSION 1.4.3
10+
VERSION 1.5.0
1111
LANGUAGES CXX C)
1212
add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
1313
set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ")
@@ -58,6 +58,24 @@ set(gpu_aware_mpi
5858
${default_gpu_aware_mpi}
5959
CACHE BOOL "Enable GPU-aware MPI")
6060

61+
set(team_policy
62+
${default_team_policy}
63+
CACHE BOOL "Enable team_policy tile-blocked deposit/pusher kernels")
64+
set(team_policy_tile_size
65+
${default_team_policy_tile_size}
66+
CACHE STRING "team_policy tile edge length in cells")
67+
set(team_policy_tile_sizes
68+
"4;6;8;10;12;14;16"
69+
CACHE STRING "team_policy tile-size choices")
70+
set(team_policy_drift
71+
${default_team_policy_drift}
72+
CACHE STRING
73+
"team_policy tiled-deposit scratch halo drift in cells (max cells a particle may move between two sorts). Sizes the deposit scratch halo only; the sort cadence is set at runtime via spatial_sorting_interval. Default 1.")
74+
set(vendor_sort
75+
${default_vendor_sort}
76+
CACHE BOOL
77+
"Use the vendor sort_by_key (oneDPL/Thrust/rocThrust) for the team_policy spatial sort when available. OFF forces the Kokkos::BinSort fallback, which sorts each SoA member in place (lower peak memory, no maxnpart gather buffer) at the cost of sort speed.")
78+
6179
# -------------------------- Compilation settings -------------------------- #
6280
set(CMAKE_CXX_STANDARD 20)
6381
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -136,6 +154,87 @@ else()
136154
set(DEVICE_ENABLED OFF)
137155
endif()
138156

157+
# ------------------------------ team_policy wiring ------------------------ #
158+
if(${team_policy})
159+
list(FIND team_policy_tile_sizes "${team_policy_tile_size}" _tps_idx)
160+
if(_tps_idx EQUAL -1)
161+
message(FATAL_ERROR
162+
"${Red}team_policy_tile_size must be one of ${team_policy_tile_sizes}, "
163+
"got '${team_policy_tile_size}'${ColorReset}")
164+
endif()
165+
add_compile_options("-D TEAM_POLICY")
166+
add_compile_options("-D TEAM_POLICY_TILE_SIZE=${team_policy_tile_size}")
167+
168+
# Compile-time tiled-deposit scratch halo drift. Sizes the halo so a
169+
# particle that drifts up to DRIFT cells between two sorts still deposits
170+
# inside its tile scratch; particles drifting further take the
171+
# per-particle global-J escape valve (correct, only slower). This is
172+
# independent of the sort cadence, which is set at runtime via
173+
# `spatial_sorting_interval`. Defaults to 1 (the sorted-every-step case).
174+
add_compile_options("-D TEAM_POLICY_DRIFT=${team_policy_drift}")
175+
176+
# Vendor sort: oneDPL on SYCL, Thrust on CUDA, rocThrust/rocprim on HIP.
177+
# When `vendor_sort` is ON (default) the available library is detected
178+
# and used; the spatial sort then builds a single permutation that
179+
# gathers all SoA members. When `vendor_sort` is OFF, or no library is
180+
# found, the code falls back to Kokkos::BinSort, which sorts each member
181+
# in place -- lower peak memory and no maxnpart gather buffer, at the
182+
# cost of sort speed (negligible when sorting is a small fraction of the
183+
# step). The `vendor_sort` knob lets you force the BinSort fallback even
184+
# when a vendor library is present.
185+
if(${vendor_sort})
186+
if("${Kokkos_DEVICES}" MATCHES "SYCL")
187+
find_package(oneDPL QUIET)
188+
if(oneDPL_FOUND)
189+
message(STATUS "team_policy: oneDPL found, enabling SYCL sort_by_key")
190+
add_compile_options("-D ONEDPL_ENABLED")
191+
set(DEPENDENCIES ${DEPENDENCIES} oneDPL)
192+
else()
193+
message(STATUS "team_policy: oneDPL not found; using BinSort fallback "
194+
"for SYCL sort_by_key")
195+
endif()
196+
endif()
197+
198+
if("${Kokkos_DEVICES}" MATCHES "CUDA")
199+
find_package(Thrust QUIET)
200+
if(Thrust_FOUND)
201+
message(STATUS "team_policy: Thrust enabled for CUDA sort_by_key")
202+
add_compile_options("-D THRUST_ENABLED")
203+
else()
204+
message(STATUS "team_policy: Thrust not found; using BinSort fallback "
205+
"for CUDA sort_by_key")
206+
endif()
207+
endif()
208+
209+
if("${Kokkos_DEVICES}" MATCHES "HIP")
210+
# rocThrust ships with ROCm. The HIP sort_by_key path uses rocprim's
211+
# bounded-bit radix sort directly (rocprim is rocThrust's own
212+
# dependency, so its headers come in transitively; we find it
213+
# explicitly to keep the include path robust). This builds a single
214+
# permutation that gathers all SoA members, instead of the legacy
215+
# per-member Kokkos::BinSort path which allocates a fresh
216+
# `sorted_values` buffer for every member every step (the dominant
217+
# source of allocator churn / fragmentation on ROCm).
218+
find_package(rocthrust QUIET)
219+
if(rocthrust_FOUND)
220+
message(STATUS "team_policy: rocThrust enabled for HIP sort_by_key")
221+
add_compile_options("-D ROCTHRUST_ENABLED")
222+
set(DEPENDENCIES ${DEPENDENCIES} roc::rocthrust)
223+
find_package(rocprim QUIET)
224+
if(rocprim_FOUND)
225+
set(DEPENDENCIES ${DEPENDENCIES} roc::rocprim)
226+
endif()
227+
else()
228+
message(STATUS "team_policy: rocThrust not found; using BinSort "
229+
"fallback for HIP sort_by_key")
230+
endif()
231+
endif()
232+
else()
233+
message(STATUS "team_policy: vendor_sort=OFF; forcing Kokkos::BinSort "
234+
"fallback for spatial sort_by_key")
235+
endif()
236+
endif()
237+
139238
# MPI
140239
if(${mpi})
141240
find_or_fetch_dependency(MPI FALSE REQUIRED)
@@ -145,6 +244,62 @@ if(${mpi})
145244
if(${DEVICE_ENABLED})
146245
if(${gpu_aware_mpi})
147246
add_compile_options("-D GPU_AWARE_MPI")
247+
248+
# On Cray systems (e.g. Frontier) GPU-aware Cray MPICH can only
249+
# handle device pointers if the GPU Transport Layer (GTL) library
250+
# is linked. The Cray compiler wrappers (cc/CC) inject this
251+
# automatically, but we build with hipcc/nvcc directly, so
252+
# find_package(MPI) only finds base libmpi and the GTL is left
253+
# out -> MPI_Sendrecv on a device pointer fails with
254+
# "OFI ... Bad address". Add it explicitly here.
255+
#
256+
# Cray PE exports PE_MPICH_GTL_DIR_<accel> / PE_MPICH_GTL_LIBS_<accel>
257+
# (e.g. amd_gfx90a -> -lmpi_gtl_hsa). Their absence means this is
258+
# not a Cray MPICH build, in which case nothing extra is needed.
259+
if("${Kokkos_DEVICES}" MATCHES "HIP")
260+
set(_gtl_accels amd_gfx942 amd_gfx940 amd_gfx90a amd_gfx908 amd_gfx906)
261+
elseif("${Kokkos_DEVICES}" MATCHES "CUDA")
262+
set(_gtl_accels nvidia90 nvidia80 nvidia70)
263+
elseif("${Kokkos_DEVICES}" MATCHES "SYCL")
264+
set(_gtl_accels ponteVecchio)
265+
else()
266+
set(_gtl_accels "")
267+
endif()
268+
269+
set(_gtl_dir "")
270+
set(_gtl_libflag "")
271+
foreach(_accel ${_gtl_accels})
272+
if((NOT _gtl_dir) AND (DEFINED ENV{PE_MPICH_GTL_DIR_${_accel}}))
273+
# strip the leading "-L" from the Cray-provided value
274+
string(REGEX REPLACE "^-L" ""
275+
_gtl_dir "$ENV{PE_MPICH_GTL_DIR_${_accel}}")
276+
string(REGEX REPLACE "^-l" ""
277+
_gtl_libflag "$ENV{PE_MPICH_GTL_LIBS_${_accel}}")
278+
endif()
279+
endforeach()
280+
281+
if(_gtl_dir AND _gtl_libflag)
282+
find_library(MPI_GTL_LIBRARY
283+
NAMES ${_gtl_libflag}
284+
HINTS "${_gtl_dir}"
285+
NO_DEFAULT_PATH)
286+
if(MPI_GTL_LIBRARY)
287+
message(STATUS
288+
"GPU-aware MPI: linking Cray GTL library ${MPI_GTL_LIBRARY}")
289+
set(DEPENDENCIES ${DEPENDENCIES} ${MPI_GTL_LIBRARY})
290+
else()
291+
message(FATAL_ERROR
292+
"${Red}gpu_aware_mpi=ON: Cray MPICH detected but the GTL "
293+
"library 'lib${_gtl_libflag}' was not found in '${_gtl_dir}'. "
294+
"GPU-aware MPI will crash at runtime without it. Make sure the "
295+
"craype-accel module is loaded, or build with gpu_aware_mpi=OFF."
296+
"${ColorReset}")
297+
endif()
298+
else()
299+
message(STATUS
300+
"GPU-aware MPI: no Cray GTL environment found; assuming the MPI "
301+
"implementation is GPU-aware without an extra transport library.")
302+
endif()
148303
endif()
149304
else()
150305
set(gpu_aware_mpi

‎cmake/defaults.cmake‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,36 @@ else()
9292
endif()
9393

9494
set_property(CACHE default_gpu_aware_mpi PROPERTY TYPE BOOL)
95+
96+
if(DEFINED ENV{Entity_ENABLE_TEAM_POLICY})
97+
set(default_team_policy
98+
$ENV{Entity_ENABLE_TEAM_POLICY}
99+
CACHE INTERNAL "Default flag for team_policy tile-blocked kernels")
100+
else()
101+
set(default_team_policy
102+
OFF
103+
CACHE INTERNAL "Default flag for team_policy tile-blocked kernels")
104+
endif()
105+
set_property(CACHE default_team_policy PROPERTY TYPE BOOL)
106+
107+
if(DEFINED ENV{Entity_ENABLE_VENDOR_SORT})
108+
set(default_vendor_sort
109+
$ENV{Entity_ENABLE_VENDOR_SORT}
110+
CACHE INTERNAL
111+
"Default flag for vendor sort_by_key (oneDPL/Thrust/rocThrust)")
112+
else()
113+
set(default_vendor_sort
114+
ON
115+
CACHE INTERNAL
116+
"Default flag for vendor sort_by_key (oneDPL/Thrust/rocThrust)")
117+
endif()
118+
set_property(CACHE default_vendor_sort PROPERTY TYPE BOOL)
119+
120+
set(default_team_policy_tile_size
121+
8
122+
CACHE INTERNAL "Default tile edge length in cells for team_policy")
123+
124+
set(default_team_policy_drift
125+
1
126+
CACHE INTERNAL
127+
"Default tiled-deposit scratch halo drift for team_policy (cells between sorts)")

‎cmake/report.cmake‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,44 @@ if(${mpi} AND ${DEVICE_ENABLED})
122122
GPU_AWARE_MPI_REPORT
123123
46)
124124
endif()
125+
printchoices(
126+
"Team Policy"
127+
"team_policy"
128+
"${ON_OFF_VALUES}"
129+
${team_policy}
130+
OFF
131+
"${Green}"
132+
TEAM_POLICY_REPORT
133+
46)
134+
if(${team_policy})
135+
printchoices(
136+
"Team Tile Size"
137+
"team_policy_tile_size"
138+
"${team_policy_tile_sizes}"
139+
${team_policy_tile_size}
140+
${default_team_policy_tile_size}
141+
"${Blue}"
142+
TEAM_POLICY_TILE_SIZE_REPORT
143+
46)
144+
printchoices(
145+
"Team Deposit Drift"
146+
"team_policy_drift"
147+
"${team_policy_drift}"
148+
${team_policy_drift}
149+
1
150+
"${Blue}"
151+
TEAM_POLICY_DRIFT_REPORT
152+
46)
153+
printchoices(
154+
"Vendor sort"
155+
"vendor_sort"
156+
"${ON_OFF_VALUES}"
157+
${vendor_sort}
158+
ON
159+
"${Green}"
160+
VENDOR_SORT_REPORT
161+
46)
162+
endif()
125163
printchoices(
126164
"Debug mode"
127165
"DEBUG"
@@ -197,6 +235,13 @@ if(${mpi} AND ${DEVICE_ENABLED})
197235
string(APPEND REPORT_TEXT " " ${GPU_AWARE_MPI_REPORT} "\n")
198236
endif()
199237

238+
string(APPEND REPORT_TEXT " " ${TEAM_POLICY_REPORT} "\n")
239+
if(${team_policy})
240+
string(APPEND REPORT_TEXT " " ${TEAM_POLICY_TILE_SIZE_REPORT} "\n")
241+
string(APPEND REPORT_TEXT " " ${TEAM_POLICY_DRIFT_REPORT} "\n")
242+
string(APPEND REPORT_TEXT " " ${VENDOR_SORT_REPORT} "\n")
243+
endif()
244+
200245
string(
201246
APPEND
202247
REPORT_TEXT

0 commit comments

Comments
 (0)