From 4efc17b8b647a06389632ead5aa68dfbfe0a7e9a Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 01/41] GPU: stop considering OpenCL on macOS macOS ships OpenCL 1.2, below the 2.x the OpenCL backend requires, so find_package(OpenCL) there could never produce a usable backend: the version check dropped it again a few lines later. Skip the lookup on Apple instead. With that, CUDA_ENABLED, OPENCL_ENABLED and HIP_ENABLED are all necessarily off on macOS, which makes the Darwin arm of the backend dispatch dead code. Drop it along with its warning and unindent the rest. --- GPU/GPUTracking/CMakeLists.txt | 64 ++++++++++++++++------------------ dependencies/FindO2GPU.cmake | 7 ++-- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt index ca58d91212084..ce3cbca5bb197 100644 --- a/GPU/GPUTracking/CMakeLists.txt +++ b/GPU/GPUTracking/CMakeLists.txt @@ -464,40 +464,36 @@ endif() # Add CMake recipes for GPU Tracking librararies if(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED) - if(CMAKE_SYSTEM_NAME MATCHES Darwin) - message(WARNING "GPU Tracking disabled on MacOS") - else() - make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch) - set(GPU_CONST_PARAM_FILES) - foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES}) - set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par) - add_custom_command( - OUTPUT ${PARAMFILE} - COMMAND bash -c - "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'" - | root -l -b > /dev/null - VERBATIM - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch - MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C - DEPENDS ${GPU_DEFAULT_PARAMS_HEADER} - ${GPU_DEFAULT_PARAMS_HEADER_DEVICE} - ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h - ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc - COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}") - LIST(APPEND GPU_CONST_PARAM_FILES ${PARAMFILE}) - endforeach() - add_custom_target(${MODULE}_GPU_CONST_PARAM_ARCHS ALL DEPENDS ${GPU_CONST_PARAM_FILES}) - install(FILES ${GPU_CONST_PARAM_FILES} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/arch_param) - - if(CUDA_ENABLED) - add_subdirectory(Base/cuda) - endif() - if(OPENCL_ENABLED) - add_subdirectory(Base/opencl) - endif() - if(HIP_ENABLED) - add_subdirectory(Base/hip) - endif() + make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch) + set(GPU_CONST_PARAM_FILES) + foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES}) + set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par) + add_custom_command( + OUTPUT ${PARAMFILE} + COMMAND bash -c + "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'" + | root -l -b > /dev/null + VERBATIM + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch + MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C + DEPENDS ${GPU_DEFAULT_PARAMS_HEADER} + ${GPU_DEFAULT_PARAMS_HEADER_DEVICE} + ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h + ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc + COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}") + LIST(APPEND GPU_CONST_PARAM_FILES ${PARAMFILE}) + endforeach() + add_custom_target(${MODULE}_GPU_CONST_PARAM_ARCHS ALL DEPENDS ${GPU_CONST_PARAM_FILES}) + install(FILES ${GPU_CONST_PARAM_FILES} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/arch_param) + + if(CUDA_ENABLED) + add_subdirectory(Base/cuda) + endif() + if(OPENCL_ENABLED) + add_subdirectory(Base/opencl) + endif() + if(HIP_ENABLED) + add_subdirectory(Base/hip) endif() endif() diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index d2f426c448e12..5804b046c915e 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -20,8 +20,11 @@ set(HIP_AMDGPUTARGET_DEFAULT_MINIMAL gfx906) if(NOT DEFINED ENABLE_CUDA) set(ENABLE_CUDA "AUTO") endif() -if(NOT DEFINED ENABLE_OPENCL) - set(ENABLE_OPENCL "AUTO") +if(NOT APPLE) + # macOS ships OpenCL 1.2 only, below the 2.x that the OpenCL backend needs. + if(NOT DEFINED ENABLE_OPENCL) + set(ENABLE_OPENCL "AUTO") + endif() endif() if(NOT DEFINED ENABLE_HIP) set(ENABLE_HIP "AUTO") From 99542dd94d0f86e082611b42815083c3cf791fb0 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:38 +0200 Subject: [PATCH 02/41] GPU: do not place anything in the Metal constant address space The MSL generic address space covers device, threadgroup and thread but not constant, so a generic member function cannot be called on an object that lives in constant memory: 'cannot initialize object parameter of type X with an expression of type constant X'. The shared code is generic throughout, so constant memory is simply not usable on this backend. Metal therefore implies GPUCA_NO_CONSTANT_MEMORY, which already exists for the other backends and redirects GPUconstant() to GPUglobal(). GPUconstantref() has to follow it, exactly as the OpenCL block already arranges; the Metal block hardcoded 'constant' and so kept handing out constant references whatever the setting. It now falls through to the unannotated, and therefore generic, fallback. Macro expansions are unchanged for host, CUDA, HIP, OpenCL and cling. Takes the Metal translation unit from 333 errors to 235, of which the constant-versus-generic diagnostics drop from 91 to 14. --- GPU/Common/GPUCommonDef.h | 5 ++++- GPU/Common/GPUCommonDefAPI.h | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/GPU/Common/GPUCommonDef.h b/GPU/Common/GPUCommonDef.h index 6e2269650f576..90746019d9a99 100644 --- a/GPU/Common/GPUCommonDef.h +++ b/GPU/Common/GPUCommonDef.h @@ -58,7 +58,10 @@ #define GPUCA_ALIGPUCODE // Part of GPUTracking library but not of interface #endif -#if (defined(__CUDACC__) && defined(GPUCA_CUDA_NO_CONSTANT_MEMORY)) || (defined(__HIPCC__) && defined(GPUCA_HIP_NO_CONSTANT_MEMORY)) || (defined(__OPENCL__) && defined(GPUCA_OPENCL_NO_CONSTANT_MEMORY)) +// __METAL__ unconditionally: the MSL generic address space does not span +// `constant`, so a generic member function cannot be called on an object living +// there, and the shared code is generic throughout. +#if (defined(__CUDACC__) && defined(GPUCA_CUDA_NO_CONSTANT_MEMORY)) || (defined(__HIPCC__) && defined(GPUCA_HIP_NO_CONSTANT_MEMORY)) || (defined(__OPENCL__) && defined(GPUCA_OPENCL_NO_CONSTANT_MEMORY)) || defined(__METAL__) #define GPUCA_NO_CONSTANT_MEMORY #elif (defined(__CUDACC__) || defined(__HIPCC__)) && !defined(GPUCA_GPUCODE_HOSTONLY) #define GPUCA_HAS_GLOBAL_SYMBOL_CONSTANT_MEM diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index a04934304d525..7346f527e00b5 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -167,7 +167,9 @@ #define GPUglobalref() device #define GPUsharedref() threadgroup #define GPUprivateref() thread - #define GPUconstantref() constant + #if !defined(GPUCA_NO_CONSTANT_MEMORY) + #define GPUconstantref() constant + #endif #define GPUconstexprref() GPUconstexpr() #define GPUdouble() float #define GPUbarrier() threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup) From 5da0373edceb38029cb4c06e768bbb016247c14f Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:40 +0200 Subject: [PATCH 03/41] GPU: Apple Metal backend, off by default The backend itself: the Objective-C++ host side, the .metal kernel source and its build rules, plus the CMake to enable them. Off unless asked for. FindO2GPU.cmake leaves ENABLE_METAL=OFF on macOS and the subdirectory is gated on METAL_ENABLED, so macOS keeps running on the CPU until the whole chain is validated. Apple toolchain only: the source goes .metal -> AIR through xcrun metal and nothing else, with no SPIR-V translation step in between. Requires -std=metal4.1, the first MSL version with a generic address space. Earlier versions reject an unannotated pointer with 'pointer type must have explicit address space qualifier' and an unannotated 'this' with 'cannot initialize object parameter', both of which GPUCommonDefAPI.h relies on for GPUgeneric() and GPUdDefault(). Verified against Xcode 27, which ships metal4.1; Xcode 26 and earlier stop at metal4.0. Two things that look structural are not. MSL rejects derived classes, but '#pragma METAL internals : enable' -- the switch metal_stdlib itself uses in 46 paired places -- lifts that, and the kernels are class-based throughout. A derived type still cannot be a kernel argument, so the constant memory arrives as an untyped buffer and is cast inside, mirroring what the OpenCL TU does with __cl_clang_non_portable_kernel_param_types and what gpu_mem already did here. That also settles the constant address space, which generic does not span. With both in place the kernel list expands to all 104 entry points and no derived-class or kernel-argument-type errors remain. The bodies still do not compile: 1038 errors, of which 360 are MSL having no double (largely host-only headers such as PhysicsConstants.h and MathUtils/Utils.h reaching device code) and 352 are namespace-scope constexpr needing GPUglobalconstexpr(). That is bulk work rather than a missing language feature, so the .metal file still stops after the common headers until it is done. --- GPU/GPUTracking/Base/metal/CMakeLists.txt | 107 +++++ .../Base/metal/GPUReconstructionMETAL.metal | 79 ++++ .../Base/metal/GPUReconstructionMetal.h | 67 +++ .../Base/metal/GPUReconstructionMetal.mm | 414 ++++++++++++++++++ .../GPUReconstructionMetalIncludesHost.h | 61 +++ .../metal/GPUReconstructionMetalKernels.mm | 82 ++++ ...PUReconstructionMetalKernelsSpecialize.inc | 27 ++ GPU/GPUTracking/CMakeLists.txt | 4 +- dependencies/FindO2GPU.cmake | 36 +- 9 files changed, 871 insertions(+), 6 deletions(-) create mode 100644 GPU/GPUTracking/Base/metal/CMakeLists.txt create mode 100644 GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal create mode 100644 GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h create mode 100644 GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm create mode 100644 GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h create mode 100644 GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm create mode 100644 GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc diff --git a/GPU/GPUTracking/Base/metal/CMakeLists.txt b/GPU/GPUTracking/Base/metal/CMakeLists.txt new file mode 100644 index 0000000000000..577501f9e6c3c --- /dev/null +++ b/GPU/GPUTracking/Base/metal/CMakeLists.txt @@ -0,0 +1,107 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +set(MODULE GPUTrackingMETAL) +enable_language(ASM) + +message(STATUS "Building GPUTracking with Metal support") + +# convenience variables +if(ALIGPU_BUILD_TYPE STREQUAL "Standalone") + set(GPUDIR ${CMAKE_SOURCE_DIR}/../) +else() + set(GPUDIR ${CMAKE_SOURCE_DIR}/GPU/GPUTracking) +endif() +set(METAL_SRC ${GPUDIR}/Base/metal/GPUReconstructionMETAL.metal) +set(METAL_BIN ${CMAKE_CURRENT_BINARY_DIR}/GPUReconstructionMetalCode) + +# MSL 4.1 is the first version with a generic address space: earlier versions +# reject an unannotated pointer or `this` outright, which GPUCommonDefAPI.h +# relies on for GPUgeneric() and GPUdDefault(). +set(METAL_FLAGS -std=metal4.1 ${GPUCA_METAL_DENORMALS_FLAGS}) +set(METAL_DEFINES "-D$,$-D>" + "-I$,EXCLUDE,^/usr/include/?>,$-I>" + -I${CMAKE_SOURCE_DIR}/Detectors/TRD/base/src + -I${CMAKE_SOURCE_DIR}/Detectors/Base/src + -I${CMAKE_SOURCE_DIR}/DataFormats/Reconstruction/src +) + +set(SRCS GPUReconstructionMetal.mm GPUReconstructionMetalKernels.mm) +set(HDRS GPUReconstructionMetal.h GPUReconstructionMetalIncludesHost.h) + +if(ALIGPU_BUILD_TYPE STREQUAL "O2") + o2_add_library(${MODULE} + SOURCES ${SRCS} + PUBLIC_LINK_LIBRARIES O2::GPUTracking + TARGETVARNAME targetName) + + target_link_libraries(${targetName} PUBLIC ${METAL_FRAMEWORKS}) + + target_compile_definitions(${targetName} PRIVATE $) + # the compile_defitions are not propagated automatically on purpose (they are + # declared PRIVATE) so we are not leaking them outside of the GPU** + # directories +endif() + +if(ALIGPU_BUILD_TYPE STREQUAL "Standalone") + add_library(${MODULE} SHARED ${SRCS}) + target_link_libraries(${MODULE} GPUTracking) + install(TARGETS ${MODULE}) + set(targetName ${MODULE}) +endif() + +if(METAL_ENABLED) # BUILD Metal source code for runtime compilation target + + # executes clang to preprocess + add_custom_command( + OUTPUT ${METAL_BIN}.metal + COMMAND xcrun -sdk macosx metal + -Wno-unused-command-line-argument + ${METAL_FLAGS} + ${METAL_DEFINES} + -MD -MT ${METAL_BIN}.src -MF ${METAL_BIN}.src.d + -E -P ${METAL_SRC} > ${METAL_BIN}.metal + DEPENDS ${METAL_SRC} + DEPFILE ${METAL_BIN}.src.d + COMMAND_EXPAND_LISTS + COMMENT "Preparing Metal source file for run time compilation ${METAL_BIN}.metal") + + # Create the ir + add_custom_command( + OUTPUT ${METAL_BIN}.ir + COMMAND xcrun -sdk macosx metal + -Wno-unused-command-line-argument + -Wno-c++17-extensions + -ferror-limit=10000 + ${METAL_FLAGS} + ${METAL_DEFINES} + ${METAL_BIN}.metal + -o ${METAL_BIN}.ir + DEPENDS ${METAL_BIN}.metal + COMMAND_EXPAND_LISTS + COMMENT "Preparing Metal intermediate representation for run time compilation ${METAL_BIN}.ir") + + add_custom_target(metal_preprocessed_code ALL DEPENDS ${METAL_BIN}.metal COMMENT "Needed to inject dependency on its creation") + add_custom_target(metal_intermediate_representation ALL DEPENDS ${METAL_BIN}.ir COMMENT "Needed to inject dependency on its creation") + + # Pack the compiled library into __DATA,__gpu_resource during final link. This + # way we do not need to create an intermediate object. Compiling the source at + # run time is not an option: the driver's compiler service dies on it. + target_link_options(${targetName} + PRIVATE + "-Wl,-sectcreate,__DATA,__gpu_resource,${METAL_BIN}.ir") + add_dependencies(${targetName} metal_preprocessed_code) + add_dependencies(${targetName} metal_intermediate_representation) +endif() + +install(FILES ${HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU) + +target_compile_definitions(${targetName} PRIVATE GPUCA_METAL_BUILD_FLAGS=$ ) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal new file mode 100644 index 0000000000000..e8cce64146991 --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal @@ -0,0 +1,79 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUReconstructionMETAL.metal + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" +// clang-format off + +// --- Backend selection ------------------------------------------------------- +#define GPUCA_GPUTYPE_METAL 1 + +// --- Metal stdlib ------------------------------------------------------------ +#include +// MSL rejects derived classes outside of this pragma, and the kernels are +// class-based throughout. metal_stdlib itself uses it in 46 paired places. +#pragma METAL internals : enable +using namespace metal; + +// --- OpenCL compatibility shims --------------------------------------------- + +// Address space aliases (match OpenCL vernacular used by the project); constant +// is spelled the same in MSL +#define global device +#define local threadgroup + +#ifndef M_PI +#define M_PI 3.1415926535f +#endif + +// Disable assertions inside GPU code (same as OpenCL variant) +#ifdef assert +# undef assert +#endif +#define assert(param) + +// --- Project headers --------------------------------------------------------- +#include "GPUCommonDef.h" +#include "GPUCommonTypeTraits.h" +#include "GPUCommonArray.h" + +// The remaining headers do not compile as MSL yet, but nothing structural is in +// the way: with the pragma above and the untyped constant buffer below, the +// kernel list expands to all 104 entry points, with no derived-class and no +// kernel-argument-type errors left. What fails is the bodies, and it is bulk +// work rather than a missing language feature -- MSL has no double, and every +// namespace-scope constexpr needs GPUglobalconstexpr(). +#if 0 +#include "GPUConstantMem.h" +#include "GPUReconstructionIncludesDeviceAll.h" +#endif + +// --- Kernel list expansion --------------------------------------------------- +#define GPUCA_KRNL(...) GPUCA_KRNLGPU(__VA_ARGS__) + +// --- Constant memory + global heap plumbing --------------------------------- +// The heap and the constant memory arrive as buffer(0) and buffer(1). The latter +// is untyped because a buffer of GPUConstantMem, which has base classes, is not +// a valid kernel argument type. +#define GPUCA_CONSMEM_PTR \ + device char* gpu_mem [[buffer(0)]], \ + device char* pConstantRaw [[buffer(1)]], +#define GPUCA_CONSMEM (*(device GPUConstantMem*)pConstantRaw) + +// Include the actual kernels, once the headers above compile as MSL. +#if 0 +#include "GPUReconstructionKernelList.h" +#endif + +// clang-format on +#pragma clang diagnostic pop diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h new file mode 100644 index 0000000000000..66cf5ab7bf121 --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h @@ -0,0 +1,67 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef GPURECONSTRUCTIONMETAL_H +#define GPURECONSTRUCTIONMETAL_H + +#include "GPUReconstructionDeviceBase.h" + +extern "C" o2::gpu::GPUReconstruction* GPUReconstruction_Create_METAL(const o2::gpu::GPUSettingsDeviceBackend& cfg); + +namespace o2::gpu +{ +struct GPUReconstructionMetalInternals; + +class GPUReconstructionMetal : public GPUReconstructionProcessing::KernelInterface +{ + public: + GPUReconstructionMetal(const GPUSettingsDeviceBackend& cfg); + ~GPUReconstructionMetal() override; + + template + void runKernelBackend(const krnlSetupTime& _xyz, const Args&... args); + + protected: + int32_t InitDevice_Runtime() override; + int32_t ExitDevice_Runtime() override; + + virtual int32_t GPUChkErrInternal(const int64_t error, const char* file, int32_t line) const override; + + void SynchronizeGPU() override; + int32_t GPUDebug(const char* state = "UNKNOWN", int32_t stream = -1, bool force = false) override; + void SynchronizeStream(int32_t stream) override; + void SynchronizeEvents(deviceEvent* evList, int32_t nEvents = 1) override; + void StreamWaitForEvents(int32_t stream, deviceEvent* evList, int32_t nEvents = 1) override; + bool IsEventDone(deviceEvent* evList, int32_t nEvents = 1) override; + + size_t WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream = -1, deviceEvent* ev = nullptr) override; + size_t GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev = nullptr, deviceEvent* evList = nullptr, int32_t nEvents = 1) override; + void ReleaseEvent(deviceEvent ev) override; + void RecordMarker(deviceEvent* ev, int32_t stream) override; + + template + int32_t AddKernel(); + + GPUReconstructionMetalInternals* mInternals; + float mOclVersion; + + template + S& getKernelObject(); + + int32_t GetMetalPrograms(); + + private: + int32_t AddKernels(); +}; + +} // namespace o2::gpu + +#endif // GPURECONSTRUCTIONMETAL_H diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm new file mode 100644 index 0000000000000..8448a9d86184b --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -0,0 +1,414 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "GPUReconstructionMetal.h" +#include "GPUConstantMem.h" +#include "GPUDefParametersLoad.inc" +#include "GPUReconstructionMetalIncludesHost.h" + +#include +#include + +#include +#include +#include // _mh_execute_header + +#define GPUErrorReturn(...) \ + { \ + GPUError(__VA_ARGS__); \ + return (1); \ + } + +#include "utils/qGetLdBinarySymbols.h" +QGET_LD_BINARY_SYMBOLS(GPUReconstructionMetalCode_src); + +GPUReconstruction* GPUReconstruction_Create_METAL(const GPUSettingsDeviceBackend& cfg) { return new GPUReconstructionMetal(cfg); } + +GPUReconstructionMetal::GPUReconstructionMetal(const GPUSettingsDeviceBackend& cfg) : GPUReconstructionProcessing::KernelInterface(cfg, sizeof(GPUReconstructionDeviceBase)) +{ + if (mMaster == nullptr) { + mInternals = new GPUReconstructionMetalInternals; + *mParDevice = o2::gpu::internal::GPUDefParametersLoad(); + } + mDeviceBackendSettings->deviceType = DeviceType::METAL; +} + +GPUReconstructionMetal::~GPUReconstructionMetal() +{ + Exit(); // Make sure we destroy everything (in particular the ITS tracker) before we exit + if (mMaster == nullptr) { + delete mInternals; + } +} + +int32_t GPUReconstructionMetal::InitDevice_Runtime() +{ + // Propagate processing settings to PoCL runtime. + // Won't affect other OpenCL runtimes. + if (int nThreads = mProcessingSettings->nHostThreads; nThreads > 0) { + auto nThreadsStr = std::to_string(nThreads); + setenv("PMETAL_CPU_MAX_CU_COUNT", nThreadsStr.c_str(), 1); + } + + if (mMaster == nullptr) { + mInternals->device = MTLCreateSystemDefaultDevice(); + + int64_t deviceGlobalMem, deviceLocalMem; + MTLSize deviceMaxWorkGroup = mInternals->device.maxThreadsPerThreadgroup; + + std::string device_name = [mInternals->device.name UTF8String]; + // On Apple Silicon, treat recommended working set as an upper bound + deviceGlobalMem = mInternals->device.recommendedMaxWorkingSetSize; + deviceLocalMem = mInternals->device.maxThreadgroupMemoryLength; + if (GetProcessingSettings().debugLevel >= 2) { + GPUInfo("Using Metal device %s with properties:", device_name.c_str()); + GPUInfo("\tUnified Memory Architecture = %ld ", mInternals->device.hasUnifiedMemory); + GPUInfo("\tRecommended Max Working Set = %ld bytes", deviceGlobalMem); + GPUInfo("\tMax thread group memory = %ld bytes", deviceLocalMem); + GPUInfo("\tmaxWorkGroup = (%ld, %ld, %ld)", deviceMaxWorkGroup.width, deviceMaxWorkGroup.height, deviceMaxWorkGroup.depth); + GPUInfo(" "); + } + + mDeviceName = device_name.c_str(); + // Basically a random number for now. + mMaxBackendThreads = 1000; + + if (GetMetalPrograms()) { + return 1; + } + + if (GetProcessingSettings().debugLevel >= 2) { + GPUInfo("Metal program and kernels loaded successfully"); + } + + // We only support internal GPUs, so the ownership of the memory is always shared + mInternals->mem_gpu = [mInternals->device newBufferWithLength:mDeviceMemorySize options:MTLResourceStorageModeShared]; + if (mInternals->mem_gpu == nil) { + GPUErrorReturn("Metal Memory Allocation Error"); + } + + // We only support internal GPUs, so the ownership of the memory is always shared. + // FIXME: until I understand how to enable the gGPUConstantMemBufferSize constexpr + int32_t tmpGPUContantMemBufferSize = 100000; // gGPUConstantMemBufferSize + mInternals->mem_constant = [mInternals->device newBufferWithLength:tmpGPUContantMemBufferSize options:MTLResourceStorageModeShared]; + if (mInternals->mem_constant) { + GPUErrorReturn("Metal Constant Memory Allocation Error"); + } + + for (int32_t i = 0; i < mNStreams; i++) { + mInternals->commandQueues[i] = [mInternals->device newCommandQueue]; + if (mInternals->commandQueues[i] == nil) { + GPUErrorReturn("Error creating Metal command queue"); + } + mInternals->commandBuffers[i] = [mInternals->commandQueues[i] commandBuffer]; + if (mInternals->commandBuffers[i] == nil) { + GPUErrorReturn("Error creating Metal command buffer"); + } + } + + mInternals->mem_host = [mInternals->device newBufferWithLength:mHostMemorySize options:MTLResourceStorageModeShared]; + if (mInternals->mem_host == nil) { + GPUErrorReturn("Error allocating pinned host memory"); + } + + mHostMemoryBase = mInternals->mem_host.contents; + mHostMemorySize = mInternals->mem_host.allocatedSize; + mDeviceMemoryBase = mInternals->mem_gpu.contents; + mDeviceMemorySize = mInternals->mem_gpu.allocatedSize; + mDeviceConstantMem = (GPUConstantMem*)mInternals->mem_constant.contents; + + if (GetProcessingSettings().debugLevel >= 1) { + GPUInfo("Memory ptrs: GPU (%ld bytes): %p - Host (%ld bytes): %p", (int64_t)mDeviceMemorySize, mDeviceMemoryBase, (int64_t)mHostMemorySize, mHostMemoryBase); + memset(mHostMemoryBase, 0xDD, mHostMemorySize); + } + + GPUInfo("Metal Initialisation successfull"); + } else { + auto* master = dynamic_cast(mMaster); + mWarpSize = master->mWarpSize; + mMaxBackendThreads = master->mMaxBackendThreads; + mDeviceName = master->mDeviceName; + mDeviceConstantMem = master->mDeviceConstantMem; + mInternals = master->mInternals; + } + + for (uint32_t i = 0; i < mEvents.size(); i++) { + auto* events = (id*)mEvents[i].data(); + new (events) id[ mEvents[i].size() ]; + } + + return (0); +} + +int32_t GPUReconstructionMetal::ExitDevice_Runtime() +{ + // Uninitialize OPENCL + SynchronizeGPU(); + + if (mMaster == nullptr) { + if (mDeviceMemoryBase) { + [mInternals->mem_gpu release]; + [mInternals->mem_constant release]; + for (uint32_t i = 0; i < mInternals->functions.size(); i++) { + [mInternals->functions[i] release]; + } + mInternals->functions.clear(); + } + if (mHostMemoryBase) { + for (int32_t i = 0; i < mNStreams; i++) { + [mInternals->commandQueues[i] release]; + [mInternals->commandBuffers[i] release]; + } + [mInternals->mem_host release]; + } + + [mInternals->library release]; + [mInternals->device release]; + GPUInfo("Metal disposed correctly"); + } + mDeviceMemoryBase = nullptr; + mHostMemoryBase = nullptr; + + return (0); +} + +size_t GPUReconstructionMetal::GPUMemCpy(void* dst, const void* src, size_t sizeBytes, int32_t stream, int32_t toGPU, deviceEvent* ev, deviceEvent* evList, int32_t nEvents) +{ + if (evList == nullptr) { + nEvents = 0; + } + if (GetProcessingSettings().debugLevel >= 3) { + stream = -1; + } + + if (stream == -1) { + SynchronizeGPU(); + } + + auto realStream = stream == -1 ? 0 : stream; + id cb = mInternals->commandBuffers[realStream]; + id blit = [cb blitCommandEncoder]; + id sourceBuffer = nil; + id destBuffer = nil; + ptrdiff_t sourceOffset = 0; + ptrdiff_t destOffset = 0; + + // Sigh. + if (src > mHostMemoryBase && src < ((char*)mHostMemoryBase + mHostMemorySize)) { + sourceBuffer = mInternals->mem_host; + sourceOffset = (char*)src - (char*)mHostMemoryBase; + } else if (src > mDeviceMemoryBase && src < ((char*)mDeviceMemoryBase + mDeviceMemorySize)) { + sourceBuffer = mInternals->mem_gpu; + sourceOffset = (char*)src - (char*)mDeviceMemoryBase; + } else { + GPUErrorReturn("Unknown buffer at %x", src); + } + + if (dst > mHostMemoryBase && dst < ((char*)mHostMemoryBase + mHostMemorySize)) { + destBuffer = mInternals->mem_host; + destOffset = (char*)src - (char*)mHostMemoryBase; + } else if (dst > mDeviceMemoryBase && dst < ((char*)mDeviceMemoryBase + mDeviceMemorySize)) { + destBuffer = mInternals->mem_gpu; + destOffset = (char*)dst - (char*)mDeviceMemoryBase; + } else { + GPUErrorReturn("Unknown buffer at %x", src); + } + + [blit copyFromBuffer:sourceBuffer + sourceOffset:sourceOffset + toBuffer:destBuffer + destinationOffset:destOffset + size:sizeBytes]; + + [blit endEncoding]; + [cb commit]; + + if (GetProcessingSettings().serializeGPU & 2) { + GPUDebug(("GPUMemCpy " + std::to_string(toGPU)).c_str(), stream, true); + } + return sizeBytes; +} + +size_t GPUReconstructionMetal::WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream, deviceEvent* ev) +{ + if (stream == -1) { + SynchronizeGPU(); + } + + auto realStream = stream == -1 ? 0 : stream; + id cb = mInternals->commandBuffers[realStream]; + id blit = [cb blitCommandEncoder]; + id sourceBuffer = nil; + ptrdiff_t sourceOffset = 0; + if (src > mHostMemoryBase && src < ((char*)mHostMemoryBase + mHostMemorySize)) { + sourceBuffer = mInternals->mem_host; + sourceOffset = (char*)src - (char*)mHostMemoryBase; + } else if (src > mDeviceMemoryBase && src < ((char*)mDeviceMemoryBase + mDeviceMemorySize)) { + sourceBuffer = mInternals->mem_gpu; + sourceOffset = (char*)src - (char*)mDeviceMemoryBase; + } else { + GPUErrorReturn("Unknown buffer at %x", src); + } + [blit copyFromBuffer:sourceBuffer + sourceOffset:sourceOffset + toBuffer:mInternals->mem_constant + destinationOffset:offset + size:size]; + + [blit endEncoding]; + [cb commit]; + + if (GetProcessingSettings().serializeGPU & 2) { + GPUDebug("WriteToConstantMemory", stream, true); + } + return size; +} + +void GPUReconstructionMetal::ReleaseEvent(deviceEvent ev) +{ + // FIXME: is this supposed to reset the event for it to be repurposed + // or to decrease the ref count? + auto mtlEvent = (__bridge id)(ev.get()); + [mtlEvent setSignaledValue:0]; +} + +void GPUReconstructionMetal::RecordMarker(deviceEvent* ev, int32_t stream) +{ + id cb = mInternals->commandBuffers[stream]; + // Does not change the retain count, so it's important we manage + // the lifetime of the events outside here. + auto mtlEvent = (__bridge id)(ev->get()); + [cb encodeSignalEvent:mtlEvent value:1]; + [cb commit]; +} + +void GPUReconstructionMetal::SynchronizeGPU() +{ + for (int32_t i = 0; i < mNStreams; i++) { + [mInternals->commandBuffers[i] waitUntilCompleted]; + } +} + +void GPUReconstructionMetal::SynchronizeStream(int32_t stream) +{ + [mInternals->commandBuffers[stream] waitUntilCompleted]; +} + +void GPUReconstructionMetal::SynchronizeEvents(deviceEvent* evList, int32_t nEvents) +{ + // I wait for everything to complete for now... + for (int32_t si = 0; si < mNStreams; si++) { + id cb = mInternals->commandBuffers[si]; + [cb waitUntilCompleted]; + } +} + +void GPUReconstructionMetal::StreamWaitForEvents(int32_t stream, deviceEvent* evList, int32_t nEvents) +{ + // Encode commands to wait for all the events + id cb = mInternals->commandBuffers[stream]; + for (int32_t ei = 0; ei < nEvents; ei++) { + auto mtlEvent = (__bridge id)(evList[ei].get()); + [cb encodeWaitForEvent:mtlEvent value:1]; + } + [cb commit]; + [cb waitUntilCompleted]; +} + +bool GPUReconstructionMetal::IsEventDone(deviceEvent* evList, int32_t nEvents) +{ + for (int32_t i = 0; i < nEvents; i++) { + auto mtlEvent = (__bridge id)(evList[i].get()); + if (mtlEvent.signaledValue == 0) { + return false; + } + } + return true; +} + +int32_t GPUReconstructionMetal::GPUDebug(const char* state, int32_t stream, bool force) +{ + // Wait for Metal-Kernel to finish and check for Metal errors afterwards, in case of debugmode + if (!force && GetProcessingSettings().debugLevel <= 0) { + return (0); + } + for (int32_t si = 0; si < mNStreams; si++) { + [mInternals->commandBuffers[si] waitUntilCompleted]; + } + if (GetProcessingSettings().debugLevel >= 3) { + GPUInfo("GPU Sync Done"); + } + return (0); +} + +int32_t GPUReconstructionMetal::GPUChkErrInternal(const int64_t error, const char* file, int32_t line) const +{ + // Not sure how metal returns errors. + if (error != 0) { + GPUError("Metal Error: %ld / %s (%s:%d)", error, "Unknown", file, line); + } + return error != 0; +} + +// Return pointer+size for (__DATA|__DATA_CONST, "__gpu_resource") from the image +// that matches `image_name_substr` (e.g. "libO2GPUReconstruction.dylib"). +static const uint8_t* find_gpu_resource_in_image(const char* image_name_substr, + unsigned long* out_size) +{ + uint32_t count = _dyld_image_count(); + for (uint32_t i = 0; i < count; ++i) { + const char* name = _dyld_get_image_name(i); + if (!name || !strstr(name, image_name_substr)) { + continue; + } + + const struct mach_header* mh = _dyld_get_image_header(i); + + const auto* mh64 = (const struct mach_header_64*)mh; + const auto* p = (const uint8_t*) + getsectiondata(mh64, "__DATA", "__gpu_resource", out_size); + if (!p) { + p = (const uint8_t*) + getsectiondata(mh64, "__DATA_CONST", "__gpu_resource", out_size); + } + if (p) { + return p; + } + } + return nullptr; +} + +int32_t GPUReconstructionMetal::GetMetalPrograms() +{ + GPUInfo("Loading Metal library (Platform version %s)", [mInternals->device.architecture.name cStringUsingEncoding:NSUTF8StringEncoding]); + + unsigned long sz = 0; + const uint8_t* p = find_gpu_resource_in_image("libO2GPUTrackingMETAL.dylib", &sz); + if (p == nullptr || sz == 0) { + GPUError("Metal library not found in the __gpu_resource section"); + return 1; + } + + // the section is part of the mapped image, so it outlives the dispatch_data_t + // and does not have to be copied + dispatch_data_t blob = dispatch_data_create(p, sz, nullptr, ^{}); + + NSError* error = nil; + mInternals->library = [mInternals->device newLibraryWithData:blob error:&error]; + + if (mInternals->library == nil) { + NSLog(@"%@", error); + GPUError("Error loading the Metal library"); + return 1; + } + + return AddKernels(); +} diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h new file mode 100644 index 0000000000000..e8c62623cef5a --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h @@ -0,0 +1,61 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef GPURECONSTRUCTIONOMETALINCLUDESHOST_H +#define GPURECONSTRUCTIONOMETALINCLUDESHOST_H + +#define GPUCA_GPUTYPE_METAL + +#import +#ifndef __METAL_VERSION__ +// __METAL_VERSION__ is only defined in device code. +#define __METAL_HOST__ +#endif + +#import + +#include +#include +#include +#include "GPULogging.h" + +#include "GPUReconstructionMetal.h" +#include "GPUReconstructionIncludes.h" +#include "GPUCommonHelpers.h" + +using namespace o2::gpu; + +#include +#include +#include +#include + +namespace o2::gpu +{ + +struct GPUReconstructionMetalInternals { + id device; + + std::array, constants::GPU_MAX_STREAMS> commandQueues; // ~ cl_command_queue[] + std::array, constants::GPU_MAX_STREAMS> commandBuffers; + + std::vector> functions; // ~ cl_kernel (symbols) + std::vector> pipelines; // compiled kernels + + id mem_gpu; // ~ cl_mem (device/global) + id mem_constant; // ~ cl_mem (constant-like) + id mem_host; // ~ cl_mem (host-visible) + + id library; // ~ cl_program +}; +} // namespace o2::gpu + +#endif // GPURECONSTRUCTIONOMETALINCLUDESHOST_H diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm new file mode 100644 index 0000000000000..54adb3076b60d --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm @@ -0,0 +1,82 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include "GPUReconstructionKernelIncludes.h" +#include "GPUReconstructionMetalIncludesHost.h" + +#include "GPUReconstructionMetalKernelsSpecialize.inc" +#include "GPUReconstructionProcessingKernels.inc" + +template void GPUReconstructionProcessing::KernelInterface::runKernelVirtual(const int num, const void* args); + +template +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, const Args&... args) +{ + id function = mInternals->functions[GetKernelNum()]; + auto& kExec = _xyz.x; + auto& runRange = _xyz.y; + // auto& events = _xyz.z; + // auto& t = _xyz.t; + + NSError* error = nil; + auto pso = [mInternals->device newComputePipelineStateWithFunction:function error:&error]; + id computeEncoder = [mInternals->commandBuffers[kExec.stream] computeCommandEncoder]; + + // Map buffers and states + [computeEncoder setComputePipelineState:pso]; + [computeEncoder setBuffer:mInternals->mem_gpu offset:0 atIndex:0]; + [computeEncoder setBuffer:mInternals->mem_constant offset:0 atIndex:1]; + [computeEncoder setBuffer:mInternals->mem_host offset:0 atIndex:2]; + + MTLSize gridSize = MTLSizeMake(runRange.index, 1, 1); + + NSUInteger threadGroupSize = pso.maxTotalThreadsPerThreadgroup; + if (threadGroupSize > runRange.index) { + threadGroupSize = runRange.index; + } + + MTLSize threadgroupSize = MTLSizeMake(threadGroupSize, 1, 1); + [computeEncoder dispatchThreads:gridSize + threadsPerThreadgroup:threadgroupSize]; +} + +template +int32_t GPUReconstructionMetal::AddKernel() +{ + NSString* kname = [[NSString alloc] initWithFormat:@"krnl_%s", GetKernelName()]; + + id krnl = [mInternals->library newFunctionWithName:kname]; + if (krnl == nil) { + GPUError("Error creating Metal Kernel: %s", [kname cStringUsingEncoding:NSUTF8StringEncoding]); + return 1; + } + + mInternals->functions.emplace_back(krnl); + return 0; +} + +template +S& GPUReconstructionMetal::getKernelObject() +{ + return mInternals->functions[GetKernelNum()]; +} + +int32_t GPUReconstructionMetal::AddKernels() +{ +#define GPUCA_KRNL(x_class, ...) \ + if (AddKernel()) { \ + return 1; \ + } +#include "GPUReconstructionKernelList.h" +#undef GPUCA_KRNL + return 0; +} diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc new file mode 100644 index 0000000000000..1ee192cff822f --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -0,0 +1,27 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUReconstructionMetalKernelsSpecialize.inc + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size) +{ + const uint64_t offset = static_cast(ptr) - static_cast(mDeviceMemoryBase); + const uint64_t length = (size + 15ull) & ~15ull; + + id cb = mInternals->commandBuffers[_xyz.x.stream]; + id blit = [cb blitCommandEncoder]; + + [blit fillBuffer:mInternals->mem_gpu range:NSMakeRange(offset, length) value:0]; + [blit endEncoding]; + + [cb commit]; +} diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt index ce3cbca5bb197..cc44b2003a01d 100644 --- a/GPU/GPUTracking/CMakeLists.txt +++ b/GPU/GPUTracking/CMakeLists.txt @@ -463,7 +463,9 @@ if (onnxruntime_FOUND) endif() # Add CMake recipes for GPU Tracking librararies -if(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED) +if(METAL_ENABLED) + add_subdirectory(Base/metal) +elseif(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED) make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch) set(GPU_CONST_PARAM_FILES) foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES}) diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index 5804b046c915e..312d6b7f0391c 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -10,7 +10,7 @@ # or submit itself to any jurisdiction. # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources -# FindO2GPU.cmake Version 19 +# FindO2GPU.cmake Version 20 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual) set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908) @@ -20,17 +20,23 @@ set(HIP_AMDGPUTARGET_DEFAULT_MINIMAL gfx906) if(NOT DEFINED ENABLE_CUDA) set(ENABLE_CUDA "AUTO") endif() -if(NOT APPLE) - # macOS ships OpenCL 1.2 only, below the 2.x that the OpenCL backend needs. - if(NOT DEFINED ENABLE_OPENCL) - set(ENABLE_OPENCL "AUTO") +if(APPLE) + # macOS ships OpenCL 1.2 only, below the 2.x the OpenCL backend needs; Metal + # replaces it there. OFF rather than AUTO while the backend is unproven. + if(NOT DEFINED ENABLE_METAL) + set(ENABLE_METAL "OFF") endif() +elseif(NOT DEFINED ENABLE_OPENCL) + set(ENABLE_OPENCL "AUTO") endif() if(NOT DEFINED ENABLE_HIP) set(ENABLE_HIP "AUTO") endif() string(TOUPPER "${ENABLE_CUDA}" ENABLE_CUDA) string(TOUPPER "${ENABLE_OPENCL}" ENABLE_OPENCL) +if(APPLE) + string(TOUPPER "${ENABLE_METAL}" ENABLE_METAL) +endif() string(TOUPPER "${ENABLE_HIP}" ENABLE_HIP) if(NOT DEFINED CMAKE_BUILD_TYPE_UPPER) string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER) @@ -152,6 +158,7 @@ if(GPUCA_DETERMINISTIC_NO_FTZ) set(GPUCA_CXX_DENORMALS_FLAGS "") set(GPUCA_CUDA_DENORMALS_FLAGS "--ftz=false") set(GPUCA_OCL_DENORMALS_FLAGS "") + set(GPUCA_METAL_DENORMALS_FLAGS "-fno-fast-math") set(GPUCA_HIP_DENORMALS_FLAGS "-fno-gpu-flush-denormals-to-zero") else() if (CMAKE_SYSTEM_NAME MATCHES Darwin OR NOT CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") @@ -161,6 +168,7 @@ else() endif() set(GPUCA_CUDA_DENORMALS_FLAGS "--ftz=true") set(GPUCA_OCL_DENORMALS_FLAGS "-cl-denorms-are-zero") + set(GPUCA_METAL_DENORMALS_FLAGS "-ffast-math") set(GPUCA_HIP_DENORMALS_FLAGS "-fgpu-flush-denormals-to-zero") endif() set(GPUCA_CXX_NO_FAST_MATH_FLAGS "-fno-fast-math -ffp-contract=off") @@ -432,6 +440,24 @@ if(ENABLE_HIP) endif() endif() +# =================================== Metal ================================== +if(ENABLE_METAL) + find_library(METAL_FRAMEWORK Metal) + find_library(COREFOUNDATION_FRAMEWORK CoreFoundation) + find_library(FOUNDATION_FRAMEWORK Foundation) + find_library(QUARTZCORE_FRAMEWORK QuartzCore) + if(METAL_FRAMEWORK AND COREFOUNDATION_FRAMEWORK AND FOUNDATION_FRAMEWORK AND QUARTZCORE_FRAMEWORK) + set(METAL_ENABLED ON) + set(METAL_FRAMEWORKS ${METAL_FRAMEWORK} ${COREFOUNDATION_FRAMEWORK} + ${FOUNDATION_FRAMEWORK} ${QUARTZCORE_FRAMEWORK}) + message(STATUS "Found Metal frameworks") + elseif(NOT ENABLE_METAL STREQUAL "AUTO") + message(FATAL_ERROR "Metal frameworks not available") + else() + set(METAL_ENABLED OFF) + endif() +endif() + # if we end up here without a FATAL, it means we have found the "O2GPU" package set(O2GPU_FOUND TRUE) if (NOT GPUCA_FINDO2GPU_CHECK_ONLY) From b5d823aece2576bd719b7bfb208b1262af8b3609 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:42 +0200 Subject: [PATCH 04/41] TRD: let PadPlane be read from a device without double MSL has no double, and rejects it at the declaration, so PadPlane could not be declared at all in a Metal translation unit -- never mind read. It is embedded by value in GeometryBase, which is embedded in GeometryFlat, which GPUTRDGeometry inherits, so this blocked the whole TRD geometry on the device. Storage is deliberately left alone. GPUdoubleStore occupies the same eight bytes with the same alignment as a double, so the object layout is identical on both sides and the host still writes and uploads exactly the bytes it did before; only the device declaration differs, and the stored bits are decoded to float on read. On every other backend GPUdoubleStore is a plain double and GPUdoubleGet() is the identity, so CUDA, HIP and the host are untouched -- including ROOT I/O, which only ever sees the host type. Narrowing to float on the device costs nothing that was not already lost: GPUTRDGeometry truncates every one of these accessors to float anyway. The decoder is exact, not approximate: round to nearest even, with the subnormal, infinity and NaN cases handled. It agrees bit-for-bit with a real double to float conversion over four million values, two million uniform across the geometry range and two million random bit patterns. A faster branchless variant is possible and is left for later if it ever shows up in a profile. Takes PadPlane.h from 73 errors to 0 in a Metal translation unit. --- Detectors/TRD/base/include/TRDBase/PadPlane.h | 157 +++++++++--------- GPU/Common/GPUCommonDouble.h | 92 ++++++++++ 2 files changed, 172 insertions(+), 77 deletions(-) create mode 100644 GPU/Common/GPUCommonDouble.h diff --git a/Detectors/TRD/base/include/TRDBase/PadPlane.h b/Detectors/TRD/base/include/TRDBase/PadPlane.h index 24c67a8453b48..39132de72d404 100644 --- a/Detectors/TRD/base/include/TRDBase/PadPlane.h +++ b/Detectors/TRD/base/include/TRDBase/PadPlane.h @@ -15,6 +15,7 @@ // Forwards to standard header with protection for GPU compilation #include "GPUCommonRtypes.h" // for ClassDef #include "GPUCommonDef.h" +#include "GPUCommonDouble.h" //////////////////////////////////////////////////////////////////////////// // // @@ -39,53 +40,55 @@ class PadPlane PadPlane& operator=(const PadPlane& p) = delete; ~PadPlane() = default; +#ifndef GPUCA_GPUCODE_DEVICE void setLayer(int l) { mLayer = l; }; void setStack(int s) { mStack = s; }; - void setRowSpacing(double s) { mRowSpacing = s; }; - void setColSpacing(double s) { mColSpacing = s; }; - void setLengthRim(double l) { mLengthRim = l; }; - void setWidthRim(double w) { mWidthRim = w; }; + void setRowSpacing(o2::gpu::GPUdoubleValue s) { mRowSpacing = s; }; + void setColSpacing(o2::gpu::GPUdoubleValue s) { mColSpacing = s; }; + void setLengthRim(o2::gpu::GPUdoubleValue l) { mLengthRim = l; }; + void setWidthRim(o2::gpu::GPUdoubleValue w) { mWidthRim = w; }; void setNcols(int n); void setNrows(int n); - void setPadCol(int ic, double c) + void setPadCol(int ic, o2::gpu::GPUdoubleValue c) { if (ic < mNcols) { mPadCol[ic] = c; } }; - void setPadRow(int ir, double r) + void setPadRow(int ir, o2::gpu::GPUdoubleValue r) { if (ir < mNrows) { mPadRow[ir] = r; } }; - void setLength(double l) { mLength = l; }; - void setWidth(double w) { mWidth = w; }; - void setLengthOPad(double l) + void setLength(o2::gpu::GPUdoubleValue l) { mLength = l; }; + void setWidth(o2::gpu::GPUdoubleValue w) { mWidth = w; }; + void setLengthOPad(o2::gpu::GPUdoubleValue l) { mLengthOPad = l; mInverseLengthOPad = 1.0 / l; }; - void setWidthOPad(double w) + void setWidthOPad(o2::gpu::GPUdoubleValue w) { mWidthOPad = w; mInverseWidthOPad = 1.0 / w; }; - void setLengthIPad(double l) + void setLengthIPad(o2::gpu::GPUdoubleValue l) { mLengthIPad = l; mInverseLengthIPad = 1.0 / l; }; - void setWidthIPad(double w) + void setWidthIPad(o2::gpu::GPUdoubleValue w) { mWidthIPad = w; mInverseWidthIPad = 1.0 / w; }; - void setPadRowSMOffset(double o) { mPadRowSMOffset = o; }; + void setPadRowSMOffset(o2::gpu::GPUdoubleValue o) { mPadRowSMOffset = o; }; void setAnodeWireOffset(float o) { mAnodeWireOffset = o; }; - void setTiltingAngle(double t); + void setTiltingAngle(o2::gpu::GPUdoubleValue t); +#endif - GPUd() int getPadRowNumber(double z) const + GPUd() int getPadRowNumber(o2::gpu::GPUdoubleValue z) const { // // Finds the pad row number for a given z-position in local supermodule system @@ -103,10 +106,10 @@ class PadPlane nbelow = 0; while (nabove - nbelow > 1) { middle = (nabove + nbelow) / 2; - if (z == (mPadRow[middle - 1] + mPadRowSMOffset)) { + if (z == (o2::gpu::GPUdoubleGet(mPadRow[middle - 1]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset))) { row = middle; } - if (z > (mPadRow[middle - 1] + mPadRowSMOffset)) { + if (z > (o2::gpu::GPUdoubleGet(mPadRow[middle - 1]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset))) { nabove = middle; } else { nbelow = middle; @@ -118,124 +121,124 @@ class PadPlane return row; }; - GPUd() int getPadRowNumberROC(double z) const; - GPUd() double getPadRow(double z) const; - GPUd() int getPadColNumber(double rphi) const; - GPUd() double getPad(double y, double z) const; + GPUd() int getPadRowNumberROC(o2::gpu::GPUdoubleValue z) const; + GPUd() o2::gpu::GPUdoubleValue getPadRow(o2::gpu::GPUdoubleValue z) const; + GPUd() int getPadColNumber(o2::gpu::GPUdoubleValue rphi) const; + GPUd() o2::gpu::GPUdoubleValue getPad(o2::gpu::GPUdoubleValue y, o2::gpu::GPUdoubleValue z) const; - GPUd() double getTiltOffset(int row, double rowOffset) const + GPUd() o2::gpu::GPUdoubleValue getTiltOffset(int row, o2::gpu::GPUdoubleValue rowOffset) const { if (row == 0 || row == mNrows - 1) { - return mTiltingTan * (rowOffset - 0.5 * mLengthOPad); + return o2::gpu::GPUdoubleGet(mTiltingTan) * (rowOffset - 0.5 * o2::gpu::GPUdoubleGet(mLengthOPad)); } else { - return mTiltingTan * (rowOffset - 0.5 * mLengthIPad); + return o2::gpu::GPUdoubleGet(mTiltingTan) * (rowOffset - 0.5 * o2::gpu::GPUdoubleGet(mLengthIPad)); } }; - GPUd() double getPadRowOffset(int row, double z) const + GPUd() o2::gpu::GPUdoubleValue getPadRowOffset(int row, o2::gpu::GPUdoubleValue z) const { if ((row < 0) || (row >= mNrows)) { return -1.0; } else { - return mPadRow[row] + mPadRowSMOffset - z; + return o2::gpu::GPUdoubleGet(mPadRow[row]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset) - z; } }; - GPUd() double getPadRowOffsetROC(int row, double z) const + GPUd() o2::gpu::GPUdoubleValue getPadRowOffsetROC(int row, o2::gpu::GPUdoubleValue z) const { if ((row < 0) || (row >= mNrows)) { return -1.0; } else { - return mPadRow[row] - z; + return o2::gpu::GPUdoubleGet(mPadRow[row]) - z; } }; - GPUd() double getPadColOffset(int col, double rphi) const + GPUd() o2::gpu::GPUdoubleValue getPadColOffset(int col, o2::gpu::GPUdoubleValue rphi) const { if ((col < 0) || (col >= mNcols)) { return -1.0; } else { - return rphi - mPadCol[col]; + return rphi - o2::gpu::GPUdoubleGet(mPadCol[col]); } }; - GPUd() double getTiltingAngle() const { return mTiltingAngle; }; + GPUd() o2::gpu::GPUdoubleValue getTiltingAngle() const { return o2::gpu::GPUdoubleGet(mTiltingAngle); }; GPUd() int getNrows() const { return mNrows; }; GPUd() int getNcols() const { return mNcols; }; - GPUd() double getRow0() const { return mPadRow[0] + mPadRowSMOffset; }; - GPUd() double getRow0ROC() const { return mPadRow[0]; }; - GPUd() double getCol0() const { return mPadCol[0]; }; - GPUd() double getRowEnd() const { return mPadRow[mNrows - 1] - mLengthOPad + mPadRowSMOffset; }; - GPUd() double getRowEndROC() const { return mPadRow[mNrows - 1] - mLengthOPad; }; - GPUd() double getColEnd() const { return mPadCol[mNcols - 1] + mWidthOPad; }; - GPUd() double getRowPos(int row) const { return mPadRow[row] + mPadRowSMOffset; }; - GPUd() double getRowPosROC(int row) const { return mPadRow[row]; }; - GPUd() double getColPos(int col) const { return mPadCol[col]; }; - GPUd() double getRowSize(int row) const + GPUd() o2::gpu::GPUdoubleValue getRow0() const { return o2::gpu::GPUdoubleGet(mPadRow[0]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset); }; + GPUd() o2::gpu::GPUdoubleValue getRow0ROC() const { return o2::gpu::GPUdoubleGet(mPadRow[0]); }; + GPUd() o2::gpu::GPUdoubleValue getCol0() const { return o2::gpu::GPUdoubleGet(mPadCol[0]); }; + GPUd() o2::gpu::GPUdoubleValue getRowEnd() const { return o2::gpu::GPUdoubleGet(mPadRow[mNrows - 1]) - o2::gpu::GPUdoubleGet(mLengthOPad) + o2::gpu::GPUdoubleGet(mPadRowSMOffset); }; + GPUd() o2::gpu::GPUdoubleValue getRowEndROC() const { return o2::gpu::GPUdoubleGet(mPadRow[mNrows - 1]) - o2::gpu::GPUdoubleGet(mLengthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getColEnd() const { return o2::gpu::GPUdoubleGet(mPadCol[mNcols - 1]) + o2::gpu::GPUdoubleGet(mWidthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getRowPos(int row) const { return o2::gpu::GPUdoubleGet(mPadRow[row]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset); }; + GPUd() o2::gpu::GPUdoubleValue getRowPosROC(int row) const { return o2::gpu::GPUdoubleGet(mPadRow[row]); }; + GPUd() o2::gpu::GPUdoubleValue getColPos(int col) const { return o2::gpu::GPUdoubleGet(mPadCol[col]); }; + GPUd() o2::gpu::GPUdoubleValue getRowSize(int row) const { if ((row == 0) || (row == mNrows - 1)) { - return mLengthOPad; + return o2::gpu::GPUdoubleGet(mLengthOPad); } else { - return mLengthIPad; + return o2::gpu::GPUdoubleGet(mLengthIPad); } }; - GPUd() double getColSize(int col) const + GPUd() o2::gpu::GPUdoubleValue getColSize(int col) const { if ((col == 0) || (col == mNcols - 1)) { - return mWidthOPad; + return o2::gpu::GPUdoubleGet(mWidthOPad); } else { - return mWidthIPad; + return o2::gpu::GPUdoubleGet(mWidthIPad); } }; - GPUd() double getLengthRim() const { return mLengthRim; }; - GPUd() double getWidthRim() const { return mWidthRim; }; - GPUd() double getRowSpacing() const { return mRowSpacing; }; - GPUd() double getColSpacing() const { return mColSpacing; }; - GPUd() double getLengthOPad() const { return mLengthOPad; }; - GPUd() double getLengthIPad() const { return mLengthIPad; }; - GPUd() double getWidthOPad() const { return mWidthOPad; }; - GPUd() double getWidthIPad() const { return mWidthIPad; }; - GPUd() double getAnodeWireOffset() const { return mAnodeWireOffset; }; + GPUd() o2::gpu::GPUdoubleValue getLengthRim() const { return o2::gpu::GPUdoubleGet(mLengthRim); }; + GPUd() o2::gpu::GPUdoubleValue getWidthRim() const { return o2::gpu::GPUdoubleGet(mWidthRim); }; + GPUd() o2::gpu::GPUdoubleValue getRowSpacing() const { return o2::gpu::GPUdoubleGet(mRowSpacing); }; + GPUd() o2::gpu::GPUdoubleValue getColSpacing() const { return o2::gpu::GPUdoubleGet(mColSpacing); }; + GPUd() o2::gpu::GPUdoubleValue getLengthOPad() const { return o2::gpu::GPUdoubleGet(mLengthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getLengthIPad() const { return o2::gpu::GPUdoubleGet(mLengthIPad); }; + GPUd() o2::gpu::GPUdoubleValue getWidthOPad() const { return o2::gpu::GPUdoubleGet(mWidthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getWidthIPad() const { return o2::gpu::GPUdoubleGet(mWidthIPad); }; + GPUd() o2::gpu::GPUdoubleValue getAnodeWireOffset() const { return o2::gpu::GPUdoubleGet(mAnodeWireOffset); }; private: - static constexpr int MAXCOLS = 144; - static constexpr int MAXROWS = 16; + static GPUglobalconstexpr() int MAXCOLS = 144; + static GPUglobalconstexpr() int MAXROWS = 16; int mLayer; // Layer number int mStack; // Stack number - double mLength; // Length of pad plane in z-direction (row) - double mWidth; // Width of pad plane in rphi-direction (col) + o2::gpu::GPUdoubleStore mLength; // Length of pad plane in z-direction (row) + o2::gpu::GPUdoubleStore mWidth; // Width of pad plane in rphi-direction (col) - double mLengthRim; // Length of the rim in z-direction (row) - double mWidthRim; // Width of the rim in rphi-direction (col) + o2::gpu::GPUdoubleStore mLengthRim; // Length of the rim in z-direction (row) + o2::gpu::GPUdoubleStore mWidthRim; // Width of the rim in rphi-direction (col) - double mLengthOPad; // Length of an outer pad in z-direction (row) - double mWidthOPad; // Width of an outer pad in rphi-direction (col) + o2::gpu::GPUdoubleStore mLengthOPad; // Length of an outer pad in z-direction (row) + o2::gpu::GPUdoubleStore mWidthOPad; // Width of an outer pad in rphi-direction (col) - double mLengthIPad; // Length of an inner pad in z-direction (row) - double mWidthIPad; // Width of an inner pad in rphi-direction (col) + o2::gpu::GPUdoubleStore mLengthIPad; // Length of an inner pad in z-direction (row) + o2::gpu::GPUdoubleStore mWidthIPad; // Width of an inner pad in rphi-direction (col) - double mRowSpacing; // Spacing between the pad rows - double mColSpacing; // Spacing between the pad columns + o2::gpu::GPUdoubleStore mRowSpacing; // Spacing between the pad rows + o2::gpu::GPUdoubleStore mColSpacing; // Spacing between the pad columns int mNrows; // Number of rows int mNcols; // Number of columns - double mTiltingAngle; // Pad tilting angle - double mTiltingTan; // Tangens of pad tilting angle + o2::gpu::GPUdoubleStore mTiltingAngle; // Pad tilting angle + o2::gpu::GPUdoubleStore mTiltingTan; // Tangens of pad tilting angle - double mPadRow[MAXROWS]; // Pad border positions in row direction - double mPadCol[MAXCOLS]; // Pad border positions in column direction + o2::gpu::GPUdoubleStore mPadRow[MAXROWS]; // Pad border positions in row direction + o2::gpu::GPUdoubleStore mPadCol[MAXCOLS]; // Pad border positions in column direction - double mPadRowSMOffset; // To be added to translate local ROC system to local SM system + o2::gpu::GPUdoubleStore mPadRowSMOffset; // To be added to translate local ROC system to local SM system - double mAnodeWireOffset; // Distance of first anode wire from pad edge + o2::gpu::GPUdoubleStore mAnodeWireOffset; // Distance of first anode wire from pad edge - double mInverseLengthIPad; // 1 / mLengthIPad - double mInverseLengthOPad; // 1 / mLengthOPad + o2::gpu::GPUdoubleStore mInverseLengthIPad; // 1 / mLengthIPad + o2::gpu::GPUdoubleStore mInverseLengthOPad; // 1 / mLengthOPad - double mInverseWidthIPad; // 1 / mWidthIPad - double mInverseWidthOPad; // 1 / mWidthOPad + o2::gpu::GPUdoubleStore mInverseWidthIPad; // 1 / mWidthIPad + o2::gpu::GPUdoubleStore mInverseWidthOPad; // 1 / mWidthOPad ClassDefNV(PadPlane, 2); // TRD ROC pad plane }; diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h new file mode 100644 index 0000000000000..c5baa9904b7e1 --- /dev/null +++ b/GPU/Common/GPUCommonDouble.h @@ -0,0 +1,92 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUCommonDouble.h +/// \brief Storage for double precision members shared with a device that has none + +#ifndef GPUCOMMONDOUBLE_H +#define GPUCOMMONDOUBLE_H + +#include "GPUCommonDef.h" + +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif + +namespace o2::gpu +{ + +#ifdef __METAL__ + +// MSL has no double, and rejects it at the declaration, so a struct holding one +// cannot even be declared. GPUdoubleStore occupies the same eight bytes with the +// same alignment, which keeps the object layout identical to the host's, and the +// stored bits are decoded on read. Storage is therefore untouched: the host still +// writes a double and the same bytes are uploaded. +struct alignas(8) GPUdoubleStore { + uint32_t mLo; + uint32_t mHi; +}; +typedef float GPUdoubleValue; + +// IEEE-754 binary64 -> binary32, round to nearest even, including the subnormal, +// infinity and NaN cases. +GPUdi() GPUdoubleValue GPUdoubleGet(GPUdoubleStore d) +{ + const uint32_t sign = d.mHi & 0x80000000u; + const int32_t be = int32_t((d.mHi >> 20) & 0x7ffu); + const uint32_t man = ((d.mHi & 0x000fffffu) << 3) | (d.mLo >> 29); + const uint32_t drop = d.mLo & 0x1fffffffu; + uint32_t bits; + if (be == 0x7ff) { + bits = sign | 0x7f800000u | (((d.mHi & 0x000fffffu) | d.mLo) ? 0x400000u : 0u); + } else if (be == 0) { + bits = sign; + } else { + const int32_t e = be - 1023 + 127; + if (e >= 0xff) { + bits = sign | 0x7f800000u; + } else if (e > 0) { + bits = sign | (uint32_t(e) << 23) | man; + if ((drop & 0x10000000u) && ((drop & 0x0fffffffu) || (man & 1u))) { + bits += 1u; + } + } else if (e > -24) { + const uint32_t full = man | 0x800000u; + const uint32_t sh = uint32_t(1 - e); + const uint32_t lost = full & ((1u << sh) - 1u); + const uint32_t halfb = 1u << (sh - 1); + uint32_t sub = full >> sh; + if (lost > halfb || (lost == halfb && ((sub & 1u) || drop))) { + sub += 1u; + } + bits = sign | sub; + } else { + bits = sign; + } + } + return __builtin_bit_cast(float, bits); +} + +#else + +typedef double GPUdoubleStore; +typedef double GPUdoubleValue; +GPUhdi() GPUdoubleValue GPUdoubleGet(GPUdoubleStore d) { return d; } + +#endif + +static_assert(sizeof(GPUdoubleStore) == 8, "GPUdoubleStore must match the size of a double"); +static_assert(alignof(GPUdoubleStore) == 8, "GPUdoubleStore must match the alignment of a double"); + +} // namespace o2::gpu + +#endif // GPUCOMMONDOUBLE_H From 2977ddb6af8b855047337ad66e0529b6e3524756 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:44 +0200 Subject: [PATCH 05/41] Common: keep the double-only constants and helpers off a device without double Two headers reach GPUConstantMem.h through PID.h and Propagator.h and account for most of the remaining double in device code. Neither needs the PadPlane treatment, because neither holds transferred state: these are compile-time constants and free functions, so there is no layout to preserve. PhysicsConstants.h: the particle masses only ever serve as compile-time initialisers, and the table built from them, PID::sMasses, is float already. They now use a MassType alias that is float on Metal and double everywhere else, so nothing is lost that was not already being narrowed. The declarations sit in a generated block, so make_pdg_header.py emits the same thing and a regeneration will not undo this. Checked that the double to float narrowing is exact for all 123 literal masses: none of them hits a double rounding case, so sMasses comes out bit identical either way. MathUtils/Utils.h: every helper here has a float version and a d suffixed double twin, and nothing under GPU/ calls a d variant. The double twins are now compiled out on Metal only. Guarded on __METAL__ rather than GPUCA_GPUCODE_DEVICE on purpose. The latter is also set for the CUDA, HIP and OpenCL device passes, which do have double, and using it would have silently dropped those helpers from CUDA and turned its masses into floats. Preprocessed declarations are unchanged for host, CUDA, HIP, OpenCL and cling. Together this takes both headers from 226 errors to 0 in a Metal translation unit, and the whole kernel translation unit from 820 to 590. --- .../CommonConstants/PhysicsConstants.h | 285 +++++++++--------- .../CommonConstants/make_pdg_header.py | 4 +- Common/MathUtils/include/MathUtils/Utils.h | 60 ++++ 3 files changed, 210 insertions(+), 139 deletions(-) diff --git a/Common/Constants/include/CommonConstants/PhysicsConstants.h b/Common/Constants/include/CommonConstants/PhysicsConstants.h index 051fb6d2a6e89..86e5162e2b654 100644 --- a/Common/Constants/include/CommonConstants/PhysicsConstants.h +++ b/Common/Constants/include/CommonConstants/PhysicsConstants.h @@ -18,8 +18,19 @@ #ifndef ALICEO2_PHYSICSCONSTANTS_H_ #define ALICEO2_PHYSICSCONSTANTS_H_ +#include "GPUCommonDef.h" + namespace o2::constants::physics { +#ifdef __METAL__ +// MSL has no double. The masses only ever serve as compile-time initialisers, +// and the tables built from them (PID::sMasses) are float already, so nothing +// is lost. Every other backend, device included, keeps double. +using MassType = float; +#else +using MassType = double; +#endif + // particles masses // BEGINNING OF THE GENERATED BLOCK. @@ -102,153 +113,153 @@ enum Pdg { }; /// \brief Declarations of masses for additional particles -constexpr double MassEta = 0.547862; -constexpr double MassOmega = 0.78266; -constexpr double MassEtaPrime = 0.95778; -constexpr double MassB0 = 5.27966; -constexpr double MassB0Bar = 5.27966; -constexpr double MassBPlus = 5.27934; -constexpr double MassBCPlus = 6.27447; -constexpr double MassBS = 5.36692; -constexpr double MassBSBar = 5.36692; -constexpr double MassD0 = 1.86484; -constexpr double MassD0Bar = 1.86484; -constexpr double MassD0StarPlus = 2.272; -constexpr double MassD0Star0 = 2.343; -constexpr double MassD1Plus = 2.372; -constexpr double MassD10 = 2.412; -constexpr double MassD2StarPlus = 2.4601; -constexpr double MassD2Star0 = 2.4611; -constexpr double MassDMinus = 1.86966; -constexpr double MassDPlus = 1.86966; -constexpr double MassDS = 1.96835; -constexpr double MassDSBar = 1.96835; -constexpr double MassDSStar = 2.1122; -constexpr double MassDS1 = 2.53511; -constexpr double MassDS1Star2700 = 2.714; -constexpr double MassDS1Star2860 = 2.859; -constexpr double MassDS2Star = 2.5691; -constexpr double MassDS3Star2860 = 2.86; -constexpr double MassDStar = 2.01026; -constexpr double MassDStar0 = 2.00685; -constexpr double MassChiC1 = 3.51067; -constexpr double MassJPsi = 3.0969; -constexpr double MassLambdaB0 = 5.6196; -constexpr double MassLambdaCPlus = 2.28646; -constexpr double MassLambdaCPlus2860 = 2.8561; -constexpr double MassLambdaCPlus2880 = 2.8816; -constexpr double MassLambdaCPlus2940 = 2.9396; -constexpr double MassOmegaC0 = 2.6952; -constexpr double MassK0Star892 = 0.89555; -constexpr double MassKPlusStar892 = 0.89167; -constexpr double MassPhi = 1.019461; -constexpr double MassSigmaC0 = 2.45375; -constexpr double MassSigmaCPlusPlus = 2.45397; -constexpr double MassSigmaCStar0 = 2.51848; -constexpr double MassSigmaCStarPlusPlus = 2.51841; -constexpr double MassX3872 = 3.87165; -constexpr double MassXi0 = 1.31486; -constexpr double MassXiB0 = 5.7919; -constexpr double MassXiCCPlusPlus = 3.62155; -constexpr double MassXiCPlus = 2.46771; -constexpr double MassXiC0 = 2.47044; -constexpr double MassXiC3055Plus = 3.0559; -constexpr double MassXiC3080Plus = 3.0772; -constexpr double MassXiC3055_0 = 3.059; -constexpr double MassXiC3080_0 = 3.0799; -constexpr double MassDeuteron = 1.87561294257; -constexpr double MassTriton = 2.80892113298; -constexpr double MassHelium3 = 2.80839160743; -constexpr double MassAlpha = 3.7273794066; -constexpr double MassLithium4 = 3.7513; -constexpr double MassHyperTriton = 2.991134; -constexpr double MassHyperHydrogen4 = 3.922434; -constexpr double MassHyperHelium4 = 3.921728; -constexpr double MassHyperHelium5 = 4.839961; -constexpr double MassHyperHelium4Sigma = 3.995; -constexpr double MassLambda1520_Py = 1.5195; -constexpr double MassK1_1270_0 = 1.253; -constexpr double MassK1_1270Plus = 1.272; -constexpr double MassCDeuteron = 3.226; +GPUglobalconstexpr() MassType MassEta = 0.547862; +GPUglobalconstexpr() MassType MassOmega = 0.78266; +GPUglobalconstexpr() MassType MassEtaPrime = 0.95778; +GPUglobalconstexpr() MassType MassB0 = 5.27966; +GPUglobalconstexpr() MassType MassB0Bar = 5.27966; +GPUglobalconstexpr() MassType MassBPlus = 5.27934; +GPUglobalconstexpr() MassType MassBCPlus = 6.27447; +GPUglobalconstexpr() MassType MassBS = 5.36692; +GPUglobalconstexpr() MassType MassBSBar = 5.36692; +GPUglobalconstexpr() MassType MassD0 = 1.86484; +GPUglobalconstexpr() MassType MassD0Bar = 1.86484; +GPUglobalconstexpr() MassType MassD0StarPlus = 2.272; +GPUglobalconstexpr() MassType MassD0Star0 = 2.343; +GPUglobalconstexpr() MassType MassD1Plus = 2.372; +GPUglobalconstexpr() MassType MassD10 = 2.412; +GPUglobalconstexpr() MassType MassD2StarPlus = 2.4601; +GPUglobalconstexpr() MassType MassD2Star0 = 2.4611; +GPUglobalconstexpr() MassType MassDMinus = 1.86966; +GPUglobalconstexpr() MassType MassDPlus = 1.86966; +GPUglobalconstexpr() MassType MassDS = 1.96835; +GPUglobalconstexpr() MassType MassDSBar = 1.96835; +GPUglobalconstexpr() MassType MassDSStar = 2.1122; +GPUglobalconstexpr() MassType MassDS1 = 2.53511; +GPUglobalconstexpr() MassType MassDS1Star2700 = 2.714; +GPUglobalconstexpr() MassType MassDS1Star2860 = 2.859; +GPUglobalconstexpr() MassType MassDS2Star = 2.5691; +GPUglobalconstexpr() MassType MassDS3Star2860 = 2.86; +GPUglobalconstexpr() MassType MassDStar = 2.01026; +GPUglobalconstexpr() MassType MassDStar0 = 2.00685; +GPUglobalconstexpr() MassType MassChiC1 = 3.51067; +GPUglobalconstexpr() MassType MassJPsi = 3.0969; +GPUglobalconstexpr() MassType MassLambdaB0 = 5.6196; +GPUglobalconstexpr() MassType MassLambdaCPlus = 2.28646; +GPUglobalconstexpr() MassType MassLambdaCPlus2860 = 2.8561; +GPUglobalconstexpr() MassType MassLambdaCPlus2880 = 2.8816; +GPUglobalconstexpr() MassType MassLambdaCPlus2940 = 2.9396; +GPUglobalconstexpr() MassType MassOmegaC0 = 2.6952; +GPUglobalconstexpr() MassType MassK0Star892 = 0.89555; +GPUglobalconstexpr() MassType MassKPlusStar892 = 0.89167; +GPUglobalconstexpr() MassType MassPhi = 1.019461; +GPUglobalconstexpr() MassType MassSigmaC0 = 2.45375; +GPUglobalconstexpr() MassType MassSigmaCPlusPlus = 2.45397; +GPUglobalconstexpr() MassType MassSigmaCStar0 = 2.51848; +GPUglobalconstexpr() MassType MassSigmaCStarPlusPlus = 2.51841; +GPUglobalconstexpr() MassType MassX3872 = 3.87165; +GPUglobalconstexpr() MassType MassXi0 = 1.31486; +GPUglobalconstexpr() MassType MassXiB0 = 5.7919; +GPUglobalconstexpr() MassType MassXiCCPlusPlus = 3.62155; +GPUglobalconstexpr() MassType MassXiCPlus = 2.46771; +GPUglobalconstexpr() MassType MassXiC0 = 2.47044; +GPUglobalconstexpr() MassType MassXiC3055Plus = 3.0559; +GPUglobalconstexpr() MassType MassXiC3080Plus = 3.0772; +GPUglobalconstexpr() MassType MassXiC3055_0 = 3.059; +GPUglobalconstexpr() MassType MassXiC3080_0 = 3.0799; +GPUglobalconstexpr() MassType MassDeuteron = 1.87561294257; +GPUglobalconstexpr() MassType MassTriton = 2.80892113298; +GPUglobalconstexpr() MassType MassHelium3 = 2.80839160743; +GPUglobalconstexpr() MassType MassAlpha = 3.7273794066; +GPUglobalconstexpr() MassType MassLithium4 = 3.7513; +GPUglobalconstexpr() MassType MassHyperTriton = 2.991134; +GPUglobalconstexpr() MassType MassHyperHydrogen4 = 3.922434; +GPUglobalconstexpr() MassType MassHyperHelium4 = 3.921728; +GPUglobalconstexpr() MassType MassHyperHelium5 = 4.839961; +GPUglobalconstexpr() MassType MassHyperHelium4Sigma = 3.995; +GPUglobalconstexpr() MassType MassLambda1520_Py = 1.5195; +GPUglobalconstexpr() MassType MassK1_1270_0 = 1.253; +GPUglobalconstexpr() MassType MassK1_1270Plus = 1.272; +GPUglobalconstexpr() MassType MassCDeuteron = 3.226; /// \brief Declarations of masses for particles in ROOT PDG_t -constexpr double MassDown = 0.00467; -constexpr double MassDownBar = 0.00467; -constexpr double MassUp = 0.00216; -constexpr double MassUpBar = 0.00216; -constexpr double MassStrange = 0.0934; -constexpr double MassStrangeBar = 0.0934; -constexpr double MassCharm = 1.27; -constexpr double MassCharmBar = 1.27; -constexpr double MassBottom = 4.18; -constexpr double MassBottomBar = 4.18; -constexpr double MassTop = 172.5; -constexpr double MassTopBar = 172.5; -constexpr double MassGluon = 0.0; -constexpr double MassElectron = 0.000510999; -constexpr double MassPositron = 0.000510999; -constexpr double MassNuE = 0.0; -constexpr double MassNuEBar = 0.0; -constexpr double MassMuonMinus = 0.1056584; -constexpr double MassMuonPlus = 0.1056584; -constexpr double MassNuMu = 0.0; -constexpr double MassNuMuBar = 0.0; -constexpr double MassTauMinus = 1.77686; -constexpr double MassTauPlus = 1.77686; -constexpr double MassNuTau = 0.0; -constexpr double MassNuTauBar = 0.0; -constexpr double MassGamma = 0.0; -constexpr double MassZ0 = 91.1876; -constexpr double MassWPlus = 80.377; -constexpr double MassWMinus = 80.377; -constexpr double MassPi0 = 0.1349768; -constexpr double MassK0Long = 0.497611; -constexpr double MassPiPlus = 0.1395704; -constexpr double MassPiMinus = 0.1395704; -constexpr double MassProton = 0.9382721; -constexpr double MassProtonBar = 0.9382721; -constexpr double MassNeutron = 0.9395654; -constexpr double MassNeutronBar = 0.9395654; -constexpr double MassK0Short = 0.497611; -constexpr double MassK0 = 0.497611; -constexpr double MassK0Bar = 0.497611; -constexpr double MassKPlus = 0.493677; -constexpr double MassKMinus = 0.493677; -constexpr double MassLambda0 = 1.115683; -constexpr double MassLambda0Bar = 1.115683; -constexpr double MassLambda1520 = 1.519; -constexpr double MassSigmaMinus = 1.197449; -constexpr double MassSigmaBarPlus = 1.197449; -constexpr double MassSigmaPlus = 1.18937; -constexpr double MassSigmaBarMinus = 1.18937; -constexpr double MassSigma0 = 1.192642; -constexpr double MassSigma0Bar = 1.192642; -constexpr double MassXiMinus = 1.32171; -constexpr double MassXiPlusBar = 1.32171; -constexpr double MassOmegaMinus = 1.67245; -constexpr double MassOmegaPlusBar = 1.67245; +GPUglobalconstexpr() MassType MassDown = 0.00467; +GPUglobalconstexpr() MassType MassDownBar = 0.00467; +GPUglobalconstexpr() MassType MassUp = 0.00216; +GPUglobalconstexpr() MassType MassUpBar = 0.00216; +GPUglobalconstexpr() MassType MassStrange = 0.0934; +GPUglobalconstexpr() MassType MassStrangeBar = 0.0934; +GPUglobalconstexpr() MassType MassCharm = 1.27; +GPUglobalconstexpr() MassType MassCharmBar = 1.27; +GPUglobalconstexpr() MassType MassBottom = 4.18; +GPUglobalconstexpr() MassType MassBottomBar = 4.18; +GPUglobalconstexpr() MassType MassTop = 172.5; +GPUglobalconstexpr() MassType MassTopBar = 172.5; +GPUglobalconstexpr() MassType MassGluon = 0.0; +GPUglobalconstexpr() MassType MassElectron = 0.000510999; +GPUglobalconstexpr() MassType MassPositron = 0.000510999; +GPUglobalconstexpr() MassType MassNuE = 0.0; +GPUglobalconstexpr() MassType MassNuEBar = 0.0; +GPUglobalconstexpr() MassType MassMuonMinus = 0.1056584; +GPUglobalconstexpr() MassType MassMuonPlus = 0.1056584; +GPUglobalconstexpr() MassType MassNuMu = 0.0; +GPUglobalconstexpr() MassType MassNuMuBar = 0.0; +GPUglobalconstexpr() MassType MassTauMinus = 1.77686; +GPUglobalconstexpr() MassType MassTauPlus = 1.77686; +GPUglobalconstexpr() MassType MassNuTau = 0.0; +GPUglobalconstexpr() MassType MassNuTauBar = 0.0; +GPUglobalconstexpr() MassType MassGamma = 0.0; +GPUglobalconstexpr() MassType MassZ0 = 91.1876; +GPUglobalconstexpr() MassType MassWPlus = 80.377; +GPUglobalconstexpr() MassType MassWMinus = 80.377; +GPUglobalconstexpr() MassType MassPi0 = 0.1349768; +GPUglobalconstexpr() MassType MassK0Long = 0.497611; +GPUglobalconstexpr() MassType MassPiPlus = 0.1395704; +GPUglobalconstexpr() MassType MassPiMinus = 0.1395704; +GPUglobalconstexpr() MassType MassProton = 0.9382721; +GPUglobalconstexpr() MassType MassProtonBar = 0.9382721; +GPUglobalconstexpr() MassType MassNeutron = 0.9395654; +GPUglobalconstexpr() MassType MassNeutronBar = 0.9395654; +GPUglobalconstexpr() MassType MassK0Short = 0.497611; +GPUglobalconstexpr() MassType MassK0 = 0.497611; +GPUglobalconstexpr() MassType MassK0Bar = 0.497611; +GPUglobalconstexpr() MassType MassKPlus = 0.493677; +GPUglobalconstexpr() MassType MassKMinus = 0.493677; +GPUglobalconstexpr() MassType MassLambda0 = 1.115683; +GPUglobalconstexpr() MassType MassLambda0Bar = 1.115683; +GPUglobalconstexpr() MassType MassLambda1520 = 1.519; +GPUglobalconstexpr() MassType MassSigmaMinus = 1.197449; +GPUglobalconstexpr() MassType MassSigmaBarPlus = 1.197449; +GPUglobalconstexpr() MassType MassSigmaPlus = 1.18937; +GPUglobalconstexpr() MassType MassSigmaBarMinus = 1.18937; +GPUglobalconstexpr() MassType MassSigma0 = 1.192642; +GPUglobalconstexpr() MassType MassSigma0Bar = 1.192642; +GPUglobalconstexpr() MassType MassXiMinus = 1.32171; +GPUglobalconstexpr() MassType MassXiPlusBar = 1.32171; +GPUglobalconstexpr() MassType MassOmegaMinus = 1.67245; +GPUglobalconstexpr() MassType MassOmegaPlusBar = 1.67245; // END OF THE GENERATED BLOCK // legacy names -constexpr double MassPhoton = MassGamma; -constexpr double MassMuon = MassMuonMinus; -constexpr double MassPionCharged = MassPiPlus; -constexpr double MassPionNeutral = MassPi0; -constexpr double MassKaonCharged = MassKPlus; -constexpr double MassKaonNeutral = MassK0; -constexpr double MassLambda = MassLambda0; -constexpr double MassHyperhydrog4 = MassHyperHydrogen4; -constexpr double MassHyperhelium4 = MassHyperHelium4; -constexpr double MassHyperhelium4sigma = MassHyperHelium4Sigma; +GPUglobalconstexpr() MassType MassPhoton = MassGamma; +GPUglobalconstexpr() MassType MassMuon = MassMuonMinus; +GPUglobalconstexpr() MassType MassPionCharged = MassPiPlus; +GPUglobalconstexpr() MassType MassPionNeutral = MassPi0; +GPUglobalconstexpr() MassType MassKaonCharged = MassKPlus; +GPUglobalconstexpr() MassType MassKaonNeutral = MassK0; +GPUglobalconstexpr() MassType MassLambda = MassLambda0; +GPUglobalconstexpr() MassType MassHyperhydrog4 = MassHyperHydrogen4; +GPUglobalconstexpr() MassType MassHyperhelium4 = MassHyperHelium4; +GPUglobalconstexpr() MassType MassHyperhelium4sigma = MassHyperHelium4Sigma; // Light speed -constexpr float LightSpeedCm2S = 299792458.e2; // C in cm/s -constexpr float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns -constexpr float LightSpeedCm2PS = LightSpeedCm2S * 1e-12; // C in cm/ps +GPUglobalconstexpr() float LightSpeedCm2S = 299792458.e2; // C in cm/s +GPUglobalconstexpr() float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns +GPUglobalconstexpr() float LightSpeedCm2PS = LightSpeedCm2S * 1e-12; // C in cm/ps // Light speed inverse -constexpr float invLightSpeedCm2PS = 1. / LightSpeedCm2PS; // 1/C in ps/cm +GPUglobalconstexpr() float invLightSpeedCm2PS = 1. / LightSpeedCm2PS; // 1/C in ps/cm } // namespace o2::constants::physics diff --git a/Common/Constants/include/CommonConstants/make_pdg_header.py b/Common/Constants/include/CommonConstants/make_pdg_header.py index b2dac688fd098..512f3f9223a52 100755 --- a/Common/Constants/include/CommonConstants/make_pdg_header.py +++ b/Common/Constants/include/CommonConstants/make_pdg_header.py @@ -169,9 +169,9 @@ def mass(code): return dbPdg.Mass(code, success) -def declare_mass(pdg, mass_type="double") -> str: +def declare_mass(pdg, mass_type="MassType") -> str: """Returns a C++ declaration of a particle mass constant.""" - return f"constexpr {mass_type} Mass{pdg.name[1:]} = {mass(pdg.value)};" + return f"GPUglobalconstexpr() {mass_type} Mass{pdg.name[1:]} = {mass(pdg.value)};" def main(): diff --git a/Common/MathUtils/include/MathUtils/Utils.h b/Common/MathUtils/include/MathUtils/Utils.h index 3c51245fc6c29..2dc9b965586d9 100644 --- a/Common/MathUtils/include/MathUtils/Utils.h +++ b/Common/MathUtils/include/MathUtils/Utils.h @@ -32,111 +32,133 @@ GPUdi() float to02Pi(float phi) return detail::to02Pi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double to02Pid(double phi) { return detail::to02Pi(phi); } +#endif GPUdi() void bringTo02Pi(float& phi) { detail::bringTo02Pi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void bringTo02Pid(double& phi) { detail::bringTo02Pi(phi); } +#endif inline float toPMPiGen(float phi) { return detail::toPMPiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double toPMPiGend(double phi) { return detail::toPMPiGen(phi); } +#endif inline void bringToPMPiGen(float& phi) { detail::bringToPMPiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline void bringToPMPiGend(double& phi) { detail::bringToPMPiGen(phi); } +#endif inline float to02PiGen(float phi) { return detail::to02PiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double to02PiGend(double phi) { return detail::to02PiGen(phi); } +#endif inline void bringTo02PiGen(float& phi) { detail::bringTo02PiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline void bringTo02PiGend(double& phi) { detail::bringTo02PiGen(phi); } +#endif inline float toPMPi(float phi) { return detail::toPMPi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double toPMPid(double phi) { return detail::toPMPi(phi); } +#endif inline void bringToPMPi(float& phi) { return detail::bringToPMPi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline void bringToPMPid(double& phi) { return detail::bringToPMPi(phi); } +#endif GPUdi() void sincos(float ang, float& s, float& c) { detail::sincos(ang, s, c); } #ifndef __OPENCL__ +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void sincosd(double ang, double& s, double& c) { detail::sincos(ang, s, c); } #endif +#endif GPUdi() void rotateZ(float xL, float yL, float& xG, float& yG, float snAlp, float csAlp) { return detail::rotateZ(xL, yL, xG, yG, snAlp, csAlp); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void rotateZd(double xL, double yL, double& xG, double& yG, double snAlp, double csAlp) { return detail::rotateZ(xL, yL, xG, yG, snAlp, csAlp); } +#endif GPUdi() void rotateZInv(float xG, float yG, float& xL, float& yL, float snAlp, float csAlp) { detail::rotateZInv(xG, yG, xL, yL, snAlp, csAlp); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void rotateZInvd(double xG, double yG, double& xL, double& yL, double snAlp, double csAlp) { detail::rotateZInv(xG, yG, xL, yL, snAlp, csAlp); } +#endif #ifndef GPUCA_GPUCODE_DEVICE inline std::tuple rotateZInv(float xG, float yG, float snAlp, float csAlp) @@ -185,40 +207,48 @@ inline int angle2Sector(float phi) return detail::angle2Sector(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline int angle2Sectord(double phi) { return detail::angle2Sector(phi); } +#endif inline float sector2Angle(int sect) { return detail::sector2Angle(sect); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double sector2Angled(int sect) { return detail::sector2Angle(sect); } +#endif inline float angle2Alpha(float phi) { return detail::angle2Alpha(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double angle2Alphad(double phi) { return detail::angle2Alpha(phi); } +#endif GPUhdi() constexpr float fastATan2(float y, float x) { return detail::fastATan2(y, x); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() constexpr double fastATan2d(double y, double x) { return detail::fastATan2(y, x); } +#endif template GPUhdi() T min(const T x, const T y) @@ -226,10 +256,12 @@ GPUhdi() T min(const T x, const T y) return detail::min(x, y); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double mind(const double x, const double y) { return detail::min(x, y); }; +#endif template GPUhdi() T max(const T x, const T y) @@ -237,130 +269,156 @@ GPUhdi() T max(const T x, const T y) return detail::max(x, y); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double maxd(const double x, const double y) { return detail::max(x, y); }; +#endif GPUhdi() float sqrt(float x) { return detail::sqrt(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double sqrtd(double x) { return detail::sqrt(x); }; +#endif GPUhdi() float abs(float x) { return detail::abs(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double absd(double x) { return detail::abs(x); }; +#endif GPUdi() float asin(float x) { return detail::asin(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double asind(double x) { return detail::asin(x); }; +#endif GPUdi() float atan(float x) { return detail::atan(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double atand(double x) { return detail::atan(x); }; +#endif GPUdi() float atan2(float y, float x) { return detail::atan2(y, x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double atan2d(double y, double x) { return detail::atan2(y, x); }; +#endif GPUdi() float sin(float x) { return detail::sin(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double sind(double x) { return detail::sin(x); }; +#endif GPUdi() float cos(float x) { return detail::cos(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double cosd(double x) { return detail::cos(x); }; +#endif GPUdi() float tan(float x) { return detail::tan(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double tand(double x) { return detail::tan(x); }; +#endif GPUdi() float twoPi() { return detail::twoPi(); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double twoPid() { return detail::twoPi(); }; +#endif GPUdi() float pi() { return detail::pi(); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double pid() { return detail::pi(); } +#endif GPUdi() int nint(float x) { return detail::nint(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() int nintd(double x) { return detail::nint(x); }; +#endif GPUdi() bool finite(float x) { return detail::finite(x); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() bool finited(double x) { return detail::finite(x); } +#endif GPUdi() unsigned int clz(unsigned int val) { @@ -377,10 +435,12 @@ GPUdi() float log(float x) return detail::log(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double logd(double x) { return detail::log(x); }; +#endif using detail::StatAccumulator; From 69e119f4e4f447aaf570c541378a6754ef808c53 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:46 +0200 Subject: [PATCH 06/41] ReconstructionDataFormats: keep the double SMatrix aliases off Metal TrackParametrizationWithError declares MatrixDSym5 and MatrixD5 as SMatrix over double. MSL has no double, so those two alias declarations fail, and because the failure is mid-declaration the parser then loses the rest of the alias and every later mention of the name. The class comes out malformed, which is why TrackParCov stopped being recognised as a base of TrackTPCITS and why every subsequent call to a TrackParametrization method through it reported 'cannot initialize object parameter ... with an expression of type const TrackParCov'. Those diagnostics read like an address-space problem and are not one: they are all fallout from these two lines. Guarding the aliases, and the three declarations that mention them, removes all of them. The three methods are declared here but defined out of line, so device code could not call them on Metal in any case. Preprocessed output is unchanged for host, CUDA, HIP and cling; the diff adds only #ifndef __METAL__ guards and removes nothing. Takes the Metal translation unit from 235 errors to 199, with the constant-versus-generic diagnostics down from 14 to 0. --- .../TrackParametrizationWithError.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h index 81280d090be71..2af42953e43fe 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h @@ -41,8 +41,10 @@ class TrackParametrizationWithError : public TrackParametrization #endif using covMat_t = std::array; +#ifndef __METAL__ using MatrixDSym5 = o2::math_utils::SMatrix>; using MatrixD5 = o2::math_utils::SMatrix>; +#endif GPUhd() TrackParametrizationWithError(); GPUd() TrackParametrizationWithError(value_t x, value_t alpha, const params_t& par, const covMat_t& cov, int charge = 1, const PID pid = PID::Pion); @@ -111,12 +113,18 @@ class TrackParametrizationWithError : public TrackParametrization template GPUd() value_t getPredictedChi2Quiet(const BaseCluster& p) const; +#ifndef __METAL__ GPUd() void buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const; +#endif +#ifndef __METAL__ GPUd() value_t getPredictedChi2(const TrackParametrizationWithError& rhs, MatrixDSym5& covToSet) const; +#endif GPUd() value_t getPredictedChi2(const TrackParametrizationWithError& rhs) const; GPUd() value_t getPredictedChi2Fast(const TrackParametrizationWithError& rhs) const; GPUd() value_t getPredictedChi2Quiet(const TrackParametrizationWithError& rhs) const; +#ifndef __METAL__ GPUd() bool update(const TrackParametrizationWithError& rhs, const MatrixDSym5& covInv); +#endif GPUd() bool update(const TrackParametrizationWithError& rhs); GPUd() bool update(const dim2_t& p, const dim3_t& cov); From 02d994437a70934069193beac0645603f5a37232 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 07/41] MathUtils: make SMatrixGPU compile as MSL Two things stop the GPU SMatrix port from being usable on Metal, neither of them to do with the matrix maths. MSL rejects variables declared static at function scope, which is how MatRepSymGPU::off() caches its offset table. Function-scope constexpr without static is accepted, so Metal uses that; every other backend keeps the static. The streaming operator is constrained with a C++20 requires-clause, already hidden from OpenCL because C++ for OpenCL 2021 is C++17. MSL 4.1 reports __cplusplus 201703L for the same reason, so it needs the same treatment. Left unhidden the declaration does not parse, and the failure cascades through the rest of the header. With these, SMatrixGPU> compiles as MSL and SMatrixGPU.h itself reports no diagnostics. Host, CUDA, HIP, OpenCL and cling are untouched. --- Common/MathUtils/include/MathUtils/SMatrixGPU.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Common/MathUtils/include/MathUtils/SMatrixGPU.h b/Common/MathUtils/include/MathUtils/SMatrixGPU.h index 8158a93666a92..21b3f3ca30406 100644 --- a/Common/MathUtils/include/MathUtils/SMatrixGPU.h +++ b/Common/MathUtils/include/MathUtils/SMatrixGPU.h @@ -340,7 +340,11 @@ class MatRepSymGPU static GPUdi() int off(int i) { +#ifdef __METAL__ // MSL rejects variables declared static at function scope + constexpr auto v = row_offsets_utils::make(off1); +#else static constexpr auto v = row_offsets_utils::make(off1); +#endif return v[i]; } @@ -518,7 +522,7 @@ class SMatrixGPU R mRep; }; -#ifndef __OPENCL__ // TODO: current C++ for OpenCL 2021 is at C++17, so no concepts. But we don't need this trick for OpenCL anyway, so we can just hide it. +#if !defined(__OPENCL__) && !defined(__METAL__) // TODO: current C++ for OpenCL 2021 and MSL 4.1 are both at C++17, so no concepts. But we don't need this trick there anyway, so we can just hide it. template requires(sizeof(typename X::traits_type::pos_type) != 0) // do not provide a template to fair::Logger, etc... (pos_type is a member type of all std::ostream classes) GPUd() X& operator<<(Y& y, const SMatrixGPU&) From 2dc2b5c205c6faadf4ebe7e852d31d7c0f9c9f29 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 08/41] GPU: keep the double math helpers off Metal SinCosd and the double specialisation of Abs are the double twins of the float versions, in the same style as the d suffixed helpers in MathUtils; MSL has no double, so they are compiled out there and kept everywhere else. Deterministic mode is refused outright rather than quietly degraded. Its SinCos path computes in double on purpose, for reproducibility against the other backends, and MSL cannot do that at all, so a Metal build that asked for it would silently produce different numbers. It is off by default and opt-in through O2_OVERRIDE_GPUCA_DETERMINISTIC_MODE. The diff adds guards and removes nothing; host, CUDA, HIP, OpenCL and cling are untouched, deterministic mode included. --- GPU/Common/GPUCommonDef.h | 6 ++++++ GPU/Common/GPUCommonMath.h | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/GPU/Common/GPUCommonDef.h b/GPU/Common/GPUCommonDef.h index 90746019d9a99..38e2ac35c5b8d 100644 --- a/GPU/Common/GPUCommonDef.h +++ b/GPU/Common/GPUCommonDef.h @@ -81,6 +81,12 @@ #define GPUCA_RTC_CONSTEXPR #endif +#if defined(GPUCA_DETERMINISTIC_MODE) && defined(__METAL__) + // The deterministic paths compute in double (see GPUCommonMath::SinCos) and + // MSL has no double, so the results could not match the other backends. + #error "GPUCA_DETERMINISTIC_MODE is not supported on Metal" +#endif + #ifndef GPUCA_DETERMINISTIC_CODE #ifdef GPUCA_DETERMINISTIC_MODE #define GPUCA_DETERMINISTIC_CODE(det, indet) det // In deterministic mode, take deterministic code path diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 7a78a5881dcfa..abad2a80abd17 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -87,7 +87,9 @@ class GPUCommonMath GPUd() constexpr static float Sin(float x); GPUd() constexpr static float Cos(float x); GPUhdni() static void SinCos(float x, float& s, float& c); +#ifndef __METAL__ // MSL has no double; every other backend keeps the twin GPUhdni() static void SinCosd(double x, double& s, double& c); +#endif GPUd() constexpr static float Tan(float x); GPUd() constexpr static float Pow(float x, float y); GPUd() constexpr static float Log(float x); @@ -308,6 +310,7 @@ GPUhdi() void GPUCommonMath::SinCos(float x, float& s, float& c) ) // clang-format on } +#ifndef __METAL__ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c) { #if !defined(GPUCA_GPUCODE_DEVICE) && defined(__APPLE__) @@ -318,6 +321,7 @@ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c) GPUCA_CHOICE((void)((s = sin(x)) + (c = cos(x))), sincos(x, &s, &c), s = sincos(x, &c)); #endif } +#endif GPUdi() constexpr uint32_t GPUCommonMath::Clz(uint32_t x) { @@ -444,11 +448,13 @@ GPUhdi() constexpr float GPUCommonMath::Abs(float x) return GPUCA_CHOICE(fabsf(x), fabsf(x), fabs(x)); } +#ifndef __METAL__ template <> GPUhdi() constexpr double GPUCommonMath::Abs(double x) { return GPUCA_CHOICE(fabs(x), fabs(x), fabs(x)); } +#endif template <> GPUhdi() constexpr int32_t GPUCommonMath::Abs(int32_t x) From 2586a921badedd6f26e4828970cc7a6645c4d74d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 09/41] GPU: adapt the remaining Common and Utils headers to Metal Three unrelated things, all in the GPU folder. MSL rejects the noexcept specifier. It appears in one header only, GPUCommonAlgorithm.h, so it now goes through GPUnoexcept(), which is noexcept on every other backend and empty on Metal. NDPiecewisePolynomials::getStepWidth and getVertexPosition return double and are not GPUd(), so device code cannot call them in any case; they join the host-only members the file already guards. Spline1DContainerBase::setXrange computes its range width in double for precision. That is device code, so on Metal it uses float instead. This is the one place here where Metal gets a different result rather than simply losing a declaration it could not use. GPUnoexcept() expands to noexcept for host, CUDA, HIP, OpenCL and cling. --- GPU/Common/GPUCommonAlgorithm.h | 24 ++++++++++++------------ GPU/Common/GPUCommonDefAPI.h | 5 +++++ GPU/Utils/NDPiecewisePolynomials.h | 4 ++++ GPU/Utils/Spline1DSpec.h | 4 ++++ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/GPU/Common/GPUCommonAlgorithm.h b/GPU/Common/GPUCommonAlgorithm.h index be88973561e0a..938c60b0b0cf2 100644 --- a/GPU/Common/GPUCommonAlgorithm.h +++ b/GPU/Common/GPUCommonAlgorithm.h @@ -51,32 +51,32 @@ class GPUCommonAlgorithm private: // Quicksort implementation template - GPUd() static void QuickSort(I f, I l) noexcept; + GPUd() static void QuickSort(I f, I l) GPUnoexcept(); // Quicksort implementation template - GPUd() static void QuickSort(I f, I l, Cmp cmp) noexcept; + GPUd() static void QuickSort(I f, I l, Cmp cmp) GPUnoexcept(); // Insertionsort implementation template - GPUd() static void InsertionSort(I f, I l, Cmp cmp) noexcept; + GPUd() static void InsertionSort(I f, I l, Cmp cmp) GPUnoexcept(); // Helper for Quicksort implementation template - GPUd() static I MedianOf3Select(I f, I l, Cmp cmp) noexcept; + GPUd() static I MedianOf3Select(I f, I l, Cmp cmp) GPUnoexcept(); // Helper for Quicksort implementation template - GPUd() static I UnguardedPartition(I f, I l, T piv, Cmp cmp) noexcept; + GPUd() static I UnguardedPartition(I f, I l, T piv, Cmp cmp) GPUnoexcept(); // Helper template - GPUd() static void IterSwap(I a, I b) noexcept; + GPUd() static void IterSwap(I a, I b) GPUnoexcept(); }; #ifndef GPUCA_ALGORITHM_STD template -GPUdi() void GPUCommonAlgorithm::IterSwap(I a, I b) noexcept +GPUdi() void GPUCommonAlgorithm::IterSwap(I a, I b) GPUnoexcept() { auto tmp = *a; *a = *b; @@ -84,7 +84,7 @@ GPUdi() void GPUCommonAlgorithm::IterSwap(I a, I b) noexcept } template -GPUdi() void GPUCommonAlgorithm::InsertionSort(I f, I l, Cmp cmp) noexcept +GPUdi() void GPUCommonAlgorithm::InsertionSort(I f, I l, Cmp cmp) GPUnoexcept() { auto it0{f}; while (it0 != l) { @@ -102,7 +102,7 @@ GPUdi() void GPUCommonAlgorithm::InsertionSort(I f, I l, Cmp cmp) noexcept } template -GPUdi() I GPUCommonAlgorithm::MedianOf3Select(I f, I l, Cmp cmp) noexcept +GPUdi() I GPUCommonAlgorithm::MedianOf3Select(I f, I l, Cmp cmp) GPUnoexcept() { auto m = f + (l - f) / 2; @@ -126,7 +126,7 @@ GPUdi() I GPUCommonAlgorithm::MedianOf3Select(I f, I l, Cmp cmp) noexcept } template -GPUdi() I GPUCommonAlgorithm::UnguardedPartition(I f, I l, T piv, Cmp cmp) noexcept +GPUdi() I GPUCommonAlgorithm::UnguardedPartition(I f, I l, T piv, Cmp cmp) GPUnoexcept() { do { while (cmp(*f, piv)) { @@ -146,7 +146,7 @@ GPUdi() I GPUCommonAlgorithm::UnguardedPartition(I f, I l, T piv, Cmp cmp) noexc } template -GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l, Cmp cmp) noexcept +GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l, Cmp cmp) GPUnoexcept() { if (f == l) { return; @@ -204,7 +204,7 @@ GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l, Cmp cmp) noexcept } template -GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l) noexcept +GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l) GPUnoexcept() { QuickSort(f, l, [](auto&& x, auto&& y) { return x < y; }); } diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index 7346f527e00b5..604b3dd855bd0 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -49,6 +49,7 @@ #define GPUconstant() // constant memory variable declaraion #define GPUconstexpr() static constexpr // constexpr on GPU that needs to be instantiated for dynamic access (e.g. arrays), becomes __constant on GPU #define GPUglobalconstexpr() constexpr // constexpr variable at program scope, needs the constant address space in MSL + #define GPUnoexcept() noexcept // noexcept where the backend supports it #define GPUprivate() // private memory variable declaration #define GPUgeneric() // reference / ptr to generic address space #define GPUbarrier() // synchronize all GPU threads in block @@ -162,6 +163,7 @@ #define GPUconstant() constant // TODO: possibly add const __restrict where possible later! #define GPUconstexpr() constant #define GPUglobalconstexpr() constant constexpr + #define GPUnoexcept() #define GPUprivate() thread #define GPUgeneric() #define GPUglobalref() device @@ -262,6 +264,9 @@ #ifndef GPUglobalconstexpr #define GPUglobalconstexpr() constexpr #endif +#ifndef GPUnoexcept +#define GPUnoexcept() noexcept +#endif #define GPUrestrict() __restrict__ diff --git a/GPU/Utils/NDPiecewisePolynomials.h b/GPU/Utils/NDPiecewisePolynomials.h index 8a629c64affbb..d72204a594873 100644 --- a/GPU/Utils/NDPiecewisePolynomials.h +++ b/GPU/Utils/NDPiecewisePolynomials.h @@ -296,12 +296,16 @@ class NDPiecewisePolynomials : public FlatObject /// \return returns step width of the inner grid /// \param dim dimension /// \param nAuxiliaryPoints number of Auxiliary points for given dimension +#ifndef __METAL__ // host-only accessors, and MSL has no double double getStepWidth(const uint32_t dim, const int32_t nAuxiliaryPoints) const { return 1 / (static_cast(mInvSpacing[dim]) * (nAuxiliaryPoints - 1)); } +#endif /// \return returns vertex position for given index and dimension /// \param ix index /// \param dim dimension +#ifndef __METAL__ double getVertexPosition(const uint32_t ix, const int32_t dim) const { return ix / static_cast(mInvSpacing[dim]) + mMin[dim]; } +#endif #if !defined(GPUCA_GPUCODE) /// \return returns the size of the parameters diff --git a/GPU/Utils/Spline1DSpec.h b/GPU/Utils/Spline1DSpec.h index b3f895cbde4ed..b6f1a1695f213 100644 --- a/GPU/Utils/Spline1DSpec.h +++ b/GPU/Utils/Spline1DSpec.h @@ -296,7 +296,11 @@ template GPUdi() void Spline1DContainerBase::setXrange(DataT xMin, DataT xMax) { mXmin = xMin; +#ifdef __METAL__ // MSL has no double + float l = ((float)xMax) - xMin; +#else double l = ((double)xMax) - xMin; +#endif if (l < 1.e-8) { l = 1.e-8; } From ed4dcd3ccd4afe7c8848d302f842622df4b82057 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 10/41] MathUtils: keep the double-only declarations off Metal Four shapes, all the same underlying point: MSL has no double, and diagnoses it at the declaration, so these break a device translation unit that merely includes them. The double specialisations of sincos, twoPi and pi are guarded; the primary templates still serve float. StatAccumulator has no GPUd() members and is not referenced under GPU/, so the whole struct is host-only. The CircleXYd_t, IntervalXYd_t, Bracketd_t and Rotation2Dd_t aliases are guarded. Bracket.h also used noexcept, which MSL rejects, so it goes through GPUnoexcept() like GPUCommonAlgorithm.h. Nothing is removed for host, CUDA, HIP, OpenCL or cling. --- Common/MathUtils/include/MathUtils/Cartesian.h | 2 ++ Common/MathUtils/include/MathUtils/Primitive2D.h | 6 ++++++ .../MathUtils/include/MathUtils/detail/Bracket.h | 14 ++++++++------ .../include/MathUtils/detail/StatAccumulator.h | 2 ++ .../include/MathUtils/detail/trigonometric.h | 6 ++++++ 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Common/MathUtils/include/MathUtils/Cartesian.h b/Common/MathUtils/include/MathUtils/Cartesian.h index e61b10a7caee9..99b946069f089 100644 --- a/Common/MathUtils/include/MathUtils/Cartesian.h +++ b/Common/MathUtils/include/MathUtils/Cartesian.h @@ -152,7 +152,9 @@ class Rotation2D }; using Rotation2Df_t = Rotation2D; +#ifndef __METAL__ using Rotation2Dd_t = Rotation2D; +#endif #if (!defined(GPUCA_STANDALONE) || !defined(DGPUCA_NO_ROOT)) && !defined(GPUCA_GPUCODE) && !defined(GPUCOMMONRTYPES_H_ACTIVE) diff --git a/Common/MathUtils/include/MathUtils/Primitive2D.h b/Common/MathUtils/include/MathUtils/Primitive2D.h index 052926d111594..e654b9cfb1f13 100644 --- a/Common/MathUtils/include/MathUtils/Primitive2D.h +++ b/Common/MathUtils/include/MathUtils/Primitive2D.h @@ -28,17 +28,23 @@ namespace math_utils template using CircleXY = detail::CircleXY; using CircleXYf_t = detail::CircleXY; +#ifndef __METAL__ using CircleXYd_t = detail::CircleXY; +#endif template using IntervalXY = detail::IntervalXY; using IntervalXYf_t = detail::IntervalXY; +#ifndef __METAL__ using IntervalXYd_t = detail::IntervalXY; +#endif template using Bracket = detail::Bracket; using Bracketf_t = detail::Bracket; +#ifndef __METAL__ using Bracketd_t = detail::Bracket; +#endif } // namespace math_utils } // namespace o2 diff --git a/Common/MathUtils/include/MathUtils/detail/Bracket.h b/Common/MathUtils/include/MathUtils/detail/Bracket.h index 2da6949c4a6f8..450174dc9737f 100644 --- a/Common/MathUtils/include/MathUtils/detail/Bracket.h +++ b/Common/MathUtils/include/MathUtils/detail/Bracket.h @@ -16,6 +16,8 @@ #ifndef ALICEO2_BRACKET_H #define ALICEO2_BRACKET_H +#include "GPUCommonDef.h" + #include #ifndef GPUCA_GPUCODE_DEVICE #include @@ -53,9 +55,9 @@ class Bracket bool operator==(const Bracket& other) const; bool operator!=(const Bracket& other) const; - void setMax(T v) noexcept; - void setMin(T v) noexcept; - void set(T minv, T maxv) noexcept; + void setMax(T v) GPUnoexcept(); + void setMin(T v) GPUnoexcept(); + void set(T minv, T maxv) GPUnoexcept(); T& getMax(); T& getMin(); @@ -129,19 +131,19 @@ inline bool Bracket::operator!=(const Bracket& rhs) const } template -inline void Bracket::setMax(T v) noexcept +inline void Bracket::setMax(T v) GPUnoexcept() { mMax = v; } template -inline void Bracket::setMin(T v) noexcept +inline void Bracket::setMin(T v) GPUnoexcept() { mMin = v; } template -inline void Bracket::set(T minv, T maxv) noexcept +inline void Bracket::set(T minv, T maxv) GPUnoexcept() { this->setMin(minv); this->setMax(maxv); diff --git a/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h b/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h index abb8a716cc5ee..2d0eb94fd951a 100644 --- a/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h +++ b/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h @@ -27,6 +27,7 @@ namespace math_utils namespace detail { +#ifndef __METAL__ // host-only accumulator, and MSL has no double struct StatAccumulator { // mean / RMS accumulator double sum = 0.; @@ -83,6 +84,7 @@ struct StatAccumulator { n = 0; } }; +#endif } // namespace detail } // namespace math_utils diff --git a/Common/MathUtils/include/MathUtils/detail/trigonometric.h b/Common/MathUtils/include/MathUtils/detail/trigonometric.h index e13d965663dc9..6723e5908ea0c 100644 --- a/Common/MathUtils/include/MathUtils/detail/trigonometric.h +++ b/Common/MathUtils/include/MathUtils/detail/trigonometric.h @@ -122,12 +122,14 @@ GPUhdi() void sincos(T ang, T& s, T& c) { return o2::gpu::GPUCommonMath::SinCos(ang, s, c); } +#ifndef __METAL__ // MSL has no double; the primary template still serves float template <> GPUhdi() void sincos(double ang, double& s, double& c) { return o2::gpu::GPUCommonMath::SinCosd(ang, s, c); } #endif +#endif #ifndef GPUCA_GPUCODE_DEVICE @@ -358,11 +360,13 @@ GPUdi() T twoPi() return o2::gpu::GPUCommonMath::TwoPi(); }; +#ifndef __METAL__ // MSL has no double; the primary template still serves float template <> GPUdi() double twoPi() { return o2::constants::math::TwoPI; }; +#endif template GPUdi() T pi() @@ -370,11 +374,13 @@ GPUdi() T pi() return o2::gpu::GPUCommonMath::Pi(); } +#ifndef __METAL__ // MSL has no double; the primary template still serves float template <> GPUdi() double pi() { return o2::constants::math::PI; } +#endif #ifndef GPUCA_GPUCODE_DEVICE template <> From 8a9de60b858d2bb3bef0c4b171791922a5552f28 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 11/41] Reuse GPUdoubleValue for the last double declarations reaching device code The LHC and TPC geometry constants are namespace-scope constexpr double, so on Metal they hit both restrictions at once: no double, and program scope needs an address space. GPUglobalconstexpr() with GPUdoubleValue covers both, and GPUdoubleValue is already double everywhere except Metal, so nothing else moves. LHCBunchSpacingMUS is genuinely read from GPUd() code, which consumes it as float in any case. Propagator's double getFieldXYZ and getBz overloads, PropagatorD, TrackParD and TrackParCovD are all double twins whose float versions remain, so they are compiled out on Metal. TrackUtils computes one local in double inside device code; on Metal that is a float, which together with Spline1DContainerBase::setXrange makes two places where Metal gets a different number rather than simply losing a declaration. With this the Metal translation unit has no double and no noexcept diagnostics left, down from 57. Host, CUDA, HIP, OpenCL and cling keep double throughout. --- .../include/CommonConstants/LHCConstants.h | 14 ++++++++------ .../Detectors/TPC/include/DataFormatsTPC/Defs.h | 8 +++++--- .../include/ReconstructionDataFormats/Track.h | 4 ++++ .../include/ReconstructionDataFormats/TrackUtils.h | 4 ++++ Detectors/Base/include/DetectorsBase/Propagator.h | 6 ++++++ 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Common/Constants/include/CommonConstants/LHCConstants.h b/Common/Constants/include/CommonConstants/LHCConstants.h index 1582f2166ca0f..2cdd0b5118731 100644 --- a/Common/Constants/include/CommonConstants/LHCConstants.h +++ b/Common/Constants/include/CommonConstants/LHCConstants.h @@ -16,6 +16,8 @@ #ifndef ALICEO2_LHCCONSTANTS_H_ #define ALICEO2_LHCCONSTANTS_H_ +#include "GPUCommonDouble.h" + #include "GPUCommonDef.h" namespace o2 @@ -31,12 +33,12 @@ enum BeamDirection : int { BeamA, // beamA = beam 0, InteractingBC = -1 // as used in the BunchFilling class }; GPUglobalconstexpr() int LHCMaxBunches = 3564; // max N bunches -constexpr double LHCRFFreq = 400.789e6; // LHC RF frequency in Hz -constexpr double LHCBunchSpacingNS = 10 * 1.e9 / LHCRFFreq; // bunch spacing in ns (10 RFbuckets) -constexpr double LHCOrbitNS = LHCMaxBunches * LHCBunchSpacingNS; // orbit duration in ns -constexpr double LHCRevFreq = 1.e9 / LHCOrbitNS; // revolution frequency -constexpr double LHCBunchSpacingMUS = LHCBunchSpacingNS * 1e-3; // bunch spacing in \mus (10 RFbuckets) -constexpr double LHCOrbitMUS = LHCOrbitNS * 1e-3; // orbit duration in \mus +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCRFFreq = 400.789e6; // LHC RF frequency in Hz +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCBunchSpacingNS = 10 * 1.e9 / LHCRFFreq; // bunch spacing in ns (10 RFbuckets) +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCOrbitNS = LHCMaxBunches * LHCBunchSpacingNS; // orbit duration in ns +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCRevFreq = 1.e9 / LHCOrbitNS; // revolution frequency +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCBunchSpacingMUS = LHCBunchSpacingNS * 1e-3; // bunch spacing in \mus (10 RFbuckets) +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCOrbitMUS = LHCOrbitNS * 1e-3; // orbit duration in \mus GPUglobalconstexpr() unsigned int MaxNOrbits = 0xffffffff; // Offsets of A, C beam bunches at P2 diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h index a5be0da32f641..15f72d6e2f68e 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h @@ -19,6 +19,8 @@ #ifndef AliceO2_TPC_Defs_H #define AliceO2_TPC_Defs_H +#include "GPUCommonDouble.h" + #include "GPUCommonDef.h" #ifndef GPUCA_GPUCODE_DEVICE @@ -42,9 +44,9 @@ enum Side { A = 0, GPUglobalconstexpr() unsigned char SECTORSPERSIDE = 18; GPUglobalconstexpr() unsigned char SIDES = 2; -constexpr double PI = 3.14159265358979323846; -constexpr double TWOPI = 2. * PI; -constexpr double SECPHIWIDTH = TWOPI / 18.; +GPUglobalconstexpr() o2::gpu::GPUdoubleValue PI = 3.14159265358979323846; +GPUglobalconstexpr() o2::gpu::GPUdoubleValue TWOPI = 2. * PI; +GPUglobalconstexpr() o2::gpu::GPUdoubleValue SECPHIWIDTH = TWOPI / 18.; /// TPC ROC types enum RocType { IROC = 0, diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h index c4b17158f7dc5..2d4d67574a12a 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h @@ -25,11 +25,15 @@ namespace track { using TrackParF = TrackParametrization; +#ifndef __METAL__ using TrackParD = TrackParametrization; +#endif using TrackPar = TrackParF; using TrackParCovF = TrackParametrizationWithError; +#ifndef __METAL__ using TrackParCovD = TrackParametrizationWithError; +#endif using TrackParCov = TrackParCovF; } // namespace track diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h index 6befcd0dfc898..1e96ab224a308 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h @@ -138,7 +138,11 @@ GPUd() value_T BetheBlochSolid(value_T bg, value_T rho, value_T kp1, value_T kp2 if (x > kp2) { d2 = lhwI + x - value_T(0.5); } else if (x > kp1) { +#ifdef __METAL__ // MSL has no double + float r = (kp2 - x) / (kp2 - kp1); +#else double r = (kp2 - x) / (kp2 - kp1); +#endif d2 = lhwI + x - value_T(0.5) + (value_T(0.5) - lhwI - kp1) * r * r * r; } auto dedx = mK * meanZA / beta2 * (value_T(0.5) * gpu::CAMath::Log(value_T(2) * me * bg2 * maxT / (meanI * meanI)) - beta2 - d2); diff --git a/Detectors/Base/include/DetectorsBase/Propagator.h b/Detectors/Base/include/DetectorsBase/Propagator.h index 38f4e089a1a1a..789d167ea1c70 100644 --- a/Detectors/Base/include/DetectorsBase/Propagator.h +++ b/Detectors/Base/include/DetectorsBase/Propagator.h @@ -192,11 +192,15 @@ class PropagatorImpl GPUd() void getFieldXYZ(const math_utils::Point3D xyz, float* bxyz) const; +#ifndef __METAL__ // MSL has no double; the float twin remains GPUd() void getFieldXYZ(const math_utils::Point3D xyz, double* bxyz) const; +#endif GPUd() float getBz(const math_utils::Point3D xyz) const; +#ifndef __METAL__ // MSL has no double; the float twin remains GPUd() double getBz(const math_utils::Point3D xyz) const; +#endif private: #ifndef GPUCA_GPUCODE @@ -221,7 +225,9 @@ class PropagatorImpl }; using PropagatorF = PropagatorImpl; +#ifndef __METAL__ // MSL has no double; the float twin remains using PropagatorD = PropagatorImpl; +#endif using Propagator = PropagatorF; } // namespace base From ff132af6b5b3aadaa0b816428f09ae3b04a7e089 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:59 +0200 Subject: [PATCH 12/41] GPU: give Metal a column in the tuning parameter table Without one, GPUDefParametersDefaultsDevice.h falls through to '#error GPU TYPE NOT SET' for GPUCA_GPUTYPE_METAL, which the .metal source defines, so nothing under Definitions/ could be compiled for the backend at all. The column is seeded from OPENCL, which is the other portable backend with no vendor-specific tuning; Metal ends up with the same WARP_SIZE 32 and THREAD_COUNT_DEFAULT 256. Real numbers want measuring on a device once the kernels run. detect_gpu_arch reports METAL alongside the others so the generator emits the block. Every existing architecture comes out byte-identical; the generated device header gains only the Metal block and the architecture comment. --- .../Definitions/Parameters/GPUParameters.csv | 234 +++++++++--------- dependencies/FindO2GPU.cmake | 3 + 2 files changed, 120 insertions(+), 117 deletions(-) diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 12c87afceb86f..f846641cefc9e 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -1,117 +1,117 @@ -Architecture,default,default_cpu,MI100,VEGA,TAHITI,TESLA,FERMI,PASCAL,KEPLER,AMPERE,TURING,HOPPER,ADA,OPENCL,RDNA,MI210,BLACKWELL,MI300 -,,,,,,,,,,,,,,,,,, -CORE:,,,,,,,,,,,,,,,,,, -WARP_SIZE,0,,64,64,32,32,32,32,32,32,32,32,32,32,32,64,32,64 -THREAD_COUNT_DEFAULT,256,,256,256,,,,,,512,512,,512,256,512,512,512, -,,,,,,,,,,,,,,,,,, -LB:,,,,,,,,,,,,,,,,,, -GPUTPCCreateTrackingData,256,,"[256, 7]","[192, 2]",,,,,,"[224, 7]",256,"[128, 14]",416,,"[64, 21]",,384,"[320, 2]" -GPUTPCTrackletConstructor,256,,"[768, 8]","[512, 10]","[256, 2]","[256, 1]","[256, 2]","[1024, 2]","[512, 4]",1024,"[256, 2]",1024,"[1024, 1]",,"[768, 2]",,768,512 -GPUTPCTrackletSelector,256,,"[384, 5]","[192, 10]","[256, 3]","[256, 1]","[256, 3]","[512, 4]","[256, 3]","[288, 3]","[192, 3]","[544, 1]","[32, 2]",,"[384, 3]",,992,"[256, 6]" -GPUTPCNeighboursFinder,256,,"[192, 8]","[960, 8]",256,256,256,512,256,864,"[640, 1]","[512, 2]","[736, 1]",,"[480, 3]",,992,"[704, 1]" -GPUTPCNeighboursCleaner,256,,"[128, 5]","[384, 9]",256,256,256,256,256,544,512,"[192, 9]","[512, 1]",,"[384, 5]",,672,"[640, 1]" -GPUTPCExtrapolationTracking,256,,"[256, 7]","[256, 2]",,,,,,"[352, 4]","[192, 2]","[896, 1]","[352, 1]",,"[1024, 1]",,896,1024 -GPUTRDTrackerKernels_gpuVersion,512,,,,,,,,,512,,512,512,,512,,,512 -GPUTPCCreateOccupancyMap_fill,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCCreateOccupancyMap_fold,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTRDTrackerKernels_o2Version,512,,,,,,,,,512,,512,512,,512,,,512 -GPUTPCCompressionKernels_step0attached,256,,"[128, 1]","[64, 2]",,,,,,"[160, 2]",128,"[448, 1]",352,,"[1024, 1]",,"[96, 3]","[128, 4]" -GPUTPCCompressionKernels_step1unattached,256,,"[512, 2]","[512, 2]",,,,,,"[288, 4]","[512, 2]","[256, 4]","[512, 2]",,"[512, 3]",,"[512, 2]","[512, 3]" -GPUTPCDecompressionKernels_step0attached,256,,"[128, 2]","[128, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,"[128, 1]",,"[32, 1]","[128, 1]" -GPUTPCDecompressionKernels_step1unattached,256,,"[64, 2]","[64, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,"[64, 1]",,"[32, 1]","[64, 1]" -GPUTPCDecompressionUtilKernels_sortPerSectorRow,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCDecompressionUtilKernels_countFilteredClusters,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCDecompressionUtilKernels_storeFilteredClusters,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCCFDecodeZS,"[128, 4]",,"[64, 4]","[64, 1]",,,,,,"[32, 10]","[64, 8]","[32, 10]","[32, 10]",,"[64, 1]",,"[64, 10]","[64, 1]" -GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,,,,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,64,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""" -GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,"[64, 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]" -GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,"[928, 1]",,"[1024, 1]","[320, 2]" -COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,, -GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]" -GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,"[128, 16]",,"[224, 3]","[256, 7]" -GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,"[32, 20]",,"[32, 10]","[64, 4]" -GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerUnpackGlobal,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step0,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step1,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step2,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step3,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step4,256,,512,256,,,,,,"[256, 4]","[256, 4]","[256, 4]","[256, 4]",,256,,"[256, 4]",256 -GPUTPCGMMergerClearLinks,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerMergeWithinPrepare,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerMergeSectorsPrepare,256,,256,256,,,,,,"[256, 2]","[256, 2]","[256, 2]","[256, 2]",,256,,"[256, 2]",256 -GPUTPCGMMergerMergeBorders_step0,256,,512,256,,,,,,192,192,192,192,,256,,192,256 -GPUTPCGMMergerMergeBorders_step2,256,,512,256,,,,,,"[64, 2]",256,"[64, 2]","[64, 2]",,256,,"[64, 2]",256 -GPUTPCGMMergerMergeCE,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerLinkExtrapolatedTracks,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerCollect,256,,"[768, 1]","[1024, 1]",,,,,,"[864, 1]","[128, 2]","[896, 1]",128,,1024,,"[288, 1]","[384, 4]" -GPUTPCGMMergerSortTracksPrepare,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerPrepareForFit_step0,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerPrepareForFit_step1,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMO2Output_prepare,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMO2Output_output,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCStartHitsFinder,256,,"[1024, 2]","[1024, 7]",256,256,256,256,256,"[224, 1]",512,"[416, 4]",928,,"[320, 5]",,608,"[448, 3]" -GPUTPCStartHitsSorter,256,,"[1024, 5]","[512, 7]",256,256,256,256,256,"[320, 2]","[512, 1]","[864, 1]","[96, 2]",,"[192, 5]",,608,"[448, 1]" -GPUTPCCFCheckPadBaseline,576,,"[576, 2]","[576, 2]",,,,,,"[576, 3]",,"[576, 1]","[576, 1]",,"[576, 2]",,"[576, 2]",576 -GPUTPCCFHIPTailConnector,256,,256,256,,,,,,"[224, 2]",,"[320, 5]","[704, 1]",,"[128, 7]",,,"[448, 4]" -GPUTPCCFHIPClusterizer,256,,256,256,,,,,,"[288, 5]",,"[480, 3]","[448, 3]",,352,,,"[512, 3]" -GPUTPCCFChargeMapFiller_fillIndexMap,512,,512,512,,,,,,448,,448,448,,512,,448,512 -GPUTPCCFChargeMapFiller_fillFromDigits,512,,512,512,,,,,,448,,448,448,,512,,448,512 -GPUTPCCFChargeMapFiller_findFragmentStart,512,,512,512,,,,,,448,,448,448,,512,,448,512 -GPUTPCCFPeakFinder,512,,"[512, 9]","[512, 4]",,,,,,416,,992,"[672, 1]",,"[384, 2]",,"[128, 5]","[192, 10]" -GPUTPCCFNoiseSuppression,512,,512,512,,,,,,608,,896,480,,160,,,448 -GPUTPCCFDeconvolution,512,,"[512, 5]","[512, 5]",,,,,,"[480, 4]",,224,512,,480,,384,"[448, 3]" -GPUTPCCFClusterizer,512,,"[448, 3]","[512, 2]",,,,,,"[608, 3]",,736,"[192, 3]",,576,,"[160, 5]","[832, 2]" -GPUTPCNNClusterizerKernels,512,,,,,,,,,,,,,,,,, -GPUTrackingRefitKernel_mode0asGPU,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTrackingRefitKernel_mode1asTrackParCov,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUMemClean16,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,"[256, 1]",,,"[256, 1]" -GPUitoa,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,"[256, 1]",,,"[256, 1]" -GPUTPCCFNoiseSuppression_noiseSuppression,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,448, -GPUTPCCFNoiseSuppression_updatePeaks,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,448, -GPUTPCNNClusterizerKernels_runCfClusterizer,"""GPUCA_LB_GPUTPCCFClusterizer""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_fillInputNNCPU,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_fillInputNNGPU,1024,,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_determineClass1Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_determineClass2Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_publishClass1Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_publishClass2Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_publishDeconvolutionFlags,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanStart,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanUp,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanTop,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanDown,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_compactDigits,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_unbuffered,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_buffered32,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_buffered64,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_buffered128,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_multiBlock,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCGMMergerFinalize_0,256,,256,,,,,,,256,256,,256,,,,256, -GPUTPCGMMergerFinalize_1,256,,256,,,,,,,256,256,,256,,,,256, -GPUTPCGMMergerFinalize_2,256,,256,,,,,,,256,256,,256,,,,256, -GPUTPCConvertKernel,,,,,,,,,,,,,,,256,,,256 -,,,,,,,,,,,,,,,,,, -PAR:,,,,,,,,,,,,,,,,,, -AMD_EUS_PER_CU,0,0,4,4,,,,,,,,,,,4,,0,4 -SORT_STARTHITS,1,0,,,,,,,,1,,1,1,,1,,1,1 -NEIGHBOURS_FINDER_MAX_NNEIGHUP,6,0,10,4,,,,,,4,4,4,4,,5,,2,5 -NEIGHBOURS_FINDER_UNROLL_GLOBAL,4,0,4,2,,,,,,2,,8,8,,4,,2,2 -NEIGHBOURS_FINDER_UNROLL_SHARED,1,0,0,0,,,,,,1,,1,0,,1,,1,1 -TRACKLET_SELECTOR_HITS_REG_SIZE,12,0,9,27,,,,,,20,20,20,20,,20,,2,20 -ALTERNATE_BORDER_SORT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1 -SORT_BEFORE_FIT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1 -NO_ATOMIC_PRECHECK,0,0,1,1,,,,,,1,1,1,1,,1,,1,1 -DEDX_STORAGE_TYPE,"""half""","""float""",,,,,,,,,,,,,,,, -MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,, -COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,4,,4,4 -COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,3,,3,3 -CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,1024,,,448 -MERGER_SPLIT_LOOP_INTERPOLATION,,,,,,,,,,,,,,,1,,,1 +Architecture,default,default_cpu,MI100,VEGA,TAHITI,TESLA,FERMI,PASCAL,KEPLER,AMPERE,TURING,HOPPER,ADA,OPENCL,METAL,RDNA,MI210,BLACKWELL,MI300 +,,,,,,,,,,,,,,,,,,, +CORE:,,,,,,,,,,,,,,,,,,, +WARP_SIZE,0,,64,64,32,32,32,32,32,32,32,32,32,32,32,32,64,32,64 +THREAD_COUNT_DEFAULT,256,,256,256,,,,,,512,512,,512,256,256,512,512,512, +,,,,,,,,,,,,,,,,,,, +LB:,,,,,,,,,,,,,,,,,,, +GPUTPCCreateTrackingData,256,,"[256, 7]","[192, 2]",,,,,,"[224, 7]",256,"[128, 14]",416,,,"[64, 21]",,384,"[320, 2]" +GPUTPCTrackletConstructor,256,,"[768, 8]","[512, 10]","[256, 2]","[256, 1]","[256, 2]","[1024, 2]","[512, 4]",1024,"[256, 2]",1024,"[1024, 1]",,,"[768, 2]",,768,512 +GPUTPCTrackletSelector,256,,"[384, 5]","[192, 10]","[256, 3]","[256, 1]","[256, 3]","[512, 4]","[256, 3]","[288, 3]","[192, 3]","[544, 1]","[32, 2]",,,"[384, 3]",,992,"[256, 6]" +GPUTPCNeighboursFinder,256,,"[192, 8]","[960, 8]",256,256,256,512,256,864,"[640, 1]","[512, 2]","[736, 1]",,,"[480, 3]",,992,"[704, 1]" +GPUTPCNeighboursCleaner,256,,"[128, 5]","[384, 9]",256,256,256,256,256,544,512,"[192, 9]","[512, 1]",,,"[384, 5]",,672,"[640, 1]" +GPUTPCExtrapolationTracking,256,,"[256, 7]","[256, 2]",,,,,,"[352, 4]","[192, 2]","[896, 1]","[352, 1]",,,"[1024, 1]",,896,1024 +GPUTRDTrackerKernels_gpuVersion,512,,,,,,,,,512,,512,512,,,512,,,512 +GPUTPCCreateOccupancyMap_fill,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCCreateOccupancyMap_fold,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTRDTrackerKernels_o2Version,512,,,,,,,,,512,,512,512,,,512,,,512 +GPUTPCCompressionKernels_step0attached,256,,"[128, 1]","[64, 2]",,,,,,"[160, 2]",128,"[448, 1]",352,,,"[1024, 1]",,"[96, 3]","[128, 4]" +GPUTPCCompressionKernels_step1unattached,256,,"[512, 2]","[512, 2]",,,,,,"[288, 4]","[512, 2]","[256, 4]","[512, 2]",,,"[512, 3]",,"[512, 2]","[512, 3]" +GPUTPCDecompressionKernels_step0attached,256,,"[128, 2]","[128, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,,"[128, 1]",,"[32, 1]","[128, 1]" +GPUTPCDecompressionKernels_step1unattached,256,,"[64, 2]","[64, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,,"[64, 1]",,"[32, 1]","[64, 1]" +GPUTPCDecompressionUtilKernels_sortPerSectorRow,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCDecompressionUtilKernels_countFilteredClusters,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCDecompressionUtilKernels_storeFilteredClusters,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCCFDecodeZS,"[128, 4]",,"[64, 4]","[64, 1]",,,,,,"[32, 10]","[64, 8]","[32, 10]","[32, 10]",,,"[64, 1]",,"[64, 10]","[64, 1]" +GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,,,,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,,64,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""" +GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,,"[64, 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]" +GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,,"[928, 1]",,"[1024, 1]","[320, 2]" +COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,,, +GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,,"[32, 24]",,"[64, 8]","[64, 6]" +GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,,"[128, 16]",,"[224, 3]","[256, 7]" +GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,,"[32, 20]",,"[32, 10]","[64, 4]" +GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerUnpackGlobal,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step0,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step1,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step2,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step3,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step4,256,,512,256,,,,,,"[256, 4]","[256, 4]","[256, 4]","[256, 4]",,,256,,"[256, 4]",256 +GPUTPCGMMergerClearLinks,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerMergeWithinPrepare,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerMergeSectorsPrepare,256,,256,256,,,,,,"[256, 2]","[256, 2]","[256, 2]","[256, 2]",,,256,,"[256, 2]",256 +GPUTPCGMMergerMergeBorders_step0,256,,512,256,,,,,,192,192,192,192,,,256,,192,256 +GPUTPCGMMergerMergeBorders_step2,256,,512,256,,,,,,"[64, 2]",256,"[64, 2]","[64, 2]",,,256,,"[64, 2]",256 +GPUTPCGMMergerMergeCE,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerLinkExtrapolatedTracks,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerCollect,256,,"[768, 1]","[1024, 1]",,,,,,"[864, 1]","[128, 2]","[896, 1]",128,,,1024,,"[288, 1]","[384, 4]" +GPUTPCGMMergerSortTracksPrepare,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerPrepareForFit_step0,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerPrepareForFit_step1,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMO2Output_prepare,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMO2Output_output,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCStartHitsFinder,256,,"[1024, 2]","[1024, 7]",256,256,256,256,256,"[224, 1]",512,"[416, 4]",928,,,"[320, 5]",,608,"[448, 3]" +GPUTPCStartHitsSorter,256,,"[1024, 5]","[512, 7]",256,256,256,256,256,"[320, 2]","[512, 1]","[864, 1]","[96, 2]",,,"[192, 5]",,608,"[448, 1]" +GPUTPCCFCheckPadBaseline,576,,"[576, 2]","[576, 2]",,,,,,"[576, 3]",,"[576, 1]","[576, 1]",,,"[576, 2]",,"[576, 2]",576 +GPUTPCCFHIPTailConnector,256,,256,256,,,,,,"[224, 2]",,"[320, 5]","[704, 1]",,,"[128, 7]",,,"[448, 4]" +GPUTPCCFHIPClusterizer,256,,256,256,,,,,,"[288, 5]",,"[480, 3]","[448, 3]",,,352,,,"[512, 3]" +GPUTPCCFChargeMapFiller_fillIndexMap,512,,512,512,,,,,,448,,448,448,,,512,,448,512 +GPUTPCCFChargeMapFiller_fillFromDigits,512,,512,512,,,,,,448,,448,448,,,512,,448,512 +GPUTPCCFChargeMapFiller_findFragmentStart,512,,512,512,,,,,,448,,448,448,,,512,,448,512 +GPUTPCCFPeakFinder,512,,"[512, 9]","[512, 4]",,,,,,416,,992,"[672, 1]",,,"[384, 2]",,"[128, 5]","[192, 10]" +GPUTPCCFNoiseSuppression,512,,512,512,,,,,,608,,896,480,,,160,,,448 +GPUTPCCFDeconvolution,512,,"[512, 5]","[512, 5]",,,,,,"[480, 4]",,224,512,,,480,,384,"[448, 3]" +GPUTPCCFClusterizer,512,,"[448, 3]","[512, 2]",,,,,,"[608, 3]",,736,"[192, 3]",,,576,,"[160, 5]","[832, 2]" +GPUTPCNNClusterizerKernels,512,,,,,,,,,,,,,,,,,, +GPUTrackingRefitKernel_mode0asGPU,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTrackingRefitKernel_mode1asTrackParCov,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUMemClean16,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,,"[256, 1]",,,"[256, 1]" +GPUitoa,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,,"[256, 1]",,,"[256, 1]" +GPUTPCCFNoiseSuppression_noiseSuppression,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,,448, +GPUTPCCFNoiseSuppression_updatePeaks,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,,448, +GPUTPCNNClusterizerKernels_runCfClusterizer,"""GPUCA_LB_GPUTPCCFClusterizer""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_fillInputNNCPU,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_fillInputNNGPU,1024,,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_determineClass1Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_determineClass2Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_publishClass1Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_publishClass2Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_publishDeconvolutionFlags,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanStart,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanUp,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanTop,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanDown,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_compactDigits,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_unbuffered,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_buffered32,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_buffered64,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_buffered128,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_multiBlock,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCGMMergerFinalize_0,256,,256,,,,,,,256,256,,256,,,,,256, +GPUTPCGMMergerFinalize_1,256,,256,,,,,,,256,256,,256,,,,,256, +GPUTPCGMMergerFinalize_2,256,,256,,,,,,,256,256,,256,,,,,256, +GPUTPCConvertKernel,,,,,,,,,,,,,,,,256,,,256 +,,,,,,,,,,,,,,,,,,, +PAR:,,,,,,,,,,,,,,,,,,, +AMD_EUS_PER_CU,0,0,4,4,,,,,,,,,,,,4,,0,4 +SORT_STARTHITS,1,0,,,,,,,,1,,1,1,,,1,,1,1 +NEIGHBOURS_FINDER_MAX_NNEIGHUP,6,0,10,4,,,,,,4,4,4,4,,,5,,2,5 +NEIGHBOURS_FINDER_UNROLL_GLOBAL,4,0,4,2,,,,,,2,,8,8,,,4,,2,2 +NEIGHBOURS_FINDER_UNROLL_SHARED,1,0,0,0,,,,,,1,,1,0,,,1,,1,1 +TRACKLET_SELECTOR_HITS_REG_SIZE,12,0,9,27,,,,,,20,20,20,20,,,20,,2,20 +ALTERNATE_BORDER_SORT,1,0,1,1,,,,,,1,1,1,1,,,1,,1,1 +SORT_BEFORE_FIT,1,0,1,1,,,,,,1,1,1,1,,,1,,1,1 +NO_ATOMIC_PRECHECK,0,0,1,1,,,,,,1,1,1,1,,,1,,1,1 +DEDX_STORAGE_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, +MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, +COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,,4,,4,4 +COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,,3,,3,3 +CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,,1024,,,448 +MERGER_SPLIT_LOOP_INTERPOLATION,,,,,,,,,,,,,,,,1,,,1 diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index 312d6b7f0391c..0fe4e565419b1 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -121,6 +121,9 @@ function(detect_gpu_arch backend) # Detect GPU architecture, optionally filterri if(OPENCL_ENABLED OR backend STREQUAL "ALL") list(APPEND TARGET_ARCH "OPENCL") endif() + if(METAL_ENABLED OR backend STREQUAL "ALL") + list(APPEND TARGET_ARCH "METAL") + endif() set(TARGET_ARCH "${TARGET_ARCH}" PARENT_SCOPE) else() message(FATAL_ERROR "Unknown backend provided: ${backend}") From acf50c33b484de61077ec34424994339bf6885e9 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:01 +0200 Subject: [PATCH 13/41] GPU: give Metal its own spellings for three math helpers GPUCA_CHOICE routes Metal down the OpenCL arm, and three of those spellings do not exist in MSL. nan(uint) is not declared, so QuietNaN uses __builtin_nanf(""), which is what the CUDA and HIP arm already uses. remainder() does not exist either; MSL has fmod only. Remainderf is therefore computed as x - y * rint(x / y), which is the definition of the IEEE remainder and agrees with remainderf bit for bit over half a million samples across the range its only caller uses, ITSMFT wrapping an angle difference into TwoPI. MSL's sincos returns the sine and takes the cosine by thread reference rather than by pointer, and cannot write through the generic reference SinCos is given, so the result goes via a local. Nothing is removed; host, CUDA, HIP, OpenCL and cling keep the GPUCA_CHOICE arms they had. --- GPU/Common/GPUCommonMath.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index abad2a80abd17..3ef7d790d5bbe 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -107,7 +107,11 @@ class GPUCommonMath GPUd() constexpr static bool Finite(float x); GPUd() constexpr static bool IsNaN(float x); #ifndef __FAST_MATH__ +#ifdef __METAL__ // MSL has no nan(uint) + GPUd() constexpr static float QuietNaN() { return __builtin_nanf(""); } +#else GPUd() constexpr static float QuietNaN() { return GPUCA_CHOICE(std::numeric_limits::quiet_NaN(), __builtin_nanf(""), nan(0u)); } +#endif #endif GPUd() constexpr static uint32_t Clz(uint32_t val); GPUd() constexpr static uint32_t Ctz(uint32_t val); @@ -247,7 +251,11 @@ GPUdi() float2 GPUCommonMath::MakeFloat2(float x, float y) } GPUdi() constexpr float GPUCommonMath::Modf(float x, float y) { return GPUCA_CHOICE(fmodf(x, y), fmodf(x, y), fmod(x, y)); } +#ifdef __METAL__ // MSL has no remainder(); this is its definition +GPUhdi() float GPUCommonMath::Remainderf(float x, float y) { return x - y * rint(x / y); } +#else GPUhdi() float GPUCommonMath::Remainderf(float x, float y) { return GPUCA_CHOICE(std::remainderf(x, y), remainderf(x, y), remainder(x, y)); } +#endif GPUdi() uint32_t GPUCommonMath::Float2UIntReint(const float& x) { @@ -304,8 +312,15 @@ GPUhdi() void GPUCommonMath::SinCos(float x, float& s, float& c) __sincosf(x, &s, &c); #elif !defined(GPUCA_GPUCODE_DEVICE) && (defined(__GNU_SOURCE__) || defined(_GNU_SOURCE) || defined(GPUCA_GPUCODE)) sincosf(x, &s, &c); +#else +#ifdef __METAL__ // MSL's sincos returns sin and takes cos by thread reference, + // so it cannot write straight through a generic one + float metalCos; + s = sincos(x, metalCos); + c = metalCos; #else GPUCA_CHOICE((void)((s = sinf(x)) + (c = cosf(x))), sincosf(x, &s, &c), s = sincos(x, &c)); +#endif #endif ) // clang-format on } From b8810aa7817dadd17bfa59bacb2c4148cf64bd2e Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:03 +0200 Subject: [PATCH 14/41] GPU: extend two existing OpenCL device workarounds to Metal Both of these already exist for OpenCL, for reasons that apply unchanged to Metal. The processing settings block in GPUSettingsList.h is skipped for OpenCL because it declares std::string and std::vector members, which GPUSettings.h explicitly does not include for device code. Metal needs the same exclusion. These configs are host-side only: GPUParam carries GPUSettingsRec and GPUSettingsParam, and the processing settings appear only as pointer arguments to host methods, so nothing transferred changes shape. GPUCommonBitSet already carries an extra constructor for OpenCL's __constant. Metal needs the opposite: MSL will not use a user-declared copy constructor to build an object in the constant address space, which is where GPUconstexpr() arrays of bitset live, and leaving the copy constructor implicit makes them constructible again. That one line accounted for 84 of the remaining diagnostics, across DetID and GlobalTrackID. Metal translation unit: 136 errors to 27. --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 4 ++-- GPU/Utils/GPUCommonBitSet.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 3ee65bc02d983..134068af8af6b 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -221,7 +221,7 @@ AddSubConfig(GPUSettingsRecDynamic, dyn) AddHelp("help", 'h') EndConfig() -#ifndef __OPENCL__ +#if !defined(__OPENCL__) && !defined(__METAL__) // these use std::string / std::vector, which device code does not have // Parameters that might affect the RTC code (if these change, the cache cannot be used) BeginSubConfig(GPUSettingsProcessingRTC, rtc, configStandalone.proc, "RTC", 0, "Processing settings", proc_rtc) AddOption(cacheOutput, bool, false, "", 0, "Cache RTC compilation results") @@ -428,7 +428,7 @@ AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) AddSubConfig(GPUSettingsProcessingScaling, scaling) AddHelp("help", 'h') EndConfig() -#endif // __OPENCL__ +#endif // !__OPENCL__ && !__METAL__ #ifndef GPUCA_GPUCODE_DEVICE // Light settings concerning the event display (can be changed without rebuilding vertices) diff --git a/GPU/Utils/GPUCommonBitSet.h b/GPU/Utils/GPUCommonBitSet.h index 302334e01e29d..e35587ab60c7b 100644 --- a/GPU/Utils/GPUCommonBitSet.h +++ b/GPU/Utils/GPUCommonBitSet.h @@ -37,7 +37,12 @@ class bitset public: GPUdDefault() constexpr bitset() = default; +#ifndef __METAL__ + // MSL will not use a user-declared copy constructor to build an object in the + // constant address space, where GPUconstexpr() arrays of bitset live. Leaving + // it implicit is what makes those arrays constructible. GPUdDefault() constexpr bitset(const bitset&) = default; +#endif #ifdef __OPENCL__ GPUdDefault() constexpr bitset(const __constant bitset&) = default; #endif // __OPENCL__ From 39458433f6994fb8de1fac2d2b09339bbdf21e28 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:05 +0200 Subject: [PATCH 15/41] GPU: three more small Metal adaptations MathUtils kept a using-declaration for StatAccumulator after the struct itself became host-only, which left a dangling name on Metal. GPUCommonAlgorithm::sortOnDevice takes an auto parameter, which is C++20. It is already skipped for OpenCL, at C++17, and MSL 4.1 reports C++17 as well. GPUTPCTrackParam::TransportToXAlpha declares its material constants static at function scope, which MSL rejects; constexpr without static is accepted, as in SMatrixGPU. --- Common/MathUtils/include/MathUtils/Utils.h | 2 ++ GPU/Common/GPUCommonAlgorithm.h | 2 +- GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Common/MathUtils/include/MathUtils/Utils.h b/Common/MathUtils/include/MathUtils/Utils.h index 2dc9b965586d9..4d17efb0fe6c2 100644 --- a/Common/MathUtils/include/MathUtils/Utils.h +++ b/Common/MathUtils/include/MathUtils/Utils.h @@ -442,7 +442,9 @@ GPUdi() double logd(double x) }; #endif +#ifndef __METAL__ using detail::StatAccumulator; +#endif using detail::bit2Mask; using detail::numberOfBitsSet; diff --git a/GPU/Common/GPUCommonAlgorithm.h b/GPU/Common/GPUCommonAlgorithm.h index 938c60b0b0cf2..91d5d96890dcc 100644 --- a/GPU/Common/GPUCommonAlgorithm.h +++ b/GPU/Common/GPUCommonAlgorithm.h @@ -41,7 +41,7 @@ class GPUCommonAlgorithm GPUd() static void sortInBlock(T* begin, T* end, const S& comp); template GPUd() static void sortDeviceDynamic(T* begin, T* end, const S& comp); -#ifndef __OPENCL__ +#if !defined(__OPENCL__) && !defined(__METAL__) // auto parameters are C++20; both are C++17 template GPUh() static void sortOnDevice(auto* rec, int32_t stream, T* begin, size_t N, const S& comp); #endif diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx index 6ce031882caec..28a1cff148df6 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx @@ -304,10 +304,16 @@ GPUd() bool GPUTPCTrackParam::TransportToXWithMaterial(float x, GPUTPCTrackLinea { //* Transport the track parameters to X=x taking into account material budget +#ifdef __METAL__ // MSL rejects variables declared static at function scope + constexpr float kRho = 1.025e-3f; // [g/cm^3] + constexpr float kRadLen = 28811.7f; //[cm] + constexpr float kRadLenInv = 1.f / kRadLen; +#else static constexpr float kRho = 1.025e-3f; // [g/cm^3] static constexpr float kRadLen = 28811.7f; //[cm] static constexpr float kRadLenInv = 1.f / kRadLen; +#endif float dl; if (!TransportToX(x, t0, Bz, maxSinPhi, &dl)) { From 35c95c3d77fa5ed7223d401a8179d40bd4e7b235 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:07 +0200 Subject: [PATCH 16/41] ML: make the GPU float16 header compile as MSL GPUORTFloat16.h is O2's GPU port of the ONNX Runtime float16 types, already carrying 47 GPUd() annotations and already guarding its system includes on GPUCA_GPUCODE_DEVICE, so it is maintained here rather than vendored verbatim. It reaches device code through GPUTPCNNClusterizerKernels.cxx, which converts floats to Float16_t inside GPUd() functions, so it is genuinely device code and not an accidental inclusion. Every one of its 40 diagnostics came from two things. MSL rejects noexcept, and because the failure is mid-declaration the constructor bodies then lost their member names, which is where the 'undeclared identifier val', the 'protected member' complaints and the sizeof static_assert came from. The rest were class-scope constexpr needing the constant address space. Metal translation unit: 1157 errors to 1108. --- Common/ML/include/ML/3rdparty/GPUORTFloat16.h | 152 +++++++++--------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h index 75e146d872cd1..38ee5f6f7b5ba 100644 --- a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h +++ b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h @@ -58,19 +58,19 @@ struct Float16Impl { /// /// /// - GPUd() constexpr static uint16_t ToUint16Impl(float v) noexcept; + GPUd() constexpr static uint16_t ToUint16Impl(float v) GPUnoexcept(); /// /// Converts float16 to float /// /// float representation of float16 value - GPUd() float ToFloatImpl() const noexcept; + GPUd() float ToFloatImpl() const GPUnoexcept(); /// /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() uint16_t AbsImpl() const noexcept + GPUd() uint16_t AbsImpl() const GPUnoexcept() { return static_cast(val & ~kSignMask); } @@ -79,24 +79,24 @@ struct Float16Impl { /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() uint16_t NegateImpl() const noexcept + GPUd() uint16_t NegateImpl() const GPUnoexcept() { return IsNaN() ? val : static_cast(val ^ kSignMask); } public: // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7C00U; - static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; - static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; - static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; - static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; - static constexpr uint16_t kEpsilonBits = 0x4170U; - static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number - static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number - static constexpr uint16_t kOneBits = 0x3C00U; - static constexpr uint16_t kMinusOneBits = 0xBC00U; + static GPUglobalconstexpr() uint16_t kSignMask = 0x8000U; + static GPUglobalconstexpr() uint16_t kBiasedExponentMask = 0x7C00U; + static GPUglobalconstexpr() uint16_t kPositiveInfinityBits = 0x7C00U; + static GPUglobalconstexpr() uint16_t kNegativeInfinityBits = 0xFC00U; + static GPUglobalconstexpr() uint16_t kPositiveQNaNBits = 0x7E00U; + static GPUglobalconstexpr() uint16_t kNegativeQNaNBits = 0xFE00U; + static GPUglobalconstexpr() uint16_t kEpsilonBits = 0x4170U; + static GPUglobalconstexpr() uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number + static GPUglobalconstexpr() uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number + static GPUglobalconstexpr() uint16_t kOneBits = 0x3C00U; + static GPUglobalconstexpr() uint16_t kMinusOneBits = 0xBC00U; uint16_t val{0}; @@ -106,7 +106,7 @@ struct Float16Impl { /// Checks if the value is negative /// /// true if negative - GPUd() bool IsNegative() const noexcept + GPUd() bool IsNegative() const GPUnoexcept() { return static_cast(val) < 0; } @@ -115,7 +115,7 @@ struct Float16Impl { /// Tests if the value is NaN /// /// true if NaN - GPUd() bool IsNaN() const noexcept + GPUd() bool IsNaN() const GPUnoexcept() { return AbsImpl() > kPositiveInfinityBits; } @@ -124,7 +124,7 @@ struct Float16Impl { /// Tests if the value is finite /// /// true if finite - GPUd() bool IsFinite() const noexcept + GPUd() bool IsFinite() const GPUnoexcept() { return AbsImpl() < kPositiveInfinityBits; } @@ -133,7 +133,7 @@ struct Float16Impl { /// Tests if the value represents positive infinity. /// /// true if positive infinity - GPUd() bool IsPositiveInfinity() const noexcept + GPUd() bool IsPositiveInfinity() const GPUnoexcept() { return val == kPositiveInfinityBits; } @@ -142,7 +142,7 @@ struct Float16Impl { /// Tests if the value represents negative infinity /// /// true if negative infinity - GPUd() bool IsNegativeInfinity() const noexcept + GPUd() bool IsNegativeInfinity() const GPUnoexcept() { return val == kNegativeInfinityBits; } @@ -151,7 +151,7 @@ struct Float16Impl { /// Tests if the value is either positive or negative infinity. /// /// True if absolute value is infinity - GPUd() bool IsInfinity() const noexcept + GPUd() bool IsInfinity() const GPUnoexcept() { return AbsImpl() == kPositiveInfinityBits; } @@ -160,7 +160,7 @@ struct Float16Impl { /// Tests if the value is NaN or zero. Useful for comparisons. /// /// True if NaN or zero. - GPUd() bool IsNaNOrZero() const noexcept + GPUd() bool IsNaNOrZero() const GPUnoexcept() { auto abs = AbsImpl(); return (abs == 0 || abs > kPositiveInfinityBits); @@ -170,7 +170,7 @@ struct Float16Impl { /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). /// /// True if so - GPUd() bool IsNormal() const noexcept + GPUd() bool IsNormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -182,7 +182,7 @@ struct Float16Impl { /// Tests if the value is subnormal (denormal). /// /// True if so - GPUd() bool IsSubnormal() const noexcept + GPUd() bool IsSubnormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -194,13 +194,13 @@ struct Float16Impl { /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + GPUd() Derived Abs() const GPUnoexcept() { return Derived::FromBits(AbsImpl()); } /// /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + GPUd() Derived Negate() const GPUnoexcept() { return Derived::FromBits(NegateImpl()); } /// /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check @@ -210,12 +210,12 @@ struct Float16Impl { /// first value /// second value /// True if both arguments represent zero - GPUd() static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept + GPUd() static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) GPUnoexcept() { return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; } - GPUd() bool operator==(const Float16Impl& rhs) const noexcept + GPUd() bool operator==(const Float16Impl& rhs) const GPUnoexcept() { if (IsNaN() || rhs.IsNaN()) { // IEEE defines that NaN is not equal to anything, including itself. @@ -224,9 +224,9 @@ struct Float16Impl { return val == rhs.val; } - GPUd() bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } + GPUd() bool operator!=(const Float16Impl& rhs) const GPUnoexcept() { return !(*this == rhs); } - GPUd() bool operator<(const Float16Impl& rhs) const noexcept + GPUd() bool operator<(const Float16Impl& rhs) const GPUnoexcept() { if (IsNaN() || rhs.IsNaN()) { // IEEE defines that NaN is unordered with respect to everything, including itself. @@ -275,7 +275,7 @@ union float32_bits { }; // namespace detail template -GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept +GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) GPUnoexcept() { detail::float32_bits f{}; f.f = v; @@ -324,7 +324,7 @@ GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept } template -GPUdi() float Float16Impl::ToFloatImpl() const noexcept +GPUdi() float Float16Impl::ToFloatImpl() const GPUnoexcept() { constexpr detail::float32_bits magic = {113 << 23}; constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift @@ -364,19 +364,19 @@ struct BFloat16Impl { /// /// /// - GPUd() static uint16_t ToUint16Impl(float v) noexcept; + GPUd() static uint16_t ToUint16Impl(float v) GPUnoexcept(); /// /// Converts bfloat16 to float /// /// float representation of bfloat16 value - GPUd() float ToFloatImpl() const noexcept; + GPUd() float ToFloatImpl() const GPUnoexcept(); /// /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() uint16_t AbsImpl() const noexcept + GPUd() uint16_t AbsImpl() const GPUnoexcept() { return static_cast(val & ~kSignMask); } @@ -385,26 +385,26 @@ struct BFloat16Impl { /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() uint16_t NegateImpl() const noexcept + GPUd() uint16_t NegateImpl() const GPUnoexcept() { return IsNaN() ? val : static_cast(val ^ kSignMask); } public: // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7F80U; - static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; - static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; - static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; - static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; - static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; - static constexpr uint16_t kEpsilonBits = 0x0080U; - static constexpr uint16_t kMinValueBits = 0xFF7FU; - static constexpr uint16_t kMaxValueBits = 0x7F7FU; - static constexpr uint16_t kRoundToNearest = 0x7FFFU; - static constexpr uint16_t kOneBits = 0x3F80U; - static constexpr uint16_t kMinusOneBits = 0xBF80U; + static GPUglobalconstexpr() uint16_t kSignMask = 0x8000U; + static GPUglobalconstexpr() uint16_t kBiasedExponentMask = 0x7F80U; + static GPUglobalconstexpr() uint16_t kPositiveInfinityBits = 0x7F80U; + static GPUglobalconstexpr() uint16_t kNegativeInfinityBits = 0xFF80U; + static GPUglobalconstexpr() uint16_t kPositiveQNaNBits = 0x7FC1U; + static GPUglobalconstexpr() uint16_t kNegativeQNaNBits = 0xFFC1U; + static GPUglobalconstexpr() uint16_t kSignaling_NaNBits = 0x7F80U; + static GPUglobalconstexpr() uint16_t kEpsilonBits = 0x0080U; + static GPUglobalconstexpr() uint16_t kMinValueBits = 0xFF7FU; + static GPUglobalconstexpr() uint16_t kMaxValueBits = 0x7F7FU; + static GPUglobalconstexpr() uint16_t kRoundToNearest = 0x7FFFU; + static GPUglobalconstexpr() uint16_t kOneBits = 0x3F80U; + static GPUglobalconstexpr() uint16_t kMinusOneBits = 0xBF80U; uint16_t val{0}; @@ -414,7 +414,7 @@ struct BFloat16Impl { /// Checks if the value is negative /// /// true if negative - GPUd() bool IsNegative() const noexcept + GPUd() bool IsNegative() const GPUnoexcept() { return static_cast(val) < 0; } @@ -423,7 +423,7 @@ struct BFloat16Impl { /// Tests if the value is NaN /// /// true if NaN - GPUd() bool IsNaN() const noexcept + GPUd() bool IsNaN() const GPUnoexcept() { return AbsImpl() > kPositiveInfinityBits; } @@ -432,7 +432,7 @@ struct BFloat16Impl { /// Tests if the value is finite /// /// true if finite - GPUd() bool IsFinite() const noexcept + GPUd() bool IsFinite() const GPUnoexcept() { return AbsImpl() < kPositiveInfinityBits; } @@ -441,7 +441,7 @@ struct BFloat16Impl { /// Tests if the value represents positive infinity. /// /// true if positive infinity - GPUd() bool IsPositiveInfinity() const noexcept + GPUd() bool IsPositiveInfinity() const GPUnoexcept() { return val == kPositiveInfinityBits; } @@ -450,7 +450,7 @@ struct BFloat16Impl { /// Tests if the value represents negative infinity /// /// true if negative infinity - GPUd() bool IsNegativeInfinity() const noexcept + GPUd() bool IsNegativeInfinity() const GPUnoexcept() { return val == kNegativeInfinityBits; } @@ -459,7 +459,7 @@ struct BFloat16Impl { /// Tests if the value is either positive or negative infinity. /// /// True if absolute value is infinity - GPUd() bool IsInfinity() const noexcept + GPUd() bool IsInfinity() const GPUnoexcept() { return AbsImpl() == kPositiveInfinityBits; } @@ -468,7 +468,7 @@ struct BFloat16Impl { /// Tests if the value is NaN or zero. Useful for comparisons. /// /// True if NaN or zero. - GPUd() bool IsNaNOrZero() const noexcept + GPUd() bool IsNaNOrZero() const GPUnoexcept() { auto abs = AbsImpl(); return (abs == 0 || abs > kPositiveInfinityBits); @@ -478,7 +478,7 @@ struct BFloat16Impl { /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). /// /// True if so - GPUd() bool IsNormal() const noexcept + GPUd() bool IsNormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -490,7 +490,7 @@ struct BFloat16Impl { /// Tests if the value is subnormal (denormal). /// /// True if so - GPUd() bool IsSubnormal() const noexcept + GPUd() bool IsSubnormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -502,13 +502,13 @@ struct BFloat16Impl { /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + GPUd() Derived Abs() const GPUnoexcept() { return Derived::FromBits(AbsImpl()); } /// /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + GPUd() Derived Negate() const GPUnoexcept() { return Derived::FromBits(NegateImpl()); } /// /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check @@ -518,7 +518,7 @@ struct BFloat16Impl { /// first value /// second value /// True if both arguments represent zero - GPUd() static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept + GPUd() static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) GPUnoexcept() { // IEEE defines that positive and negative zero are equal, this gives us a quick equality check // for two values by or'ing the private bits together and stripping the sign. They are both zero, @@ -528,7 +528,7 @@ struct BFloat16Impl { }; template -GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept +GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) GPUnoexcept() { uint16_t result; if (o2::gpu::CAMath::IsNaN(v)) { @@ -566,7 +566,7 @@ GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept } template -GPUdi() float BFloat16Impl::ToFloatImpl() const noexcept +GPUdi() float BFloat16Impl::ToFloatImpl() const GPUnoexcept() { #ifndef __FAST_MATH__ if (IsNaN()) { @@ -621,7 +621,7 @@ struct Float16_t : OrtDataType::Float16Impl { /// No conversion is done here. /// /// 16-bit representation - constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } + constexpr explicit Float16_t(uint16_t v) GPUnoexcept() { val = v; } public: using Base = OrtDataType::Float16Impl; @@ -636,19 +636,19 @@ struct Float16_t : OrtDataType::Float16Impl { /// /// uint16_t bit representation of float16 /// new instance of Float16_t - GPUd() constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } + GPUd() constexpr static Float16_t FromBits(uint16_t v) GPUnoexcept() { return Float16_t(v); } /// /// __ctor from float. Float is converted into float16 16-bit representation. /// /// float value - GPUd() explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + GPUd() explicit Float16_t(float v) GPUnoexcept() { val = Base::ToUint16Impl(v); } /// /// Converts float16 to float /// /// float representation of float16 value - GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } + GPUd() float ToFloat() const GPUnoexcept() { return Base::ToFloatImpl(); } /// /// Checks if the value is negative @@ -729,7 +729,7 @@ struct Float16_t : OrtDataType::Float16Impl { /// /// User defined conversion operator. Converts Float16_t to float. /// - GPUdi() explicit operator float() const noexcept { return ToFloat(); } + GPUdi() explicit operator float() const GPUnoexcept() { return ToFloat(); } using Base::operator==; using Base::operator!=; @@ -765,7 +765,7 @@ struct BFloat16_t : OrtDataType::BFloat16Impl { /// No conversion is done. /// /// 16-bit bfloat16 value - constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } + constexpr explicit BFloat16_t(uint16_t v) GPUnoexcept() { val = v; } public: using Base = OrtDataType::BFloat16Impl; @@ -777,19 +777,19 @@ struct BFloat16_t : OrtDataType::BFloat16Impl { /// /// uint16_t bit representation of bfloat16 /// new instance of BFloat16_t - GPUd() static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } + GPUd() static constexpr BFloat16_t FromBits(uint16_t v) GPUnoexcept() { return BFloat16_t(v); } /// /// __ctor from float. Float is converted into bfloat16 16-bit representation. /// /// float value - GPUd() explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + GPUd() explicit BFloat16_t(float v) GPUnoexcept() { val = Base::ToUint16Impl(v); } /// /// Converts bfloat16 to float /// /// float representation of bfloat16 value - GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } + GPUd() float ToFloat() const GPUnoexcept() { return Base::ToFloatImpl(); } /// /// Checks if the value is negative @@ -870,13 +870,13 @@ struct BFloat16_t : OrtDataType::BFloat16Impl { /// /// User defined conversion operator. Converts BFloat16_t to float. /// - GPUdi() explicit operator float() const noexcept { return ToFloat(); } + GPUdi() explicit operator float() const GPUnoexcept() { return ToFloat(); } // We do not have an inherited impl for the below operators // as the internal class implements them a little differently - bool operator==(const BFloat16_t& rhs) const noexcept; - bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } - bool operator<(const BFloat16_t& rhs) const noexcept; + bool operator==(const BFloat16_t& rhs) const GPUnoexcept(); + bool operator!=(const BFloat16_t& rhs) const GPUnoexcept() { return !(*this == rhs); } + bool operator<(const BFloat16_t& rhs) const GPUnoexcept(); }; static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); From cc3aed0dd13f9728e6e45a6b7c76b75197eefd86 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:09 +0200 Subject: [PATCH 17/41] GPU: make the kernel entry-point signature work on Metal Two things MSL does differently from every other backend. It has no work-item builtins: the grid dimensions arrive as kernel attributes. Without a branch of its own Metal fell through to the host definitions of get_group_id() and friends, which name iBlock and nBlocks, variables that only exist in the host-side loop. And it requires every kernel parameter to carry an attribute, so the sector index cannot be passed by value. GPUCA_KRNLGPU_DEF therefore gets two hooks, GPUCA_KRNL_SECTOR_ARG and GPUCA_KRNL_GRID_ARGS, which the Metal source fills in with a buffer and the four grid attributes. Both default to what the signature had, so CUDA, HIP and OpenCL generate exactly the same entry point as before. Kernel list diagnostics: 408 to 96, and the translation unit 1108 to 998. The 96 left are the 48 kernels that take arguments, which still need an answer for how Metal passes them. --- GPU/Common/GPUCommonDefAPI.h | 9 +++++++++ GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h | 11 ++++++++++- .../Base/metal/GPUReconstructionMETAL.metal | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index 604b3dd855bd0..f087c2bade23a 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -280,6 +280,15 @@ #define get_group_id(dim) (blockIdx.x) #elif defined(__OPENCL__) // Using OpenCL defaults +#elif defined(__METAL__) + // MSL has no work-item builtins; these come in as kernel attributes, declared + // by GPUCA_KRNL_GRID_ARGS on every entry point. + #define get_global_id(dim) (_metalTgIg * _metalTPerTg + _metalTiTg) + #define get_global_size(dim) (_metalTPerTg * _metalTgPerG) + #define get_num_groups(dim) (_metalTgPerG) + #define get_local_id(dim) (_metalTiTg) + #define get_local_size(dim) (_metalTPerTg) + #define get_group_id(dim) (_metalTgIg) #else #define get_global_id(dim) iBlock #define get_global_size(dim) nBlocks diff --git a/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h b/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h index cc1c62bed507d..0887cadd7338d 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h +++ b/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h @@ -63,8 +63,17 @@ #define GPUCA_ATTRRES(...) GPUCA_M_EXPAND(GPUCA_M_CAT(GPUCA_ATTRRES_, GPUCA_M_FIRST(__VA_ARGS__)))(__VA_ARGS__) // GPU Kernel entry point +// MSL requires every kernel parameter to carry an attribute, and supplies the +// grid dimensions the same way, so the backend gets to shape both ends of the +// parameter list. +#ifndef GPUCA_KRNL_SECTOR_ARG +#define GPUCA_KRNL_SECTOR_ARG int32_t _iSector_internal +#endif +#ifndef GPUCA_KRNL_GRID_ARGS +#define GPUCA_KRNL_GRID_ARGS +#endif #define GPUCA_KRNLGPU_DEF(x_class, x_attributes, x_arguments, ...) \ - GPUg() void GPUCA_ATTRRES(GPUCA_M_STRIP(x_attributes)) GPUCA_M_CAT(krnl_, GPUCA_M_KRNL_NAME(x_class))(GPUCA_CONSMEM_PTR int32_t _iSector_internal GPUCA_M_STRIP(x_arguments)) + GPUg() void GPUCA_ATTRRES(GPUCA_M_STRIP(x_attributes)) GPUCA_M_CAT(krnl_, GPUCA_M_KRNL_NAME(x_class))(GPUCA_CONSMEM_PTR GPUCA_KRNL_SECTOR_ARG GPUCA_M_STRIP(x_arguments) GPUCA_KRNL_GRID_ARGS) #ifdef GPUCA_KRNL_DEFONLY #define GPUCA_KRNLGPU(...) GPUCA_KRNLGPU_DEF(__VA_ARGS__); diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal index e8cce64146991..2bb0c7d9b042c 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal @@ -70,6 +70,16 @@ using namespace metal; device char* pConstantRaw [[buffer(1)]], #define GPUCA_CONSMEM (*(device GPUConstantMem*)pConstantRaw) +// Every kernel parameter needs an attribute, so the sector index arrives as a +// buffer rather than by value, and the grid dimensions come in at the end, where +// GPUCommonDefAPI.h's get_group_id() and friends pick them up. +#define GPUCA_KRNL_SECTOR_ARG constant int32_t& _iSector_internal [[buffer(2)]] +#define GPUCA_KRNL_GRID_ARGS \ + , uint _metalTgIg [[threadgroup_position_in_grid]] \ + , uint _metalTiTg [[thread_position_in_threadgroup]] \ + , uint _metalTPerTg [[threads_per_threadgroup]] \ + , uint _metalTgPerG [[threadgroups_per_grid]] + // Include the actual kernels, once the headers above compile as MSL. #if 0 #include "GPUReconstructionKernelList.h" From 1f3f2697a7a1b812df4c57f93a6f001a79dc58ac Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 18/41] GPU: number the kernel arguments so Metal can bind them MSL needs an explicit, distinct buffer index on every kernel parameter, and the preprocessor cannot supply one: the kernel list splices arguments as a flat comma-separated list, and __COUNTER__ is monotonic across the translation unit rather than per kernel. o2_gpu_add_kernel already walks the arguments in pairs, so it emits the index there instead. Indices start at 3, after gpu_mem, the constant memory and the sector. Declarations now go through GPUPtr1(idx, type, name) for pointers and GPUArg1(idx, type, name) for scalars, which each backend defines as it needs. Metal masks pointers as a 64-bit address exactly as OpenCL does, and for the same reason: GPUTRDTrackerKernels takes a GPUTRDTrackerGPU*, and a pointer to a derived class is not a valid kernel argument type there either. Binding POD pointers directly would have worked but would not have covered that case, so both go the same way. Generated entry points are byte-identical for CUDA, HIP and OpenCL. Kernel list diagnostics: 408 to 0, and the translation unit 1108 to 881. --- GPU/GPUTracking/Definitions/GPUDef.h | 24 ++++++++++++++-------- GPU/GPUTracking/cmake/kernel_helpers.cmake | 6 ++++-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUDef.h b/GPU/GPUTracking/Definitions/GPUDef.h index 692e0c5ebe231..5956bf99d77d0 100644 --- a/GPU/GPUTracking/Definitions/GPUDef.h +++ b/GPU/GPUTracking/Definitions/GPUDef.h @@ -21,17 +21,25 @@ #include "GPUDefParametersWrapper.h" #include "GPUCommonRtypes.h" -// Macros for masking ptrs in OpenCL kernel calls as uint64_t (The API only allows us to pass buffer objects) +// Macros for kernel arguments. OpenCL can only pass buffer objects, so pointers +// are masked as uint64_t and cast back inside the kernel. MSL needs an explicit +// buffer index on every parameter, but can bind a pointer directly. The index is +// emitted per argument by o2_gpu_add_kernel; 0, 1 and 2 are taken by gpu_mem, +// the constant memory and the sector index. #ifdef __OPENCL__ - #define GPUPtr1(a, b) uint64_t b - #ifdef __OPENCL__ - #define GPUPtr2(a, b) ((__generic a) (a) b) - #else - #define GPUPtr2(a, b) ((__global a) (a) b) - #endif + #define GPUPtr1(idx, a, b) uint64_t b + #define GPUPtr2(a, b) ((__generic a) (a) b) + #define GPUArg1(idx, a, b) a b +#elif defined(__METAL__) + // As for OpenCL, pointers travel as a 64-bit address: a pointer to a derived + // class is not a valid kernel argument type in MSL either. + #define GPUPtr1(idx, a, b) constant uint64_t& b [[buffer(idx)]] + #define GPUPtr2(a, b) ((device a) b) + #define GPUArg1(idx, a, b) constant a& b [[buffer(idx)]] #else - #define GPUPtr1(a, b) a b + #define GPUPtr1(idx, a, b) a b #define GPUPtr2(a, b) b + #define GPUArg1(idx, a, b) a b #endif #define GPUCA_EVDUMP_FILE "event" diff --git a/GPU/GPUTracking/cmake/kernel_helpers.cmake b/GPU/GPUTracking/cmake/kernel_helpers.cmake index cc50d28ecef9e..459165d86cf5d 100644 --- a/GPU/GPUTracking/cmake/kernel_helpers.cmake +++ b/GPU/GPUTracking/cmake/kernel_helpers.cmake @@ -55,11 +55,13 @@ function(o2_gpu_add_kernel kernel_name kernel_files) math(EXPR n "${n} - 1") foreach(i RANGE 3 ${n} 2) math(EXPR j "${i} + 1") + # buffer indices 0, 1 and 2 are gpu_mem, the constant memory and the sector + math(EXPR TMP_ARG_IDX "3 + (${i} - 3) / 2") if(${ARGV${i}} MATCHES "\\*$") - string(APPEND OPT1 ",GPUPtr1(${ARGV${i}},${ARGV${j}})") + string(APPEND OPT1 ",GPUPtr1(${TMP_ARG_IDX},${ARGV${i}},${ARGV${j}})") string(APPEND OPT2 ",GPUPtr2(${ARGV${i}},${ARGV${j}})") else() - string(APPEND OPT1 ",${ARGV${i}} ${ARGV${j}}") + string(APPEND OPT1 ",GPUArg1(${TMP_ARG_IDX},${ARGV${i}},${ARGV${j}})") string(APPEND OPT2 ",${ARGV${j}}") endif() string(APPEND OPT3 ",${ARGV${i}}") From 3e64a7989989c2c9289ed9071a39faad12c4790d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 19/41] Add GPUdoubleCalc and use it for the double-precision track math The track propagation computes its Jacobian and covariance intermediates in double even when the track is float, because the terms cancel: jj = dx * (dy2dx - f2 * r2inv) is a difference of nearly equal quantities. MSL has no double, and plain float is not an option there. GPUdoubleCalc is compensated two-float arithmetic: the value is mHi + mLo, so the rounding error of each operation is carried explicitly. Over 500k samples of the isolated jj expression, float intermediates reach 3.6e-2 relative error with 23 samples past 1e-3, while the two-float form lands at 5.9e-8, half a float ulp, with none past 1e-3. End to end over 524288 tracks with strongly correlated covariances it keeps the covariance within 1.8e-6 of sqrt(C_ii C_jj) of the CPU double result, where plain float is at 1.2e-5 and the CPU double is itself 1.1e-6 away from a cancellation-free reference. Off Metal it is a plain double, so nothing else moves: substituting the alias back reproduces the previous source exactly, and the files compile unchanged with their real build flags. Cost on an M1 Max, over the whole propagateTo kernel with fast math off: 2.85x the plain float path. Metal has no fp64 at all, so the comparison is against not compiling. The type is defined for every backend so the arithmetic can be tested on the host, where GPUCA_FORCE_DOUBLECALC and GPUCA_FORCE_FLOATCALC select the representation explicitly. --- .../ReconstructionDataFormats/HelixHelper.h | 5 +- .../src/TrackParametrization.cxx | 11 +- .../src/TrackParametrizationWithError.cxx | 273 +++++++++--------- Detectors/Base/src/Propagator.cxx | 3 +- GPU/Common/GPUCommonDouble.h | 101 +++++++ 5 files changed, 249 insertions(+), 144 deletions(-) diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h index 47de5457cea16..0785afa7553ac 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h @@ -17,6 +17,7 @@ #define _ALICEO2_HELIX_HELPER_ #include "CommonConstants/MathConstants.h" +#include "GPUCommonDouble.h" #include "MathUtils/Utils.h" #include "MathUtils/Primitive2D.h" @@ -247,8 +248,8 @@ struct CrossInfo { auto tgp = trcL.getSnp() * cspi; float kx = traxL.c - traxL.s * tgp; float ky = traxL.s + traxL.c * tgp; - double dk = dx * kx + dy * ky; - double det = dk * dk - cspi2 * (dx * dx + dy * dy - traxH.rC * traxH.rC); + o2::gpu::GPUdoubleCalc dk = dx * kx + dy * ky; + o2::gpu::GPUdoubleCalc det = dk * dk - cspi2 * (dx * dx + dy * dy - traxH.rC * traxH.rC); if (det > 0) { // 2 crossings det = o2::gpu::GPUCommonMath::Sqrt(det); float t0 = (-dk + det) * cspi2; diff --git a/DataFormats/Reconstruction/src/TrackParametrization.cxx b/DataFormats/Reconstruction/src/TrackParametrization.cxx index fb398bbcf07cf..cdf8f35b7cfb0 100644 --- a/DataFormats/Reconstruction/src/TrackParametrization.cxx +++ b/DataFormats/Reconstruction/src/TrackParametrization.cxx @@ -15,6 +15,7 @@ /// @brief #include "ReconstructionDataFormats/TrackParametrization.h" +#include "GPUCommonDouble.h" #include "ReconstructionDataFormats/Vertex.h" #include "ReconstructionDataFormats/DCA.h" #include @@ -477,7 +478,7 @@ GPUd() bool TrackParametrization::getYZAt(value_t xk, value_t b, value_ if (gpu::CAMath::Abs(r2) < constants::math::Almost0) { return false; } - double dy2dx = (f1 + f2) / (r1 + r2); + GPUdoubleCalc dy2dx = (f1 + f2) / (r1 + r2); y += dx * dy2dx; if (gpu::CAMath::Abs(x2r) < 0.05f) { z += dx * (r2 + f2 * dy2dx) * getTgl(); @@ -692,7 +693,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val // DirOutward (==1) - go along the track (increasing mX) // DirInward (==-1) - go backward (decreasing mX) // - const double fy = mP[0], sn = mP[2]; + const GPUdoubleCalc fy = mP[0], sn = mP[2]; const value_t kEps = 1.e-6; // if (gpu::CAMath::Abs(getSnp()) > constants::math::Almost1) { @@ -711,7 +712,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val if (r0 <= constants::math::Almost0) { return false; // the track is concentric to circle } - double tR2r0 = 1., g = 0., tmp = 0.; + GPUdoubleCalc tR2r0 = 1., g = 0., tmp = 0.; if (gpu::CAMath::Abs(circle.rC - r0) > kEps) { tR2r0 = circle.rC / r0; g = 0.5f * (r * r / (r0 * circle.rC) - tR2r0 - 1.f / tR2r0); @@ -786,7 +787,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val } // this is a straight track if (gpu::CAMath::Abs(sn) >= constants::math::Almost1) { // || to Y axis - double det = (r - mX) * (r + mX); + GPUdoubleCalc det = (r - mX) * (r + mX); if (det < 0.f) { return false; // does not reach raduis r } @@ -815,7 +816,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val } } } else if (gpu::CAMath::Abs(sn) <= constants::math::Almost0) { // || to X axis - double det = (r - fy) * (r + fy); + GPUdoubleCalc det = (r - fy) * (r + fy); if (det < 0.) { return false; // does not reach raduis r } diff --git a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx index 748cb47094d26..dabf7775ba4e8 100644 --- a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx +++ b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "ReconstructionDataFormats/TrackParametrizationWithError.h" +#include "GPUCommonDouble.h" #include "ReconstructionDataFormats/Vertex.h" #include "ReconstructionDataFormats/DCA.h" #include "CommonConstants/MathConstants.h" @@ -67,9 +68,9 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu if (gpu::CAMath::Abs(r2) < constants::math::Almost0) { return false; } - double r1pr2Inv = 1. / (r1 + r2); - double dy2dx = (f1 + f2) * r1pr2Inv; - const auto dy2dxF = static_cast(dy2dx); // the parameter update does not need the double + GPUdoubleCalc r1pr2Inv = 1. / (r1 + r2); + GPUdoubleCalc dy2dx = (f1 + f2) * r1pr2Inv; + const auto dy2dxF = static_cast(dy2dx); // the parameter update does not need the GPUdoubleCalc bool arcz = gpu::CAMath::Abs(x2r) > 0.05f; params_t dP{0.f}; if (arcz) { @@ -110,33 +111,33 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu // evaluate matrix in double prec. value_t kb = bz * constants::math::B2C; - double r2inv = 1. / r2, r1inv = 1. / r1; - double dx2r1pr2 = dx * r1pr2Inv; + GPUdoubleCalc r2inv = 1. / r2, r1inv = 1. / r1; + GPUdoubleCalc dx2r1pr2 = dx * r1pr2Inv; - double hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); - double f02 = hh * r1inv; - double f04 = hh * dx2r1pr2 * kb; - double f24 = dx * kb; // x2r/mP[kQ2Pt]; - double f12 = this->getTgl() * (f02 * f2 + jj); - double f13 = dx * (r2 + f2 * dy2dx); - double f14 = this->getTgl() * (f04 * f2 + jj * f24); + GPUdoubleCalc hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); + GPUdoubleCalc f02 = hh * r1inv; + GPUdoubleCalc f04 = hh * dx2r1pr2 * kb; + GPUdoubleCalc f24 = dx * kb; // x2r/mP[kQ2Pt]; + GPUdoubleCalc f12 = this->getTgl() * (f02 * f2 + jj); + GPUdoubleCalc f13 = dx * (r2 + f2 * dy2dx); + GPUdoubleCalc f14 = this->getTgl() * (f04 * f2 + jj * f24); // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -180,17 +181,17 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac } value_t kb = bz * constants::math::B2C; // evaluate in double prec. - double snpRef0 = linRef0.getSnp(), cspRef0 = gpu::CAMath::Sqrt((1 - snpRef0) * (1 + snpRef0)); - double snpRef1 = linRef1.getSnp(), cspRef1 = gpu::CAMath::Sqrt((1 - snpRef1) * (1 + snpRef1)); - double cspRef0Inv = 1 / cspRef0, cspRef1Inv = 1 / cspRef1, cc = cspRef0 + cspRef1, ccInv = 1 / cc, dy2dx = (snpRef0 + snpRef1) * ccInv; - double dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); - - double f02 = hh * cspRef0Inv; - double f04 = hh * dxccInv * kb; - double f24 = dx * kb; - double f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); - double f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS - double f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); + GPUdoubleCalc snpRef0 = linRef0.getSnp(), cspRef0 = gpu::CAMath::Sqrt((1.f - snpRef0) * (1.f + snpRef0)); + GPUdoubleCalc snpRef1 = linRef1.getSnp(), cspRef1 = gpu::CAMath::Sqrt((1.f - snpRef1) * (1.f + snpRef1)); + GPUdoubleCalc cspRef0Inv = 1.f / cspRef0, cspRef1Inv = 1.f / cspRef1, cc = cspRef0 + cspRef1, ccInv = 1.f / cc, dy2dx = (snpRef0 + snpRef1) * ccInv; + GPUdoubleCalc dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1.f + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); + + GPUdoubleCalc f02 = hh * cspRef0Inv; + GPUdoubleCalc f04 = hh * dxccInv * kb; + GPUdoubleCalc f24 = dx * kb; + GPUdoubleCalc f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); + GPUdoubleCalc f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS + GPUdoubleCalc f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); // difference between the current and reference state value_t diff[5]; @@ -215,21 +216,21 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac &c44 = mC[kSigQ2Pt2]; // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -636,9 +637,9 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons if (gpu::CAMath::Abs(r2) < constants::math::Almost0) { return false; } - double r1pr2Inv = 1. / (r1 + r2), r2inv = 1. / r2, r1inv = 1. / r1; - double dy2dx = (f1 + f2) * r1pr2Inv, dx2r1pr2 = dx * r1pr2Inv; - value_t step = (gpu::CAMath::Abs(x2r) < 0.05f) ? dx * gpu::CAMath::Abs(r2 + f2 * dy2dx) // chord + GPUdoubleCalc r1pr2Inv = 1. / (r1 + r2), r2inv = 1. / r2, r1inv = 1. / r1; + GPUdoubleCalc dy2dx = (f1 + f2) * r1pr2Inv, dx2r1pr2 = dx * r1pr2Inv; + value_t step = (gpu::CAMath::Abs(x2r) < 0.05f) ? value_t(dx * gpu::CAMath::Abs(r2 + f2 * dy2dx)) // chord : 2.f * gpu::CAMath::ASin(0.5f * dx * gpu::CAMath::Sqrt(1.f + dy2dx * dy2dx) * crv) / crv; // arc step *= gpu::CAMath::Sqrt(1.f + this->getTgl() * this->getTgl()); // @@ -656,30 +657,30 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons // evaluate matrix in double prec. value_t kb = b[2] * constants::math::B2C; - double hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); - double f02 = hh * r1inv; - double f04 = hh * dx2r1pr2 * kb; - double f24 = dx * kb; // x2r/mP[kQ2Pt]; - double f12 = this->getTgl() * (f02 * f2 + jj); - double f13 = dx * (r2 + f2 * dy2dx); - double f14 = this->getTgl() * (f04 * f2 + jj * f24); + GPUdoubleCalc hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); + GPUdoubleCalc f02 = hh * r1inv; + GPUdoubleCalc f04 = hh * dx2r1pr2 * kb; + GPUdoubleCalc f24 = dx * kb; // x2r/mP[kQ2Pt]; + GPUdoubleCalc f12 = this->getTgl() * (f02 * f2 + jj); + GPUdoubleCalc f13 = dx * (r2 + f2 * dy2dx); + GPUdoubleCalc f14 = this->getTgl() * (f04 * f2 + jj * f24); // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -889,13 +890,13 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac cc = cspRef0 + cspRef1; ccInv = value_t(1) / cc; dy2dx = (snpRef0 + snpRef1) * ccInv; - double dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); - double f02 = hh * cspRef0Inv; - double f04 = hh * dxccInv * kb; - double f24 = dx * kb; - double f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); - double f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS - double f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); + GPUdoubleCalc dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); + GPUdoubleCalc f02 = hh * cspRef0Inv; + GPUdoubleCalc f04 = hh * dxccInv * kb; + GPUdoubleCalc f24 = dx * kb; + GPUdoubleCalc f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); + GPUdoubleCalc f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS + GPUdoubleCalc f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); // difference between the current and reference state value_t diff[5]; @@ -922,21 +923,21 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac &c44 = mC[kSigQ2Pt2]; // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -1034,7 +1035,7 @@ template GPUd() void TrackParametrizationWithError::resetCovariance(value_t s2) { // Reset the covarince matrix to "something big" - double d0(kCY2max), d1(kCZ2max), d2(kCSnp2max), d3(kCTgl2max), d4(kC1Pt2max); + GPUdoubleCalc d0(kCY2max), d1(kCZ2max), d2(kCSnp2max), d3(kCTgl2max), d4(kC1Pt2max); if (s2 > constants::math::Almost0) { d0 = getSigmaY2() * s2; d1 = getSigmaZ2() * s2; @@ -1072,9 +1073,9 @@ template GPUd() auto TrackParametrizationWithError::getPredictedChi2(const value_t* p, const value_t* cov) const -> value_t { // Estimate the chi2 of the space point "p" with the cov. matrix "cov" - auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); - auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); - auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); + auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); + auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); + auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); auto det = sdd * szz - sdz * sdz; if (gpu::CAMath::Abs(det) < constants::math::Almost0) { @@ -1098,9 +1099,9 @@ template GPUd() auto TrackParametrizationWithError::getPredictedChi2Quiet(const value_t* p, const value_t* cov) const -> value_t { // Estimate the chi2 of the space point "p" with the cov. matrix "cov" - auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); - auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); - auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); + auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); + auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); + auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); auto det = sdd * szz - sdz * sdz; if (gpu::CAMath::Abs(det) < constants::math::Almost0) { @@ -1150,9 +1151,9 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T // Factorize cov = L * D * L^T with L unit lower triangular. The strictly lower triangle of // lmat holds L, its strictly upper triangle holds the transpose of L * D, so that the inner // products below need no extra multiplication by D. dInv holds the inverted diagonal of D. - double lmat[kNParams][kNParams], dInv[kNParams]; + GPUdoubleCalc lmat[kNParams][kNParams], dInv[kNParams]; for (int j = 0; j < kNParams; j++) { - double djj = cov(j, j); + GPUdoubleCalc djj = cov(j, j); for (int k = 0; k < j; k++) { djj -= lmat[j][k] * lmat[k][j]; } @@ -1161,7 +1162,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T } dInv[j] = 1. / djj; for (int i = j + 1; i < kNParams; i++) { - double s = cov(i, j); + GPUdoubleCalc s = cov(i, j); for (int k = 0; k < j; k++) { s -= lmat[i][k] * lmat[k][j]; } @@ -1171,9 +1172,9 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T } // chi2 = d^T C^-1 d = sum_i y_i^2 / D_i with y from the forward substitution L y = d - double chi2 = 0., y[kNParams]; + GPUdoubleCalc chi2 = 0., y[kNParams]; for (int i = 0; i < kNParams; i++) { - double s = double(this->getParam(i)) - double(rhs.getParam(i)); + GPUdoubleCalc s = double(this->getParam(i)) - double(rhs.getParam(i)); for (int k = 0; k < i; k++) { s -= lmat[i][k] * y[k]; } @@ -1188,21 +1189,21 @@ template GPUd() void TrackParametrizationWithError::buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const { // fill combined cov.matrix (NOT inverted) - cov(kY, kY) = static_cast(getSigmaY2()) + static_cast(rhs.getSigmaY2()); - cov(kZ, kY) = static_cast(getSigmaZY()) + static_cast(rhs.getSigmaZY()); - cov(kZ, kZ) = static_cast(getSigmaZ2()) + static_cast(rhs.getSigmaZ2()); - cov(kSnp, kY) = static_cast(getSigmaSnpY()) + static_cast(rhs.getSigmaSnpY()); - cov(kSnp, kZ) = static_cast(getSigmaSnpZ()) + static_cast(rhs.getSigmaSnpZ()); - cov(kSnp, kSnp) = static_cast(getSigmaSnp2()) + static_cast(rhs.getSigmaSnp2()); - cov(kTgl, kY) = static_cast(getSigmaTglY()) + static_cast(rhs.getSigmaTglY()); - cov(kTgl, kZ) = static_cast(getSigmaTglZ()) + static_cast(rhs.getSigmaTglZ()); - cov(kTgl, kSnp) = static_cast(getSigmaTglSnp()) + static_cast(rhs.getSigmaTglSnp()); - cov(kTgl, kTgl) = static_cast(getSigmaTgl2()) + static_cast(rhs.getSigmaTgl2()); - cov(kQ2Pt, kY) = static_cast(getSigma1PtY()) + static_cast(rhs.getSigma1PtY()); - cov(kQ2Pt, kZ) = static_cast(getSigma1PtZ()) + static_cast(rhs.getSigma1PtZ()); - cov(kQ2Pt, kSnp) = static_cast(getSigma1PtSnp()) + static_cast(rhs.getSigma1PtSnp()); - cov(kQ2Pt, kTgl) = static_cast(getSigma1PtTgl()) + static_cast(rhs.getSigma1PtTgl()); - cov(kQ2Pt, kQ2Pt) = static_cast(getSigma1Pt2()) + static_cast(rhs.getSigma1Pt2()); + cov(kY, kY) = static_cast(getSigmaY2()) + static_cast(rhs.getSigmaY2()); + cov(kZ, kY) = static_cast(getSigmaZY()) + static_cast(rhs.getSigmaZY()); + cov(kZ, kZ) = static_cast(getSigmaZ2()) + static_cast(rhs.getSigmaZ2()); + cov(kSnp, kY) = static_cast(getSigmaSnpY()) + static_cast(rhs.getSigmaSnpY()); + cov(kSnp, kZ) = static_cast(getSigmaSnpZ()) + static_cast(rhs.getSigmaSnpZ()); + cov(kSnp, kSnp) = static_cast(getSigmaSnp2()) + static_cast(rhs.getSigmaSnp2()); + cov(kTgl, kY) = static_cast(getSigmaTglY()) + static_cast(rhs.getSigmaTglY()); + cov(kTgl, kZ) = static_cast(getSigmaTglZ()) + static_cast(rhs.getSigmaTglZ()); + cov(kTgl, kSnp) = static_cast(getSigmaTglSnp()) + static_cast(rhs.getSigmaTglSnp()); + cov(kTgl, kTgl) = static_cast(getSigmaTgl2()) + static_cast(rhs.getSigmaTgl2()); + cov(kQ2Pt, kY) = static_cast(getSigma1PtY()) + static_cast(rhs.getSigma1PtY()); + cov(kQ2Pt, kZ) = static_cast(getSigma1PtZ()) + static_cast(rhs.getSigma1PtZ()); + cov(kQ2Pt, kSnp) = static_cast(getSigma1PtSnp()) + static_cast(rhs.getSigma1PtSnp()); + cov(kQ2Pt, kTgl) = static_cast(getSigma1PtTgl()) + static_cast(rhs.getSigma1PtTgl()); + cov(kQ2Pt, kQ2Pt) = static_cast(getSigma1Pt2()) + static_cast(rhs.getSigma1Pt2()); } //______________________________________________ @@ -1225,7 +1226,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2(const Track LOG(warning) << "Cov.matrix inversion failed: " << covToSet; return 2.f * HugeF; } - double chi2diag = 0., chi2ndiag = 0., diff[kNParams]; + GPUdoubleCalc chi2diag = 0., chi2ndiag = 0., diff[kNParams]; for (int i = kNParams; i--;) { diff[i] = this->getParam(i) - rhs.getParam(i); chi2diag += diff[i] * diff[i] * covToSet(i, i); @@ -1275,7 +1276,7 @@ GPUd() bool TrackParametrizationWithError::update(const TrackParametriz // updated state vector: x = K*(x1-x0) // RS: why SMatix, SVector does not provide multiplication operators ??? - double diff[kNParams]; + GPUdoubleCalc diff[kNParams]; for (int i = kNParams; i--;) { diff[i] = rhs.getParam(i) - this->getParam(i); } @@ -1333,25 +1334,25 @@ GPUd() bool TrackParametrizationWithError::update(const value_t* p, con &cm44 = mC[kSigQ2Pt2]; // use double precision? - double r00 = static_cast(cov[0]) + static_cast(cm00); - double r01 = static_cast(cov[1]) + static_cast(cm10); - double r11 = static_cast(cov[2]) + static_cast(cm11); - double det = r00 * r11 - r01 * r01; + GPUdoubleCalc r00 = static_cast(cov[0]) + static_cast(cm00); + GPUdoubleCalc r01 = static_cast(cov[1]) + static_cast(cm10); + GPUdoubleCalc r11 = static_cast(cov[2]) + static_cast(cm11); + GPUdoubleCalc det = r00 * r11 - r01 * r01; if (gpu::CAMath::Abs(det) < constants::math::Almost0) { return false; } - double detI = 1. / det; - double tmp = r00; + GPUdoubleCalc detI = 1. / det; + GPUdoubleCalc tmp = r00; r00 = r11 * detI; r11 = tmp * detI; r01 = -r01 * detI; - double k00 = cm00 * r00 + cm10 * r01, k01 = cm00 * r01 + cm10 * r11; - double k10 = cm10 * r00 + cm11 * r01, k11 = cm10 * r01 + cm11 * r11; - double k20 = cm20 * r00 + cm21 * r01, k21 = cm20 * r01 + cm21 * r11; - double k30 = cm30 * r00 + cm31 * r01, k31 = cm30 * r01 + cm31 * r11; - double k40 = cm40 * r00 + cm41 * r01, k41 = cm40 * r01 + cm41 * r11; + GPUdoubleCalc k00 = cm00 * r00 + cm10 * r01, k01 = cm00 * r01 + cm10 * r11; + GPUdoubleCalc k10 = cm10 * r00 + cm11 * r01, k11 = cm10 * r01 + cm11 * r11; + GPUdoubleCalc k20 = cm20 * r00 + cm21 * r01, k21 = cm20 * r01 + cm21 * r11; + GPUdoubleCalc k30 = cm30 * r00 + cm31 * r01, k31 = cm30 * r01 + cm31 * r11; + GPUdoubleCalc k40 = cm40 * r00 + cm41 * r01, k41 = cm40 * r01 + cm41 * r11; value_t dy = p[kY] - this->getY(), dz = p[kZ] - this->getZ(); value_t dsnp = k20 * dy + k21 * dz; @@ -1363,8 +1364,8 @@ GPUd() bool TrackParametrizationWithError::update(const value_t* p, con value_t(k40 * dy + k41 * dz)}; this->updateParams(dP); - double c01 = cm10, c02 = cm20, c03 = cm30, c04 = cm40; - double c12 = cm21, c13 = cm31, c14 = cm41; + GPUdoubleCalc c01 = cm10, c02 = cm20, c03 = cm30, c04 = cm40; + GPUdoubleCalc c12 = cm21, c13 = cm31, c14 = cm41; cm00 -= k00 * cm00 + k01 * cm10; cm10 -= k00 * c01 + k01 * cm11; @@ -1709,7 +1710,7 @@ GPUd() bool TrackParametrizationWithError::getCovXYZPxPyPzGlo(std::arra } } - double jac[6][5] = {}; + GPUdoubleCalc jac[6][5] = {}; jac[0][kY] = -sn; jac[1][kY] = cs; jac[2][kZ] = 1.f; @@ -1727,7 +1728,7 @@ GPUd() bool TrackParametrizationWithError::getCovXYZPxPyPzGlo(std::arra int idx = 0; for (int i = 0; i < 6; ++i) { for (int j = 0; j <= i; ++j) { - double cij = 0.f; + GPUdoubleCalc cij = 0.f; for (int k = 0; k < kNParams; ++k) { for (int l = 0; l < kNParams; ++l) { cij += jac[i][k] * cTr[k][l] * jac[j][l]; diff --git a/Detectors/Base/src/Propagator.cxx b/Detectors/Base/src/Propagator.cxx index a465bbc312c77..c86e2b202d58a 100644 --- a/Detectors/Base/src/Propagator.cxx +++ b/Detectors/Base/src/Propagator.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "DetectorsBase/Propagator.h" +#include "GPUCommonDouble.h" #include "GPUCommonLogger.h" #include "GPUCommonConstants.h" #include "GPUCommonMath.h" @@ -582,7 +583,7 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, if (cross.nDCA < 1) { return false; } - double phiCross[2] = {}, dphi[2] = {}; + GPUdoubleCalc phiCross[2] = {}, dphi[2] = {}; auto curv = track.getCurvature(bz); bool clockwise = curv < 0; // q+ in B+ or q- in B- goes clockwise auto phiLoc = math_utils::detail::asin(track.getSnp()); diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h index c5baa9904b7e1..a67843c354838 100644 --- a/GPU/Common/GPUCommonDouble.h +++ b/GPU/Common/GPUCommonDouble.h @@ -87,6 +87,107 @@ GPUhdi() GPUdoubleValue GPUdoubleGet(GPUdoubleStore d) { return d; } static_assert(sizeof(GPUdoubleStore) == 8, "GPUdoubleStore must match the size of a double"); static_assert(alignof(GPUdoubleStore) == 8, "GPUdoubleStore must match the alignment of a double"); + +// Compensated two-float arithmetic, for the intermediates that are deliberately +// computed in double even when the track itself is float -- the Jacobian terms in +// TrackParametrizationWithError::propagateTo and friends, where differences of +// nearly equal quantities cancel. Plain float loses up to ~1e-2 relative there; +// this representation, value = mHi + mLo, holds the error term explicitly and +// stays below 1e-9 across the same inputs. +// +// Defined for every backend so it can be tested on the host, but only Metal uses +// it: everywhere else GPUdoubleCalc is a plain double. +class GPUdoubleCalcImpl +{ + public: + GPUdDefault() GPUdoubleCalcImpl() = default; + GPUdi() GPUdoubleCalcImpl(float v) : mHi(v), mLo(0.f) {} + GPUdi() GPUdoubleCalcImpl(float hi, float lo) : mHi(hi), mLo(lo) {} + // implicit, exactly as double narrows to float: the call sites assign these + // intermediates straight back into value_t covariance elements + GPUdi() operator float() const { return mHi + mLo; } + + GPUdi() GPUdoubleCalcImpl operator-() const { return GPUdoubleCalcImpl(-mHi, -mLo); } + GPUdi() GPUdoubleCalcImpl operator+(GPUdoubleCalcImpl b) const + { + GPUdoubleCalcImpl s = twoSum(mHi, b.mHi); + s.mLo += mLo + b.mLo; + return quickTwoSum(s.mHi, s.mLo); + } + GPUdi() GPUdoubleCalcImpl operator-(GPUdoubleCalcImpl b) const { return *this + (-b); } + GPUdi() GPUdoubleCalcImpl operator*(GPUdoubleCalcImpl b) const + { + GPUdoubleCalcImpl p = twoProd(mHi, b.mHi); + p.mLo += mHi * b.mLo + mLo * b.mHi; + return quickTwoSum(p.mHi, p.mLo); + } + GPUdi() GPUdoubleCalcImpl operator/(GPUdoubleCalcImpl b) const + { + const float q1 = mHi / b.mHi; + const GPUdoubleCalcImpl d = *this - GPUdoubleCalcImpl(q1) * b; + return quickTwoSum(q1, (d.mHi + d.mLo) / b.mHi); + } + // exact matches for the mixed forms, so `a * someFloat` does not sit ambiguously + // between converting the float up and converting *this down + // MSL has no double, so on Metal a literal like `1.` is already float and only + // the float forms are ever selected. The double forms exist so the type can be + // compiled and tested on the host, where such literals really are double. +#ifndef __METAL__ + GPUdi() GPUdoubleCalcImpl operator+(double b) const { return *this + GPUdoubleCalcImpl((float)b); } + GPUdi() GPUdoubleCalcImpl operator-(double b) const { return *this - GPUdoubleCalcImpl((float)b); } + GPUdi() GPUdoubleCalcImpl operator*(double b) const { return *this * GPUdoubleCalcImpl((float)b); } + GPUdi() GPUdoubleCalcImpl operator/(double b) const { return *this / GPUdoubleCalcImpl((float)b); } +#endif + GPUdi() GPUdoubleCalcImpl operator+(float b) const { return *this + GPUdoubleCalcImpl(b); } + GPUdi() GPUdoubleCalcImpl operator-(float b) const { return *this - GPUdoubleCalcImpl(b); } + GPUdi() GPUdoubleCalcImpl operator*(float b) const { return *this * GPUdoubleCalcImpl(b); } + GPUdi() GPUdoubleCalcImpl operator/(float b) const { return *this / GPUdoubleCalcImpl(b); } + + GPUdi() GPUdoubleCalcImpl& operator+=(GPUdoubleCalcImpl b) { return *this = *this + b; } + GPUdi() GPUdoubleCalcImpl& operator-=(GPUdoubleCalcImpl b) { return *this = *this - b; } + GPUdi() GPUdoubleCalcImpl& operator*=(GPUdoubleCalcImpl b) { return *this = *this * b; } + GPUdi() GPUdoubleCalcImpl& operator/=(GPUdoubleCalcImpl b) { return *this = *this / b; } + + private: + GPUdi() static GPUdoubleCalcImpl twoSum(float a, float b) + { + const float s = a + b, bb = s - a; + return GPUdoubleCalcImpl(s, (a - (s - bb)) + (b - bb)); + } + GPUdi() static GPUdoubleCalcImpl quickTwoSum(float a, float b) + { + const float s = a + b; + return GPUdoubleCalcImpl(s, b - (s - a)); + } + GPUdi() static GPUdoubleCalcImpl twoProd(float a, float b) + { + const float p = a * b; + return GPUdoubleCalcImpl(p, __builtin_fmaf(a, b, -p)); + } + float mHi, mLo; +}; + +#ifndef __METAL__ +GPUdi() GPUdoubleCalcImpl operator+(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) + b; } +GPUdi() GPUdoubleCalcImpl operator-(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) - b; } +GPUdi() GPUdoubleCalcImpl operator*(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) * b; } +GPUdi() GPUdoubleCalcImpl operator/(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) / b; } +#endif +GPUdi() GPUdoubleCalcImpl operator+(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) + b; } +GPUdi() GPUdoubleCalcImpl operator-(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) - b; } +GPUdi() GPUdoubleCalcImpl operator*(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) * b; } +GPUdi() GPUdoubleCalcImpl operator/(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) / b; } + +// GPUCA_FORCE_DOUBLECALC lets a host test exercise the Metal representation and +// compare it against the double one. +#if defined(__METAL__) || defined(GPUCA_FORCE_DOUBLECALC) +typedef GPUdoubleCalcImpl GPUdoubleCalc; +#elif defined(GPUCA_FORCE_FLOATCALC) // for the host test only, to show what plain float would cost +typedef float GPUdoubleCalc; +#else +typedef double GPUdoubleCalc; +#endif + } // namespace o2::gpu #endif // GPUCOMMONDOUBLE_H From 88e8ebc2b3872271737c821c99bbe6848d0b08cf Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 20/41] GPU: take the work-item indices from the kernel parameters Metal has no ambient work-item builtins: the thread and threadgroup positions exist only as attributes on the kernel entry point, so get_global_id() and its siblings cannot be expressed in a device function the way CUDA's threadIdx or OpenCL's get_global_id() can. Every one of these call sites is inside a function that already receives nBlocks, nThreads, iBlock and iThread, or is one call away from one, so they now use those directly. The substitution is exact on every backend: CUDA, HIP and OpenCL pass precisely these values into Thread(), and the CPU backend passes nThreads = 1 and iThread = 0, which is what the CPU definitions of the macros already assumed. Four helpers had no index in scope and gain one parameter: sortInBlock, GPUTPCCFClusterizer::buildCluster, GPUTPCCFNoiseSuppression::findMinimaAndPeaks and GPUTPCCFPeakFinder::isPeak. GPUCA_SHARED_CACHE and GPUCA_TBB_KERNEL_LOOP likewise take the indices as arguments rather than capturing them from the expansion context. The get_*() macros remain for the kernel entry point in GPUReconstructionKernelMacros.h, which is the one place where Metal does provide them. --- GPU/Common/GPUCommonAlgorithm.h | 14 +++++----- GPU/Common/test/testGPUsortCUDA.cu | 4 +-- GPU/GPUTracking/Base/GPUGeneralKernels.cxx | 8 +++--- .../Base/GPUReconstructionThreading.h | 28 +++++++++---------- .../Base/hip/test/testGPUsortHIP.hip | 4 +-- .../GPUTPCCompressionKernels.cxx | 14 +++++----- .../GPUTPCDecompressionKernels.cxx | 10 +++---- GPU/GPUTracking/Definitions/GPUDef.h | 12 ++++---- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 +-- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 4 +-- GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx | 6 ++-- .../Refit/GPUTrackingRefitKernel.cxx | 2 +- .../GPUTPCExtrapolationTracking.cxx | 4 +-- .../GPUTPCTrackletConstructor.cxx | 6 ++-- .../GPUTPCCFChargeMapFiller.cxx | 8 +++--- .../GPUTPCCFCheckPadBaseline.cxx | 2 +- .../TPCClusterFinder/GPUTPCCFClusterizer.cxx | 2 +- .../TPCClusterFinder/GPUTPCCFClusterizer.h | 2 +- .../TPCClusterFinder/GPUTPCCFClusterizer.inc | 6 ++-- .../TPCClusterFinder/GPUTPCCFDecodeZS.cxx | 4 +-- .../GPUTPCCFDeconvolution.cxx | 6 ++-- .../GPUTPCCFMCLabelFlattener.cxx | 4 +-- .../GPUTPCCFNoiseSuppression.cxx | 12 ++++---- .../GPUTPCCFNoiseSuppression.h | 2 +- .../TPCClusterFinder/GPUTPCCFPeakFinder.cxx | 9 +++--- .../TPCClusterFinder/GPUTPCCFPeakFinder.h | 2 +- .../GPUTPCCFStreamCompaction.cxx | 12 ++++---- .../GPUTPCNNClusterizerKernels.cxx | 22 +++++++++------ .../TRDTracking/GPUTRDTrackerKernels.cxx | 4 +-- 29 files changed, 110 insertions(+), 107 deletions(-) diff --git a/GPU/Common/GPUCommonAlgorithm.h b/GPU/Common/GPUCommonAlgorithm.h index 91d5d96890dcc..24d025cebfa56 100644 --- a/GPU/Common/GPUCommonAlgorithm.h +++ b/GPU/Common/GPUCommonAlgorithm.h @@ -32,13 +32,13 @@ class GPUCommonAlgorithm template GPUd() static void sort(T* begin, T* end); template - GPUd() static void sortInBlock(T* begin, T* end); + GPUd() static void sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end); template GPUd() static void sortDeviceDynamic(T* begin, T* end); template GPUd() static void sort(T* begin, T* end, const S& comp); template - GPUd() static void sortInBlock(T* begin, T* end, const S& comp); + GPUd() static void sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end, const S& comp); template GPUd() static void sortDeviceDynamic(T* begin, T* end, const S& comp); #if !defined(__OPENCL__) && !defined(__METAL__) // auto parameters are C++20; both are C++17 @@ -268,29 +268,29 @@ GPUdi() void GPUCommonAlgorithm::sort(T* begin, T* end, const S& comp) } template -GPUdi() void GPUCommonAlgorithm::sortInBlock(T* begin, T* end) +GPUdi() void GPUCommonAlgorithm::sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end) { #ifndef GPUCA_GPUCODE GPUCommonAlgorithm::sort(begin, end); #else - GPUCommonAlgorithm::sortInBlock(begin, end, [](auto&& x, auto&& y) { return x < y; }); + GPUCommonAlgorithm::sortInBlock(nThreads, iThread, begin, end, [](auto&& x, auto&& y) { return x < y; }); #endif } template -GPUdi() void GPUCommonAlgorithm::sortInBlock(T* begin, T* end, const S& comp) +GPUdi() void GPUCommonAlgorithm::sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end, const S& comp) { #ifndef GPUCA_GPUCODE GPUCommonAlgorithm::sort(begin, end, comp); #elif defined(GPUCA_DETERMINISTIC_MODE) // Not using GPUCA_DETERMINISTIC_CODE, which is enforced in TPC compression - if (get_local_id(0) == 0) { + if (iThread == 0) { GPUCommonAlgorithm::sort(begin, end, comp); } GPUbarrier(); #else int32_t n = end - begin; for (int32_t i = 0; i < n; i++) { - for (int32_t tIdx = get_local_id(0); tIdx < n; tIdx += get_local_size(0)) { + for (int32_t tIdx = iThread; tIdx < n; tIdx += nThreads) { int32_t offset = i % 2; int32_t curPos = 2 * tIdx + offset; int32_t nextPos = curPos + 1; diff --git a/GPU/Common/test/testGPUsortCUDA.cu b/GPU/Common/test/testGPUsortCUDA.cu index b19235f9e8c6b..95464dcc96c0d 100644 --- a/GPU/Common/test/testGPUsortCUDA.cu +++ b/GPU/Common/test/testGPUsortCUDA.cu @@ -96,12 +96,12 @@ __global__ void sortInThreadWithOperator(float* data, size_t dataLength) __global__ void sortInBlock(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength); } __global__ void sortInBlockWithOperator(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength, [](float a, float b) { return a < b; }); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength, [](float a, float b) { return a < b; }); } /////////////////////////////////////////////////////////////// diff --git a/GPU/GPUTracking/Base/GPUGeneralKernels.cxx b/GPU/GPUTracking/Base/GPUGeneralKernels.cxx index e1a3ce69dd8df..d27b778701ea1 100644 --- a/GPU/GPUTracking/Base/GPUGeneralKernels.cxx +++ b/GPU/GPUTracking/Base/GPUGeneralKernels.cxx @@ -19,12 +19,12 @@ using namespace o2::gpu; template <> GPUdii() void GPUMemClean16::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() processors, GPUglobalref() void* ptr, uint64_t size) { - const uint64_t stride = get_global_size(0); + const uint64_t stride = (nBlocks * nThreads); int4 i0; i0.x = i0.y = i0.z = i0.w = 0; int4* ptra = (int4*)ptr; uint64_t len = (size + sizeof(int4) - 1) / sizeof(int4); - for (uint64_t i = get_global_id(0); i < len; i += stride) { + for (uint64_t i = (iBlock * nThreads + iThread); i < len; i += stride) { ptra[i] = i0; } } @@ -32,8 +32,8 @@ GPUdii() void GPUMemClean16::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_ template <> GPUdii() void GPUitoa::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() processors, GPUglobalref() int32_t* ptr, uint64_t size) { - const uint64_t stride = get_global_size(0); - for (uint64_t i = get_global_id(0); i < size; i += stride) { + const uint64_t stride = (nBlocks * nThreads); + for (uint64_t i = (iBlock * nThreads + iThread); i < size; i += stride) { ptr[i] = i; } } diff --git a/GPU/GPUTracking/Base/GPUReconstructionThreading.h b/GPU/GPUTracking/Base/GPUReconstructionThreading.h index 374c7545e65da..f03ac3c06102a 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionThreading.h +++ b/GPU/GPUTracking/Base/GPUReconstructionThreading.h @@ -35,25 +35,25 @@ struct GPUReconstructionThreading { #endif -#define GPUCA_TBB_KERNEL_LOOP_HOST(rec, vartype, varname, iEnd, code) \ - for (vartype varname = get_global_id(0); varname < iEnd; varname += get_global_size(0)) { \ - code \ +#define GPUCA_TBB_KERNEL_LOOP_HOST(rec, nBlocks, nThreads, iBlock, iThread, vartype, varname, iEnd, code) \ + for (vartype varname = (iBlock) * (nThreads) + (iThread); varname < iEnd; varname += (nBlocks) * (nThreads)) { \ + code \ } #ifdef GPUCA_GPUCODE #define GPUCA_TBB_KERNEL_LOOP GPUCA_TBB_KERNEL_LOOP_HOST #else -#define GPUCA_TBB_KERNEL_LOOP(rec, vartype, varname, iEnd, code) \ - if (!rec.GetProcessingSettings().inKernelParallel) { \ - rec.mThreading->activeThreads->execute([&] { \ - tbb::parallel_for(tbb::blocked_range(get_global_id(0), iEnd, get_global_size(0)), [&](const tbb::blocked_range& _r_internal) { \ - for (vartype varname = _r_internal.begin(); varname < _r_internal.end(); varname += get_global_size(0)) { \ - code \ - } \ - }); \ - }); \ - } else { \ - GPUCA_TBB_KERNEL_LOOP_HOST(rec, vartype, varname, iEnd, code) \ +#define GPUCA_TBB_KERNEL_LOOP(rec, nBlocks, nThreads, iBlock, iThread, vartype, varname, iEnd, code) \ + if (!rec.GetProcessingSettings().inKernelParallel) { \ + rec.mThreading->activeThreads->execute([&] { \ + tbb::parallel_for(tbb::blocked_range((iBlock) * (nThreads) + (iThread), iEnd, (nBlocks) * (nThreads)), [&](const tbb::blocked_range& _r_internal) { \ + for (vartype varname = _r_internal.begin(); varname < _r_internal.end(); varname += (nBlocks) * (nThreads)) { \ + code \ + } \ + }); \ + }); \ + } else { \ + GPUCA_TBB_KERNEL_LOOP_HOST(rec, nBlocks, nThreads, iBlock, iThread, vartype, varname, iEnd, code) \ } #endif diff --git a/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip b/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip index ed13124ef65df..5758faaebebda 100644 --- a/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip +++ b/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip @@ -104,12 +104,12 @@ __global__ void sortInThreadWithOperator(float* data, size_t dataLength) __global__ void sortInBlock(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength); } __global__ void sortInBlockWithOperator(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength, [](float a, float b) { return a < b; }); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength, [](float a, float b) { return a < b; }); } /////////////////////////////////////////////////////////////// diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index b499ea10e679b..7e74d209cbd68 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -33,7 +33,7 @@ GPUdii() void GPUTPCCompressionKernels::Thread(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); #else // GPUCA_DETERMINISTIC_MODE if (param.rec.tpc.compressionSortOrder == GPUSettings::SortZPadTime) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } else if (param.rec.tpc.compressionSortOrder == GPUSettings::SortZTimePad) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } else if (param.rec.tpc.compressionSortOrder == GPUSettings::SortPad) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } else if (param.rec.tpc.compressionSortOrder == GPUSettings::SortTime) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } #endif // GPUCA_DETERMINISTIC_MODE GPUbarrier(); } - for (uint32_t j = get_local_id(0); j < count; j += get_local_size(0)) { + for (uint32_t j = iThread; j < count; j += nThreads) { int32_t outidx = idOffsetOut + totalCount + j; const ClusterNative& GPUrestrict() orgCl = clusters -> clusters[iSector][iRow][sortBuffer[j]]; diff --git a/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx index 0d2140c32e4a9..f245e3b53d2bd 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx @@ -31,7 +31,7 @@ GPUdii() void GPUTPCDecompressionKernels::ThreadnClusters[sector][row]; k++) { @@ -125,7 +125,7 @@ GPUdii() void GPUTPCDecompressionUtilKernels::ThreadclusterOffset[sector][row]; diff --git a/GPU/GPUTracking/Definitions/GPUDef.h b/GPU/GPUTracking/Definitions/GPUDef.h index 5956bf99d77d0..b5b538e96a833 100644 --- a/GPU/GPUTracking/Definitions/GPUDef.h +++ b/GPU/GPUTracking/Definitions/GPUDef.h @@ -47,19 +47,19 @@ #ifdef GPUCA_GPUCODE #define GPUCA_MAKE_SHARED_REF(vartype, varname, varglobal, varshared) const GPUsharedref() vartype& __restrict__ varname = varshared; #define GPUCA_SHARED_STORAGE(storage) storage - #define GPUCA_SHARED_CACHE(target, src, size) \ + #define GPUCA_SHARED_CACHE(nThreads, iThread, target, src, size) \ static_assert((size) % sizeof(int32_t) == 0, "Invalid shared cache size"); \ - for (uint32_t i_shared_cache = get_local_id(0); i_shared_cache < (size) / sizeof(int32_t); i_shared_cache += get_local_size(0)) { \ + for (uint32_t i_shared_cache = (iThread); i_shared_cache < (size) / sizeof(int32_t); i_shared_cache += (nThreads)) { \ reinterpret_cast(target)[i_shared_cache] = reinterpret_cast(src)[i_shared_cache]; \ } - #define GPUCA_SHARED_CACHE_REF(target, src, size, reftype, ref) \ - GPUCA_SHARED_CACHE(target, src, size) \ + #define GPUCA_SHARED_CACHE_REF(nThreads, iThread, target, src, size, reftype, ref) \ + GPUCA_SHARED_CACHE(nThreads, iThread, target, src, size) \ GPUsharedref() const reftype* __restrict__ ref = (target) #else #define GPUCA_MAKE_SHARED_REF(vartype, varname, varglobal, varshared) const GPUglobalref() vartype & __restrict__ varname = varglobal; #define GPUCA_SHARED_STORAGE(storage) - #define GPUCA_SHARED_CACHE(target, src, size) - #define GPUCA_SHARED_CACHE_REF(target, src, size, reftype, ref) GPUglobalref() const reftype* __restrict__ ref = src + #define GPUCA_SHARED_CACHE(nThreads, iThread, target, src, size) + #define GPUCA_SHARED_CACHE_REF(nThreads, iThread, target, src, size, reftype, ref) GPUglobalref() const reftype* __restrict__ ref = src #endif #endif //GPUTPCDEF_H diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index e89081aa40350..93308772dfb96 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1934,7 +1934,7 @@ GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t GPUd() void GPUTPCGMMerger::MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { const float lowPtThresh = Param().rec.tpc.rejectQPtB5 * 1.1f; // Might need to merge tracks above the threshold with parts below the rejection threshold - for (uint32_t i = get_global_id(0); i < mMemory->nMergedTracks; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < mMemory->nMergedTracks; i += (nBlocks * nThreads)) { const auto& trk = mMergedTracks[i]; const auto& p = trk.GetParam(); const float qptabs = CAMath::Abs(p.GetQPt()); @@ -2003,7 +2003,7 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, } #endif - for (uint32_t i = get_global_id(0); i < mMemory->nLooperMatchCandidates; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < mMemory->nLooperMatchCandidates; i += (nBlocks * nThreads)) { for (uint32_t j = i + 1; j < mMemory->nLooperMatchCandidates; j++) { // int32_t bs = 0; assert(CAMath::Abs(candidates[i].refz) <= CAMath::Abs(candidates[j].refz)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 2a111b8ce89af..ea0620d3ea4c1 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -22,7 +22,7 @@ template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) { const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), nBlocks, nThreads, iBlock, iThread, int32_t, ii, iEnd, { const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : mode ? merger.TrackOrderProcess()[ii] : ii; GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, mode == -1); }); @@ -31,7 +31,7 @@ GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThread template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), uint32_t, i, merger.Memory()->nLoopData, { + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), nBlocks, nThreads, iBlock, iThread, uint32_t, i, merger.Memory()->nLoopData, { GPUTPCGMTrackParam::PropagateLooper(&merger, i); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index d7c8eb9c44aab..39690ae078599 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -58,7 +58,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlock GPUTPCGMMerger::tmpSort* GPUrestrict() trackSort = merger.TrackSortO2(); uint2* GPUrestrict() tmpData = merger.ClusRefTmp(); - for (uint32_t i = get_global_id(0); i < nTracks; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < nTracks; i += (nBlocks * nThreads)) { if (!tracks[i].OK()) { continue; } @@ -120,7 +120,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks uint2* GPUrestrict() tmpData = merger.ClusRefTmp(); float const SNPThresh = 0.999990f; - for (int32_t iTmp = get_global_id(0); iTmp < nTracks; iTmp += get_global_size(0)) { + for (int32_t iTmp = (iBlock * nThreads + iThread); iTmp < nTracks; iTmp += (nBlocks * nThreads)) { TrackTPC oTrack; const int32_t i = trackSort[iTmp].x; const auto& track = tracks[i]; @@ -288,7 +288,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks, in auto labelAssigner = GPUTPCTrkLbl(clusters->clustersMCTruth, 0.1f); uint32_t* clusRefs = merger.OutputClusRefsTPCO2(); - for (uint32_t i = get_global_id(0); i < merger.NOutputTracksTPCO2(); i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < merger.NOutputTracksTPCO2(); i += (nBlocks * nThreads)) { labelAssigner.reset(); const auto& trk = merger.OutputTracksTPCO2()[i]; for (int32_t j = 0; j < trk.getNClusters(); j++) { diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx index f99544f239bb7..bc67075b4f820 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx @@ -22,7 +22,7 @@ template GPUdii() void GPUTrackingRefitKernel::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() processors) { auto& refit = processors.trackingRefit; - for (uint32_t i = get_global_id(0); i < processors.ioPtrs.nMergedTracks; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < processors.ioPtrs.nMergedTracks; i += (nBlocks * nThreads)) { if (refit.mPTracks[i].OK()) { GPUTPCGMMergedTrack trk = refit.mPTracks[i]; int32_t retval; diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx index 784b60baec3d6..6280d6e7afaf2 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx @@ -160,7 +160,7 @@ GPUd() void GPUTPCExtrapolationTracking::PerformExtrapolationTracking(int32_t nB template <> GPUdii() void GPUTPCExtrapolationTracking::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() tracker) { - GPUCA_SHARED_CACHE(&smem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); + GPUCA_SHARED_CACHE(nThreads, iThread, &smem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); GPUbarrier(); if (tracker.NHitsTotal() == 0) { @@ -202,7 +202,7 @@ GPUd() void GPUTPCExtrapolationTracking::ExtrapolationTrackingSectorLeftRight(ui template <> GPUdii() void GPUTPCExtrapolationTrackingCopyNumbers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() tracker, int32_t n) { - for (int32_t i = get_global_id(0); i < n; i += get_global_size(0)) { + for (int32_t i = (iBlock * nThreads + iThread); i < n; i += (nBlocks * nThreads)) { GPUconstantref() GPUTPCTracker& GPUrestrict() trk = (&tracker)[i]; trk.CommonMemory()->nLocalTracks = trk.CommonMemory()->nTracks; trk.CommonMemory()->nLocalTrackHits = trk.CommonMemory()->nTrackHits; diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 33a3264a87ab3..ba31c30840631 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -479,14 +479,14 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::DoTracklet(GPUconstantref() GPUTPCT template <> GPUdii() void GPUTPCTrackletConstructor::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& sMem, processorType& GPUrestrict() tracker) { - if (get_local_id(0) == 0) { + if (iThread == 0) { sMem.mNStartHits = *tracker.NStartHits(); } - GPUCA_SHARED_CACHE(&sMem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); + GPUCA_SHARED_CACHE(nThreads, iThread, &sMem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); GPUbarrier(); GPUTPCThreadMemory rMem; - for (rMem.mISH = get_global_id(0); rMem.mISH < sMem.mNStartHits; rMem.mISH += get_global_size(0)) { + for (rMem.mISH = (iBlock * nThreads + iThread); rMem.mISH < sMem.mNStartHits; rMem.mISH += (nBlocks * nThreads)) { rMem.mGo = 1; DoTracklet(tracker, sMem, rMem); } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx index 752c85634f928..ed6d97dfd4e3c 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx @@ -24,7 +24,7 @@ template <> GPUdii() void GPUTPCCFChargeMapFiller::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D indexMap(clusterer.mPindexMap); - fillIndexMapImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer.mPmemory->fragment, clusterer.mPdigits, indexMap, clusterer.mPmemory->counters.nDigitsInFragment); + fillIndexMapImpl(nBlocks, nThreads, iBlock, iThread, clusterer.mPmemory->fragment, clusterer.mPdigits, indexMap, clusterer.mPmemory->counters.nDigitsInFragment); } GPUd() void GPUTPCCFChargeMapFiller::fillIndexMapImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, @@ -33,7 +33,7 @@ GPUd() void GPUTPCCFChargeMapFiller::fillIndexMapImpl(int32_t nBlocks, int32_t n CfArray2D& indexMap, size_t maxDigit) { - size_t idx = get_global_id(0); + size_t idx = (iBlock * nThreads + iThread); if (idx >= maxDigit) { return; } @@ -47,7 +47,7 @@ template <> GPUdii() void GPUTPCCFChargeMapFiller::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); - fillFromDigitsImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer, clusterer.mPmemory->fragment, clusterer.mPmemory->counters.nPositions, clusterer.mPdigits, clusterer.mPpositions, chargeMap); + fillFromDigitsImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->fragment, clusterer.mPmemory->counters.nPositions, clusterer.mPdigits, clusterer.mPpositions, chargeMap); } GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, const CfFragment& fragment, size_t digitNum, @@ -55,7 +55,7 @@ GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t CfChargePos* positions, CfArray2D& chargeMap) { - size_t idx = get_global_id(0); + size_t idx = (iBlock * nThreads + iThread); if (idx >= digitNum) { return; } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx index 7470f86877490..4c48ed3e097f4 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx @@ -668,7 +668,7 @@ GPUd() void GPUTPCCFHIPTailConnector::Thread<0>(int32_t nBlocks, int32_t nThread #ifdef GPUCA_DETERMINISTIC_MODE // Races in tail comparisons and atomic swap can lead to slightly different clusters. // So need a sequential fallback for deterministic mode - GPUCommonAlgorithm::sortInBlock(tails + 1, tails + nTails + 1, [](auto&& t1, auto&& t2) { + GPUCommonAlgorithm::sortInBlock(nThreads, iThread, tails + 1, tails + nTails + 1, [](auto&& t1, auto&& t2) { if (t1.pad != t2.pad) { return t1.pad < t2.pad; } else if (t1.tailStart != t2.tailStart) { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx index c9a8c093153a2..62e89ef4b7ddf 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx @@ -35,5 +35,5 @@ GPUdii() void GPUTPCCFClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, tpc::ClusterNative* clusterOut = onlyMC ? nullptr : clusterer.mPclusterByRow; - GPUTPCCFClusterizer::computeClustersImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, true); + GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, true); } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h index ce673c778e42d..ab40e1c3bc2c3 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h @@ -59,7 +59,7 @@ class GPUTPCCFClusterizer : public GPUKernelTemplate static GPUd() void computeClustersImpl(int32_t, int32_t, int32_t, int32_t, processorType&, const CfFragment&, GPUSharedMemory&, const CfArray2D&, const CfChargePos*, const GPUSettingsRec&, MCLabelAccumulator*, uint32_t, uint32_t, uint32_t*, tpc::ClusterNative*, uint32_t*, int8_t); - static GPUd() void buildCluster(const GPUSettingsRec&, const CfArray2D&, CfChargePos, CfChargePos*, PackedCharge*, uint8_t*, ClusterAccumulator*, MCLabelAccumulator*); + static GPUd() void buildCluster(const GPUSettingsRec&, uint16_t, const CfArray2D&, CfChargePos, CfChargePos*, PackedCharge*, uint8_t*, ClusterAccumulator*, MCLabelAccumulator*); static GPUd() uint32_t sortIntoBuckets(processorType&, const tpc::ClusterNative&, uint32_t, uint32_t, uint32_t*, tpc::ClusterNative*); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc index ca396f8aab83e..22cbeec9e86fb 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc @@ -30,7 +30,7 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t uint32_t* clusterPosInRow, int8_t isAccepted) { - uint32_t idx = get_global_id(0); + uint32_t idx = (iBlock * nThreads + iThread); // For certain configurations dummy work items are added, so the total // number of work items is dividable by 64. @@ -43,6 +43,7 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t buildCluster( calib, + iThread, chargeMap, pos, smem.posBcast, @@ -145,6 +146,7 @@ GPUdii() void GPUTPCCFClusterizer::updateClusterOuter( GPUdii() void GPUTPCCFClusterizer::buildCluster( const GPUSettingsRec& calib, + uint16_t ll, const CfArray2D& chargeMap, CfChargePos pos, CfChargePos* posBcast, @@ -153,8 +155,6 @@ GPUdii() void GPUTPCCFClusterizer::buildCluster( ClusterAccumulator* myCluster, MCLabelAccumulator* labelAcc) { - uint16_t ll = get_local_id(0); - posBcast[ll] = pos; GPUbarrier(); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index a1c4a3dc4aadd..119b25b97d8ef 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -80,7 +80,7 @@ GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUShared for (uint32_t j = minJ; j < maxJ; j++) { #endif const uint32_t* pageSrc = (const uint32_t*)(((const uint8_t*)zs.zsPtr[endpoint][i]) + j * TPCZSHDR::TPC_ZS_PAGE_SIZE); - GPUCA_SHARED_CACHE_REF(&s.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); + GPUCA_SHARED_CACHE_REF(nThreads, iThread, &s.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); GPUbarrier(); const uint8_t* page = (const uint8_t*)pageCache; const o2::header::RAWDataHeader* rdh = (const o2::header::RAWDataHeader*)page; @@ -393,7 +393,7 @@ GPUd() void GPUTPCCFDecodeZSLinkBase::Decode(int32_t nBlocks, int32_t nThreads, #endif const uint32_t* pageSrc = (const uint32_t*)(((const uint8_t*)zs.zsPtr[endpoint][i]) + j * TPCZSHDR::TPC_ZS_PAGE_SIZE); // Cache zs page in shared memory. Curiously this actually degrades performance... - // GPUCA_SHARED_CACHE_REF(&smem.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); + // GPUCA_SHARED_CACHE_REF(nThreads, iThread, &smem.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); // GPUbarrier(); // const uint8_t* page = (const uint8_t*)pageCache; const uint8_t* page = (const uint8_t*)pageSrc; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx index d6b8703a9b35d..b234094648eee 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx @@ -26,7 +26,7 @@ GPUdii() void GPUTPCCFDeconvolution::Thread<0>(int32_t nBlocks, int32_t nThreads { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CfArray2D isPeakMap(clusterer.mPpeakMap); - GPUTPCCFDeconvolution::deconvolutionImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), smem, isPeakMap, chargeMap, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, overwriteCharge); + GPUTPCCFDeconvolution::deconvolutionImpl(nBlocks, nThreads, iBlock, iThread, smem, isPeakMap, chargeMap, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, overwriteCharge); } GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, @@ -36,7 +36,7 @@ GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t const uint32_t digitnum, uint8_t overwriteCharge) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); bool iamDummy = (idx >= digitnum); idx = iamDummy ? digitnum - 1 : idx; @@ -47,7 +47,7 @@ GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t int8_t peakCount = (iamPeak) ? 1 : 0; - uint16_t ll = get_local_id(0); + uint16_t ll = iThread; uint16_t partId = ll; uint16_t in3x3 = 0; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx index 8b4f28f517782..a9e4fdbbca064 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx @@ -46,7 +46,7 @@ template <> GPUd() void GPUTPCCFMCLabelFlattener::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory&, processorType& clusterer) { #if !defined(GPUCA_GPUCODE) - const Row row = get_global_id(0); + const Row row = (iBlock * nThreads + iThread); const size_t clusterInRow = clusterer.mPclusterInRow[row]; auto& labels = clusterer.mPlabelsByRow[row].data; @@ -66,7 +66,7 @@ template <> GPUd() void GPUTPCCFMCLabelFlattener::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory&, processorType& clusterer, GPUTPCLinearLabels* out) { #if !defined(GPUCA_GPUCODE) - uint32_t row = get_global_id(0); + uint32_t row = (iBlock * nThreads + iThread); uint32_t headerOffset = clusterer.mPlabelsHeaderGlobalOffset; uint32_t dataOffset = clusterer.mPlabelsDataGlobalOffset; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx index 4dfa50d9439e4..336cfb6d801a9 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx @@ -26,14 +26,14 @@ GPUdii() void GPUTPCCFNoiseSuppression::Thread chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CfArray2D isPeakMap(clusterer.mPpeakMap); - noiseSuppressionImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), smem, clusterer.Param().rec, chargeMap, isPeakMap, clusterer.mPpeakPositions, clusterer.mPmemory->counters.nPeaks, clusterer.mPisPeak); + noiseSuppressionImpl(nBlocks, nThreads, iBlock, iThread, smem, clusterer.Param().rec, chargeMap, isPeakMap, clusterer.mPpeakPositions, clusterer.mPmemory->counters.nPeaks, clusterer.mPisPeak); } template <> GPUdii() void GPUTPCCFNoiseSuppression::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D isPeakMap(clusterer.mPpeakMap); - updatePeaksImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer.mPpeakPositions, clusterer.mPisPeak, clusterer.mPmemory->counters.nPeaks, isPeakMap); + updatePeaksImpl(nBlocks, nThreads, iBlock, iThread, clusterer.mPpeakPositions, clusterer.mPisPeak, clusterer.mPmemory->counters.nPeaks, isPeakMap); } GPUdii() void GPUTPCCFNoiseSuppression::noiseSuppressionImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, @@ -44,7 +44,7 @@ GPUdii() void GPUTPCCFNoiseSuppression::noiseSuppressionImpl(int32_t nBlocks, in const uint32_t peaknum, uint8_t* isPeakPredicate) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); CfChargePos pos = peakPositions[CAMath::Min(idx, (SizeT)(peaknum - 1))]; Charge charge = chargeMap[pos].unpack(); @@ -54,6 +54,7 @@ GPUdii() void GPUTPCCFNoiseSuppression::noiseSuppressionImpl(int32_t nBlocks, in chargeMap, peakMap, calibration, + iThread, charge, pos, smem.posBcast, @@ -80,7 +81,7 @@ GPUd() void GPUTPCCFNoiseSuppression::updatePeaksImpl(int32_t nBlocks, int32_t n const uint32_t peakNum, CfArray2D& peakMap) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); if (idx >= peakNum) { return; @@ -167,6 +168,7 @@ GPUd() void GPUTPCCFNoiseSuppression::findMinimaAndPeaks( const CfArray2D& chargeMap, const CfArray2D& peakMap, const GPUSettingsRec& calibration, + uint16_t ll, float q, const CfChargePos& pos, CfChargePos* posBcast, @@ -175,8 +177,6 @@ GPUd() void GPUTPCCFNoiseSuppression::findMinimaAndPeaks( uint64_t* bigger, uint64_t* peaks) { - uint16_t ll = get_local_id(0); - posBcast[ll] = pos; GPUbarrier(); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h index bdee75dc87732..8c251f310a1a8 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h @@ -69,7 +69,7 @@ class GPUTPCCFNoiseSuppression : public GPUKernelTemplate static GPUdi() bool keepPeak(uint64_t, uint64_t); - static GPUd() void findMinimaAndPeaks(const CfArray2D&, const CfArray2D&, const GPUSettingsRec&, float, const CfChargePos&, CfChargePos*, PackedCharge*, uint64_t*, uint64_t*, uint64_t*); + static GPUd() void findMinimaAndPeaks(const CfArray2D&, const CfArray2D&, const GPUSettingsRec&, uint16_t, float, const CfChargePos&, CfChargePos*, PackedCharge*, uint64_t*, uint64_t*, uint64_t*); }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx index 7c93435f8bef8..ff71af8838b83 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx @@ -27,11 +27,12 @@ GPUdii() void GPUTPCCFPeakFinder::Thread<0>(int32_t nBlocks, int32_t nThreads, i { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CfArray2D isPeakMap(clusterer.mPpeakMap); - findPeaksImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), smem, chargeMap, clusterer.mPpadIsNoisy, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, clusterer.Param().rec, *clusterer.GetConstantMem()->calibObjects.tpcPadGain, clusterer.mPisPeak, isPeakMap); + findPeaksImpl(nBlocks, nThreads, iBlock, iThread, smem, chargeMap, clusterer.mPpadIsNoisy, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, clusterer.Param().rec, *clusterer.GetConstantMem()->calibObjects.tpcPadGain, clusterer.mPisPeak, isPeakMap); } GPUdii() bool GPUTPCCFPeakFinder::isPeak( GPUSharedMemory& smem, + uint16_t ll, Charge q, const CfChargePos& pos, uint16_t N, @@ -40,8 +41,6 @@ GPUdii() bool GPUTPCCFPeakFinder::isPeak( CfChargePos* posBcast, PackedCharge* buf) { - uint16_t ll = get_local_id(0); - bool belowThreshold = (uint32_t)q <= calib.tpc.cfQMaxCutoff; uint16_t lookForPeaks; @@ -100,7 +99,7 @@ GPUd() void GPUTPCCFPeakFinder::findPeaksImpl(int32_t nBlocks, int32_t nThreads, uint8_t* isPeakPredicate, CfArray2D& peakMap) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); // For certain configurations dummy work items are added, so the total // number of work items is dividable by 64. @@ -111,7 +110,7 @@ GPUd() void GPUTPCCFPeakFinder::findPeaksImpl(int32_t nBlocks, int32_t nThreads, bool hasLostBaseline = pos.valid() ? padHasLostBaseline[pos.gpad] : true; charge = hasLostBaseline ? 0.f : charge; - uint8_t peak = isPeak(smem, charge, pos, SCRATCH_PAD_SEARCH_N, chargeMap, calib, smem.posBcast, smem.buf); + uint8_t peak = isPeak(smem, iThread, charge, pos, SCRATCH_PAD_SEARCH_N, chargeMap, calib, smem.posBcast, smem.buf); // Exit early if dummy. See comment above. bool iamDummy = (idx >= digitnum); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h index 0d61378d3e6f2..8cf32c1589e61 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h @@ -53,7 +53,7 @@ class GPUTPCCFPeakFinder : public GPUKernelTemplate private: static GPUd() void findPeaksImpl(int32_t, int32_t, int32_t, int32_t, GPUSharedMemory&, const CfArray2D&, const uint8_t*, const CfChargePos*, tpccf::SizeT, const GPUSettingsRec&, const TPCPadGainCalib&, uint8_t*, CfArray2D&); - static GPUd() bool isPeak(GPUSharedMemory&, tpccf::Charge, const CfChargePos&, uint16_t, const CfArray2D&, const GPUSettingsRec&, CfChargePos*, PackedCharge*); + static GPUd() bool isPeak(GPUSharedMemory&, uint16_t, tpccf::Charge, const CfChargePos&, uint16_t, const CfArray2D&, const GPUSettingsRec&, CfChargePos*, PackedCharge*); }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx index 0f2fd235dc0d0..4622638674d9a 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx @@ -30,7 +30,7 @@ GPUdii() void GPUTPCCFStreamCompaction::Thread auto* scanOffset = clusterer.GetScanBuffer(iBuf - 1); auto* scanOffsetNext = clusterer.GetScanBuffer(iBuf); - int32_t iThreadGlobal = get_global_id(0); + int32_t iThreadGlobal = (iBlock * nThreads + iThread); int32_t offsetInBlock = work_group_scan_inclusive_add((iThreadGlobal < nElems) ? scanOffset[iThreadGlobal] : 0); if (iThreadGlobal < nElems) { @@ -70,7 +70,7 @@ template <> GPUdii() void GPUTPCCFStreamCompaction::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, int32_t iBuf, int32_t nElems) { #ifdef GPUCA_GPUCODE - int32_t iThreadGlobal = get_global_id(0); + int32_t iThreadGlobal = (iBlock * nThreads + iThread); int32_t* scanOffset = clusterer.GetScanBuffer(iBuf - 1); bool inBounds = (iThreadGlobal < nElems); @@ -87,7 +87,7 @@ template <> GPUdii() void GPUTPCCFStreamCompaction::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& /*smem*/, processorType& clusterer, int32_t iBuf, uint32_t offset, int32_t nElems) { #ifdef GPUCA_GPUCODE - int32_t iThreadGlobal = get_global_id(0) + offset; + int32_t iThreadGlobal = (iBlock * nThreads + iThread) + offset; int32_t* scanOffsetPrev = clusterer.GetScanBuffer(iBuf - 1); const int32_t* scanOffset = clusterer.GetScanBuffer(iBuf); @@ -107,7 +107,7 @@ GPUdii() void GPUTPCCFStreamCompaction::Thread bufferSize) { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx index 693ee4dd78e8d..549be70fa8a1e 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx @@ -42,14 +42,14 @@ static_assert(GPUTPCNNClusterizerKernels::SCRATCH_PAD_WORK_GROUP_SIZE == GPUTPCC template <> GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CPU_ONLY(MCLabelAccumulator labelAcc(clusterer)); tpc::ClusterNative* clusterOut = clusterer.mPclusterByRow; int8_t isAccepted = (clustererNN.mNnClusterizerUseClassification ? (clustererNN.mOutputDataClass[CAMath::Min(glo_idx, (uint32_t)clusterer.mPmemory->counters.nClusters - 1)] > 0) : 1); - GPUTPCCFClusterizer::computeClustersImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, isAccepted); + GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, isAccepted); } template <> @@ -58,7 +58,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { return; } @@ -147,7 +147,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx + batchStart >= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -263,7 +263,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx + batchStart >= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -302,7 +302,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -327,6 +327,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -347,6 +348,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -498,7 +500,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -521,6 +523,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -541,6 +544,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -669,7 +673,7 @@ template <> GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint batchStart) { // Implements identical publishing logic as the heuristic clusterizer and deconvolution kernel - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx + batchStart >= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx b/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx index dea4cdbca430e..6632be71cbb75 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx +++ b/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx @@ -35,8 +35,8 @@ GPUdii() void GPUTRDTrackerKernels::Thread(int32_t nBlocks, int32_t nThreads, in } } #endif - GPUCA_TBB_KERNEL_LOOP(trdTracker->GetRec(), int32_t, i, trdTracker->NTracks(), { - trdTracker->DoTrackingThread(i, get_global_id(0)); + GPUCA_TBB_KERNEL_LOOP(trdTracker->GetRec(), nBlocks, nThreads, iBlock, iThread, int32_t, i, trdTracker->NTracks(), { + trdTracker->DoTrackingThread(i, (iBlock * nThreads + iThread)); }); } From 26b9b608e935ba2d7256e8ca51faeb2cea6b6589 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 21/41] GPU: only use GPUdoubleCalc where fast math is off Metal defaults to fast math, and fast math reassociates GPUdoubleCalcImpl's compensation terms away: measured on the full propagateTo kernel, the two-float type then costs 1.57x for exactly the accuracy of a plain float. The Metal build therefore follows what the OpenCL one already does: fast math unless GPUCA_DETERMINISTIC_MODE asks for otherwise, and GPUdoubleCalc becomes a plain float when fast math is on. This also takes back the unconditional -fno-fast-math added with the type, which is not free as claimed there: on an ALU-bound kernel it costs the plain float path a factor of two (0.055 -> 0.108 ns/propagate on an M1 Max), and that would have been paid by every Metal kernel, including the clusterer and the sector tracker, which have no double-precision intermediates at all. -fmetal-math-mode=safe would keep the compensation intact for 20% less, but it makes division and square root approximate and the GPU result then stops matching the CPU bit-for-bit (79% of covariance elements instead of 100%), which is the opposite of what deterministic mode is for. -fno-fast-math is also the exact analogue of the CUDA flags used there, --prec-div --prec-sqrt --fmad false. --- GPU/Common/GPUCommonDouble.h | 6 +++++- GPU/GPUTracking/Base/metal/CMakeLists.txt | 3 +++ dependencies/FindO2GPU.cmake | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h index a67843c354838..d89455d5f5321 100644 --- a/GPU/Common/GPUCommonDouble.h +++ b/GPU/Common/GPUCommonDouble.h @@ -180,7 +180,11 @@ GPUdi() GPUdoubleCalcImpl operator/(float a, GPUdoubleCalcImpl b) { return GPUdo // GPUCA_FORCE_DOUBLECALC lets a host test exercise the Metal representation and // compare it against the double one. -#if defined(__METAL__) || defined(GPUCA_FORCE_DOUBLECALC) +#if defined(__METAL__) && defined(__FAST_MATH__) +// Fast math reassociates the compensation terms away: the two-float type would +// then cost 1.5x for the accuracy of a plain float. +typedef float GPUdoubleCalc; +#elif defined(__METAL__) || defined(GPUCA_FORCE_DOUBLECALC) typedef GPUdoubleCalcImpl GPUdoubleCalc; #elif defined(GPUCA_FORCE_FLOATCALC) // for the host test only, to show what plain float would cost typedef float GPUdoubleCalc; diff --git a/GPU/GPUTracking/Base/metal/CMakeLists.txt b/GPU/GPUTracking/Base/metal/CMakeLists.txt index 577501f9e6c3c..9c9b7e4fd2b15 100644 --- a/GPU/GPUTracking/Base/metal/CMakeLists.txt +++ b/GPU/GPUTracking/Base/metal/CMakeLists.txt @@ -27,6 +27,9 @@ set(METAL_BIN ${CMAKE_CURRENT_BINARY_DIR}/GPUReconstructionMetalCode) # reject an unannotated pointer or `this` outright, which GPUCommonDefAPI.h # relies on for GPUgeneric() and GPUdDefault(). set(METAL_FLAGS -std=metal4.1 ${GPUCA_METAL_DENORMALS_FLAGS}) +if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_NO_FAST_MATH}) + set(METAL_FLAGS ${METAL_FLAGS} ${GPUCA_METAL_NO_FAST_MATH_FLAGS}) +endif() set(METAL_DEFINES "-D$,$-D>" "-I$,EXCLUDE,^/usr/include/?>,$-I>" -I${CMAKE_SOURCE_DIR}/Detectors/TRD/base/src diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index 0fe4e565419b1..956272c4600ee 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -177,6 +177,7 @@ endif() set(GPUCA_CXX_NO_FAST_MATH_FLAGS "-fno-fast-math -ffp-contract=off") set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--prec-div=true --prec-sqrt=true --fmad false -Xcompiler -fno-fast-math -Xcompiler -ffp-contract=off") set(GPUCA_OCL_NO_FAST_MATH_FLAGS -cl-fp32-correctly-rounded-divide-sqrt ) +set(GPUCA_METAL_NO_FAST_MATH_FLAGS "-fno-fast-math") if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_WHOLEO2}) add_definitions(-DGPUCA_DETERMINISTIC_MODE) string(APPEND CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} " ${GPUCA_CXX_NO_FAST_MATH_FLAGS}") From a5fad7b790ee43d8b5904d686b7a5a6cee90da62 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:20 +0200 Subject: [PATCH 22/41] GPU: make GPUdoubleCalcImpl cheaper and more accurate Three changes, measured on the full propagateTo kernel over 524288 tracks with strongly correlated covariances, against the CPU double result (Apple M1 Max, -fno-fast-math, median of ten runs): * mLo is no longer renormalised after each operation. The value is still mHi + mLo and nothing downstream requires |mLo| <= ulp(mHi)/2, so the results are identical to the last bit: 3.11x -> 2.22x the cost of plain float. * the mixed float forms no longer widen the float operand and run the full two-float operation. Its mLo is zero, but with reassociation disabled the compiler is not allowed to fold those terms away. * float += GPUdoubleCalcImpl now rounds once, as float += double does on the host, rather than converting to float and adding. This is what `c00 += b00 + b00 + a00` resolves to, and it is where the remaining error was: +7% cost, and the covariance error against the CPU double result drops from 1.8e-6 to 5.6e-7 of sqrt(C_ii C_jj), with 86% of elements bit-identical instead of 76%. Net: 3.11x -> 2.37x and closer to the CPU than before. For reference, plain float is at 1.2e-5 on the same metric, and the CPU double result is itself 1.1e-6 away from a cancellation-free reference, so the type is now at the noise floor of the formula it implements. Representations narrower than two floats were measured and are not viable: with four halves, limbs two to four of any value below about one fall into half subnormals, leaving fewer effective bits than a single float (worst error 0.46, 42x the cost), and fixed point Q21.42 reaches 7.4e-6 at 16x. --- GPU/Common/GPUCommonDouble.h | 105 ++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h index d89455d5f5321..6f67c1154efc2 100644 --- a/GPU/Common/GPUCommonDouble.h +++ b/GPU/Common/GPUCommonDouble.h @@ -88,12 +88,17 @@ static_assert(sizeof(GPUdoubleStore) == 8, "GPUdoubleStore must match the size o static_assert(alignof(GPUdoubleStore) == 8, "GPUdoubleStore must match the alignment of a double"); -// Compensated two-float arithmetic, for the intermediates that are deliberately -// computed in double even when the track itself is float -- the Jacobian terms in -// TrackParametrizationWithError::propagateTo and friends, where differences of -// nearly equal quantities cancel. Plain float loses up to ~1e-2 relative there; -// this representation, value = mHi + mLo, holds the error term explicitly and -// stays below 1e-9 across the same inputs. +// Compensated two-float arithmetic, value = mHi + mLo, for the intermediates that +// are deliberately computed in double even when the track itself is float -- the +// Jacobian and covariance terms in TrackParametrizationWithError::propagateTo and +// friends, where differences of nearly equal quantities cancel. +// +// Every operation here depends on the compiler not reassociating the compensation +// terms away, which is why the Metal device code is built without fast math; when +// it is built with fast math, GPUdoubleCalc below is a plain float instead. +// +// mLo is not renormalised after each operation: the value is still mHi + mLo and +// nothing downstream requires |mLo| <= ulp(mHi)/2. // // Defined for every backend so it can be tested on the host, but only Metal uses // it: everywhere else GPUdoubleCalc is a plain double. @@ -110,38 +115,52 @@ class GPUdoubleCalcImpl GPUdi() GPUdoubleCalcImpl operator-() const { return GPUdoubleCalcImpl(-mHi, -mLo); } GPUdi() GPUdoubleCalcImpl operator+(GPUdoubleCalcImpl b) const { - GPUdoubleCalcImpl s = twoSum(mHi, b.mHi); - s.mLo += mLo + b.mLo; - return quickTwoSum(s.mHi, s.mLo); + const float s = mHi + b.mHi, bb = s - mHi; + return GPUdoubleCalcImpl(s, ((mHi - (s - bb)) + (b.mHi - bb)) + (mLo + b.mLo)); } GPUdi() GPUdoubleCalcImpl operator-(GPUdoubleCalcImpl b) const { return *this + (-b); } GPUdi() GPUdoubleCalcImpl operator*(GPUdoubleCalcImpl b) const { - GPUdoubleCalcImpl p = twoProd(mHi, b.mHi); - p.mLo += mHi * b.mLo + mLo * b.mHi; - return quickTwoSum(p.mHi, p.mLo); + const float p = mHi * b.mHi; + return GPUdoubleCalcImpl(p, __builtin_fmaf(mHi, b.mHi, -p) + (mHi * b.mLo + mLo * b.mHi)); } GPUdi() GPUdoubleCalcImpl operator/(GPUdoubleCalcImpl b) const { - const float q1 = mHi / b.mHi; - const GPUdoubleCalcImpl d = *this - GPUdoubleCalcImpl(q1) * b; - return quickTwoSum(q1, (d.mHi + d.mLo) / b.mHi); + const float q = mHi / b.mHi; + const float r = (__builtin_fmaf(-q, b.mHi, mHi) + mLo) - q * b.mLo; + return GPUdoubleCalcImpl(q, r / b.mHi); } + // exact matches for the mixed forms, so `a * someFloat` does not sit ambiguously - // between converting the float up and converting *this down + // between converting the float up and converting *this down. They also skip the + // mLo terms that are zero for a float operand, which no reassociation is allowed + // to fold away here. + GPUdi() GPUdoubleCalcImpl operator+(float b) const + { + const float s = mHi + b, bb = s - mHi; + return GPUdoubleCalcImpl(s, ((mHi - (s - bb)) + (b - bb)) + mLo); + } + GPUdi() GPUdoubleCalcImpl operator-(float b) const { return *this + (-b); } + GPUdi() GPUdoubleCalcImpl operator*(float b) const + { + const float p = mHi * b; + return GPUdoubleCalcImpl(p, __builtin_fmaf(mHi, b, -p) + mLo * b); + } + GPUdi() GPUdoubleCalcImpl operator/(float b) const + { + const float q = mHi / b; + const float r = __builtin_fmaf(-q, b, mHi) + mLo; + return GPUdoubleCalcImpl(q, r / b); + } // MSL has no double, so on Metal a literal like `1.` is already float and only // the float forms are ever selected. The double forms exist so the type can be // compiled and tested on the host, where such literals really are double. #ifndef __METAL__ - GPUdi() GPUdoubleCalcImpl operator+(double b) const { return *this + GPUdoubleCalcImpl((float)b); } - GPUdi() GPUdoubleCalcImpl operator-(double b) const { return *this - GPUdoubleCalcImpl((float)b); } - GPUdi() GPUdoubleCalcImpl operator*(double b) const { return *this * GPUdoubleCalcImpl((float)b); } - GPUdi() GPUdoubleCalcImpl operator/(double b) const { return *this / GPUdoubleCalcImpl((float)b); } + GPUdi() GPUdoubleCalcImpl operator+(double b) const { return *this + (float)b; } + GPUdi() GPUdoubleCalcImpl operator-(double b) const { return *this - (float)b; } + GPUdi() GPUdoubleCalcImpl operator*(double b) const { return *this * (float)b; } + GPUdi() GPUdoubleCalcImpl operator/(double b) const { return *this / (float)b; } #endif - GPUdi() GPUdoubleCalcImpl operator+(float b) const { return *this + GPUdoubleCalcImpl(b); } - GPUdi() GPUdoubleCalcImpl operator-(float b) const { return *this - GPUdoubleCalcImpl(b); } - GPUdi() GPUdoubleCalcImpl operator*(float b) const { return *this * GPUdoubleCalcImpl(b); } - GPUdi() GPUdoubleCalcImpl operator/(float b) const { return *this / GPUdoubleCalcImpl(b); } GPUdi() GPUdoubleCalcImpl& operator+=(GPUdoubleCalcImpl b) { return *this = *this + b; } GPUdi() GPUdoubleCalcImpl& operator-=(GPUdoubleCalcImpl b) { return *this = *this - b; } @@ -149,35 +168,31 @@ class GPUdoubleCalcImpl GPUdi() GPUdoubleCalcImpl& operator/=(GPUdoubleCalcImpl b) { return *this = *this / b; } private: - GPUdi() static GPUdoubleCalcImpl twoSum(float a, float b) - { - const float s = a + b, bb = s - a; - return GPUdoubleCalcImpl(s, (a - (s - bb)) + (b - bb)); - } - GPUdi() static GPUdoubleCalcImpl quickTwoSum(float a, float b) - { - const float s = a + b; - return GPUdoubleCalcImpl(s, b - (s - a)); - } - GPUdi() static GPUdoubleCalcImpl twoProd(float a, float b) - { - const float p = a * b; - return GPUdoubleCalcImpl(p, __builtin_fmaf(a, b, -p)); - } float mHi, mLo; }; #ifndef __METAL__ -GPUdi() GPUdoubleCalcImpl operator+(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) + b; } -GPUdi() GPUdoubleCalcImpl operator-(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) - b; } -GPUdi() GPUdoubleCalcImpl operator*(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) * b; } +GPUdi() GPUdoubleCalcImpl operator+(double a, GPUdoubleCalcImpl b) { return b + (float)a; } +GPUdi() GPUdoubleCalcImpl operator-(double a, GPUdoubleCalcImpl b) { return (-b) + (float)a; } +GPUdi() GPUdoubleCalcImpl operator*(double a, GPUdoubleCalcImpl b) { return b * (float)a; } GPUdi() GPUdoubleCalcImpl operator/(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) / b; } #endif -GPUdi() GPUdoubleCalcImpl operator+(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) + b; } -GPUdi() GPUdoubleCalcImpl operator-(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) - b; } -GPUdi() GPUdoubleCalcImpl operator*(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) * b; } +GPUdi() GPUdoubleCalcImpl operator+(float a, GPUdoubleCalcImpl b) { return b + a; } +GPUdi() GPUdoubleCalcImpl operator-(float a, GPUdoubleCalcImpl b) { return (-b) + a; } +GPUdi() GPUdoubleCalcImpl operator*(float a, GPUdoubleCalcImpl b) { return b * a; } GPUdi() GPUdoubleCalcImpl operator/(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) / b; } +// rounds once, as `someFloat += someDouble` does on the host. MSL resolves the +// address spaces separately, and a generic reference would tie with the built-in +// float += float rather than beat it. +#ifdef __METAL__ +GPUdi() thread float& operator+=(thread float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +GPUdi() device float& operator+=(device float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +GPUdi() threadgroup float& operator+=(threadgroup float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +#else +GPUdi() float& operator+=(float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +#endif + // GPUCA_FORCE_DOUBLECALC lets a host test exercise the Metal representation and // compare it against the double one. #if defined(__METAL__) && defined(__FAST_MATH__) From 051a2ee5975c78b62a4dbbee994644f4c1bec378 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:22 +0200 Subject: [PATCH 23/41] MathUtils: extend the OpenCL sincos workaround to Metal A reference parameter carries its address space into template deduction, so sincos(T ang, T& s, T& c) called with thread-resident floats deduces T as float from the first argument and as thread float from the other two. OpenCL has the same rule, and the three-parameter overload already there for it works verbatim on Metal. --- Common/MathUtils/include/MathUtils/detail/trigonometric.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/MathUtils/include/MathUtils/detail/trigonometric.h b/Common/MathUtils/include/MathUtils/detail/trigonometric.h index 6723e5908ea0c..6e6c631f5197f 100644 --- a/Common/MathUtils/include/MathUtils/detail/trigonometric.h +++ b/Common/MathUtils/include/MathUtils/detail/trigonometric.h @@ -110,7 +110,7 @@ inline void bringToPMPiGen(T& phi) phi = toPMPiGen(phi); } -#ifdef __OPENCL__ // TODO: get rid of that stupid workaround for OpenCL template address spaces +#if defined(__OPENCL__) || defined(__METAL__) // TODO: get rid of that stupid workaround for OpenCL template address spaces template GPUhdi() void sincos(T ang, S& s, U& c) { From f249a4da570fb2f1de099d01fb698af1765a3c2d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:24 +0200 Subject: [PATCH 24/41] GPU: reach global memory through the generic address space on Metal GPUglobalref() expanded to device, which is what the OpenCL 1 port needed. It no longer matches the code: since the constant address space was dropped, the objects the kernels work on are reached through a generic `this`, so a member pointer or the address of a member is a generic pointer, and MSL will not pass one where a device pointer is wanted. MSL 4.1's generic address space spans device, threadgroup and thread, so leaving the annotation off is correct for every one of these. This mirrors the OpenCL C++ branch, where GPUglobalref() is likewise empty and GPUgeneric() carries the annotation. GPUsharedref() and GPUconstantref() stay explicit: threadgroup is still worth pinning where it is known, and constant is not reachable through a generic pointer at all. --- GPU/Common/GPUCommonDefAPI.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index f087c2bade23a..2ce0089ba8ca1 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -166,7 +166,7 @@ #define GPUnoexcept() #define GPUprivate() thread #define GPUgeneric() - #define GPUglobalref() device + #define GPUglobalref() #define GPUsharedref() threadgroup #define GPUprivateref() thread #if !defined(GPUCA_NO_CONSTANT_MEMORY) From bb585c3ac29035e32e0f66f383aa339aa3cc9d72 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:26 +0200 Subject: [PATCH 25/41] GPU: make the Metal atomics operate on plain counters GPUAtomic() expanded to metal::atomic, but the counters it guards are plain integers in structures that the host allocates and transfers, and the code reads them directly outside the atomic operations. That is exactly the CUDA and HIP situation, where GPUAtomic() is a no-op and atomicAdd() takes a plain pointer, so Metal now does the same. MSL's atomic_*_explicit do want metal::atomic, which has the same size and alignment, so the operand is cast at the one point where the operation is issued. The cast has to keep the address space, which a generic pointer does not carry into atomic_*_explicit, hence the two overloads: threadgroup stays threadgroup for the *Shared entry points, anything else resolves to device. GPUTPCCFDecodeZS's shared memory had to be declared GPUsharedref() for that to hold; without it the reference is generic by the time AtomicAddShared sees it. --- GPU/Common/GPUCommonDefAPI.h | 2 +- GPU/Common/GPUCommonMath.h | 21 ++++++++++++++----- .../TPCClusterFinder/GPUTPCCFDecodeZS.cxx | 4 ++-- .../TPCClusterFinder/GPUTPCCFDecodeZS.h | 4 ++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index 2ce0089ba8ca1..d3cdf5cf2db33 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -176,7 +176,7 @@ #define GPUdouble() float #define GPUbarrier() threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup) #define GPUbarrierWarp() simdgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup) - #define GPUAtomic(type) atomic // atomic variable type + #define GPUAtomic(type) type // atomic variable type #elif defined(__HIPCC__) //Defines for HIP #define GPUd() __device__ #define GPUdDefault() __device__ diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 3ef7d790d5bbe..5d8dd6e99bbe8 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -477,6 +477,17 @@ GPUhdi() constexpr int32_t GPUCommonMath::Abs(int32_t x) return GPUCA_CHOICE(abs(x), abs(x), abs(x)); } +#ifdef __METAL__ +// The counters these operate on are plain integers in the transferred structures, +// as they are for CUDA and HIP. MSL's atomic operations want metal::atomic, which +// has the same size and alignment; the overloads keep the address space, which a +// generic pointer would not carry into atomic_*_explicit. +template +GPUdi() threadgroup metal::atomic* GPUCommonMathMetalAtomic(threadgroup T* p) { return reinterpret_cast*>(p); } +template +GPUdi() device metal::atomic* GPUCommonMathMetalAtomic(T* p) { return (device metal::atomic*)p; } +#endif + template GPUdi() uint32_t GPUCommonMath::AtomicExchInternal(S* addr, T val) { @@ -487,7 +498,7 @@ GPUdi() uint32_t GPUCommonMath::AtomicExchInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) return ::atomicExch(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - return atomic_exchange_explicit(addr, val, memory_order_relaxed); + return atomic_exchange_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #elif defined(WITH_OPENMP) uint32_t old; __atomic_exchange(addr, &val, &old, __ATOMIC_SEQ_CST); @@ -507,7 +518,7 @@ GPUdi() bool GPUCommonMath::AtomicCASInternal(S* addr, T cmp, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) return ::atomicCAS(addr, cmp, val) == cmp; #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - return atomic_compare_exchange_weak_explicit(addr, &cmp, val, memory_order_relaxed, memory_order_relaxed); + return atomic_compare_exchange_weak_explicit(GPUCommonMathMetalAtomic(addr), &cmp, val, memory_order_relaxed, memory_order_relaxed); #elif defined(WITH_OPENMP) return __atomic_compare_exchange(addr, &cmp, &val, true, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); #else @@ -525,7 +536,7 @@ GPUdi() uint32_t GPUCommonMath::AtomicAddInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) return ::atomicAdd(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - return atomic_fetch_add_explicit(addr, val, memory_order_relaxed); + return atomic_fetch_add_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #elif defined(WITH_OPENMP) return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST) - val; #else @@ -543,7 +554,7 @@ GPUdi() void GPUCommonMath::AtomicMaxInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) ::atomicMax(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - atomic_fetch_max_explicit(addr, val, memory_order_relaxed); + atomic_fetch_max_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #else S current; while ((current = *(volatile S*)addr) < val && !AtomicCASInternal(addr, current, val)) { @@ -561,7 +572,7 @@ GPUdi() void GPUCommonMath::AtomicMinInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) ::atomicMin(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - atomic_fetch_min_explicit(addr, val, memory_order_relaxed); + atomic_fetch_min_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #else S current; while ((current = *(volatile S*)addr) > val && !AtomicCASInternal(addr, current, val)) { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index 119b25b97d8ef..aac5dbc0fc137 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -37,12 +37,12 @@ using namespace o2::tpc::constants; // =========================================================================== template <> -GPUdii() void GPUTPCCFDecodeZS::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, int32_t firstHBF, int32_t tpcTimeBinCut) +GPUdii() void GPUTPCCFDecodeZS::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& clusterer, int32_t firstHBF, int32_t tpcTimeBinCut) { GPUTPCCFDecodeZS::decode(clusterer, smem, nBlocks, nThreads, iBlock, iThread, firstHBF, tpcTimeBinCut); } -GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut) +GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUsharedref() GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut) { const uint32_t sector = clusterer.mISector; #ifdef GPUCA_GPUCODE diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h index 21d4ec0a28958..eed5e0d6f30ac 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h @@ -45,7 +45,7 @@ class GPUTPCCFDecodeZS : public GPUKernelTemplate decodeZS, }; - static GPUd() void decode(GPUTPCClusterFinder& clusterer, GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut); + static GPUd() void decode(GPUTPCClusterFinder& clusterer, GPUsharedref() GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut); typedef GPUTPCClusterFinder processorType; GPUhdi() static processorType* Processor(GPUConstantMem& processors) @@ -59,7 +59,7 @@ class GPUTPCCFDecodeZS : public GPUKernelTemplate } template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, Args... args); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& clusterer, Args... args); }; class GPUTPCCFDecodeZSLinkBase : public GPUKernelTemplate From 7d739bcfc86b3d9b7425adbcec1d412353192178 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:28 +0200 Subject: [PATCH 26/41] GPU: define CAMath::Abs for GPUdoubleCalcImpl CAMath::Abs is a template that deduces its parameter rather than taking a float, so a call on a GPUdoubleCalc intermediate selects the primary template, which is declared and never defined. Four call sites in the track propagation do exactly that. Compiling to an object hides it, so the Metal build only fails when the kernels are linked; on the host the type is a double and the question does not arise. Every other CAMath entry point takes a float and is reached through the implicit conversion, so only Abs needs the specialisation. --- GPU/Common/GPUCommonDouble.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h index 6f67c1154efc2..421515bad5e96 100644 --- a/GPU/Common/GPUCommonDouble.h +++ b/GPU/Common/GPUCommonDouble.h @@ -16,6 +16,7 @@ #define GPUCOMMONDOUBLE_H #include "GPUCommonDef.h" +#include "GPUCommonMath.h" #ifndef GPUCA_GPUCODE_DEVICE #include @@ -193,6 +194,15 @@ GPUdi() threadgroup float& operator+=(threadgroup float& a, GPUdoubleCalcImpl b) GPUdi() float& operator+=(float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } #endif +// CAMath::Abs is a template that deduces its parameter, so a call on this type +// picks the primary template, which has no definition. The rest of CAMath takes +// float and is reached through the implicit conversion. +template <> +GPUhdi() GPUdoubleCalcImpl GPUCommonMath::Abs(GPUdoubleCalcImpl x) +{ + return (float)x < 0.f ? -x : x; +} + // GPUCA_FORCE_DOUBLECALC lets a host test exercise the Metal representation and // compare it against the double one. #if defined(__METAL__) && defined(__FAST_MATH__) From 170d0cea25b04b3118464d408582246bf6dd389f Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:30 +0200 Subject: [PATCH 27/41] GPU: add an emulated binary64 and a selector for GPUdoubleCalc GPUdoubleCalc now names one of four representations, chosen by GPUCA_DOUBLECALC: hardware double, plain float, the compensated two-float type, or a full IEEE-754 binary64 in software. The default is unchanged, double everywhere except Metal and the two-float type there, so this is a no-op for every existing build. It replaces GPUCA_FORCE_DOUBLECALC and GPUCA_FORCE_FLOATCALC, which could only express two of the four. The binary64 emulation is round to nearest even with subnormals, infinities and NaNs, and correctly rounded division; NaN propagation follows the ARM64 order so an Apple host is a reference down to the payload. It is not for production: over the whole propagateTo kernel on an M1 Max it costs of the order of a hundred times plain float, against roughly two for the two-float type. It is here because it reproduces the CPU result exactly -- every covariance element bit-identical -- which turns a disagreement between Metal and the CPU into a search over code rather than over numerics. Measured on that kernel, 524288 tracks with strongly correlated covariances, median of ten runs, error as |dC_ij| / sqrt(C_ii C_jj) against the CPU double result: plain float 0.108 ns and 1.2e-5, two-float 0.256 ns and 5.6e-7, binary64 12.2 ns and zero. Only the binary64 kernel is unstable run to run, between 9.1 and 12.3 ns; the others vary by a few percent. For scale, the CPU double result is itself 1.1e-6 from a cancellation-free reference, so the two-float type is already at the noise floor of the formula. For scale in the other direction, the same propagateTo on this machine's CPU, where double is native, costs 23.5 ns in float and 27.1 ns in double, so real hardware double is a 1.15x proposition. None of this arithmetic would be needed if Metal had it. Validated against hardware double on 9M operand pairs per operation, over uniform bit patterns, near-equal exponents, the subnormal band, sparse mantissas for the exact and halfway cases, the special values and track-like magnitudes, plus 4M conversions each way: no mismatches. The arithmetic must stay out of line on Metal. Inlined into a kernel it drops the occupancy and runs two to five times slower than the call. --- .../src/TrackParametrizationWithError.cxx | 2 +- GPU/Common/GPUCommonDouble.h | 54 ++- GPU/Common/GPUCommonDoubleBinary64.h | 390 ++++++++++++++++++ 3 files changed, 433 insertions(+), 13 deletions(-) create mode 100644 GPU/Common/GPUCommonDoubleBinary64.h diff --git a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx index dabf7775ba4e8..284658bb7bb2b 100644 --- a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx +++ b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx @@ -1087,7 +1087,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2(const value auto chi2 = (d * (szz * d - sdz * z) + z * (sdd * z - d * sdz)) / det; if (chi2 < 0.) { #ifndef GPUCA_ALIGPUCODE - LOGP(warning, "Negative chi2={}, Cluster: {} {} {} Dy:{} Dz:{} | sdd:{} sdz:{} szz:{} det:{}", chi2, cov[0], cov[1], cov[2], d, z, sdd, sdz, szz, det); + LOGP(warning, "Negative chi2={}, Cluster: {} {} {} Dy:{} Dz:{} | sdd:{} sdz:{} szz:{} det:{}", double(chi2), cov[0], cov[1], cov[2], d, z, double(sdd), double(sdz), double(szz), double(det)); LOGP(warning, "Track: {}", asString()); #endif } diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h index 421515bad5e96..1241d97e90c09 100644 --- a/GPU/Common/GPUCommonDouble.h +++ b/GPU/Common/GPUCommonDouble.h @@ -17,6 +17,7 @@ #include "GPUCommonDef.h" #include "GPUCommonMath.h" +#include "GPUCommonDoubleBinary64.h" #ifndef GPUCA_GPUCODE_DEVICE #include @@ -194,27 +195,56 @@ GPUdi() threadgroup float& operator+=(threadgroup float& a, GPUdoubleCalcImpl b) GPUdi() float& operator+=(float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } #endif -// CAMath::Abs is a template that deduces its parameter, so a call on this type -// picks the primary template, which has no definition. The rest of CAMath takes -// float and is reached through the implicit conversion. +// CAMath::Abs is a template that deduces its parameter, so a call on one of these +// types picks the primary template, which has no definition. The rest of CAMath +// takes float and is reached through the implicit conversion. template <> GPUhdi() GPUdoubleCalcImpl GPUCommonMath::Abs(GPUdoubleCalcImpl x) { return (float)x < 0.f ? -x : x; } -// GPUCA_FORCE_DOUBLECALC lets a host test exercise the Metal representation and -// compare it against the double one. -#if defined(__METAL__) && defined(__FAST_MATH__) -// Fast math reassociates the compensation terms away: the two-float type would -// then cost 1.5x for the accuracy of a plain float. +template <> +GPUhdi() GPUdoubleBinary64 GPUCommonMath::Abs(GPUdoubleBinary64 x) +{ + return GPUdoubleBinary64::fromBits(x.bits() & ~GPUCA_B64_SIGN); +} + +// What GPUdoubleCalc is. Define GPUCA_DOUBLECALC to one of these to override the +// default, which is hardware double everywhere except Metal, and the compensated +// two-float type there. A host build can be set to any of them to compare the +// representations against each other; a Metal build can be set to BINARY64 to +// reproduce the CPU result bit for bit, at two orders of magnitude the cost. +#define GPUCA_DOUBLECALC_DOUBLE 1 +#define GPUCA_DOUBLECALC_FLOAT 2 +#define GPUCA_DOUBLECALC_TWOFLOAT 3 +#define GPUCA_DOUBLECALC_BINARY64 4 + +#ifndef GPUCA_DOUBLECALC + #if defined(__METAL__) && defined(__FAST_MATH__) + // Fast math reassociates the compensation terms away, so the two-float type + // would cost 1.5x for the accuracy of a plain float. + #define GPUCA_DOUBLECALC GPUCA_DOUBLECALC_FLOAT + #elif defined(__METAL__) + #define GPUCA_DOUBLECALC GPUCA_DOUBLECALC_TWOFLOAT + #else + #define GPUCA_DOUBLECALC GPUCA_DOUBLECALC_DOUBLE + #endif +#endif + +#if GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_DOUBLE + #ifdef __METAL__ + #error "MSL has no double; GPUCA_DOUBLECALC_DOUBLE cannot be selected for Metal" + #endif +typedef double GPUdoubleCalc; +#elif GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_FLOAT typedef float GPUdoubleCalc; -#elif defined(__METAL__) || defined(GPUCA_FORCE_DOUBLECALC) +#elif GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_TWOFLOAT typedef GPUdoubleCalcImpl GPUdoubleCalc; -#elif defined(GPUCA_FORCE_FLOATCALC) // for the host test only, to show what plain float would cost -typedef float GPUdoubleCalc; +#elif GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_BINARY64 +typedef GPUdoubleBinary64 GPUdoubleCalc; #else -typedef double GPUdoubleCalc; + #error "Invalid setting for GPUCA_DOUBLECALC" #endif } // namespace o2::gpu diff --git a/GPU/Common/GPUCommonDoubleBinary64.h b/GPU/Common/GPUCommonDoubleBinary64.h new file mode 100644 index 0000000000000..07ef68cd834de --- /dev/null +++ b/GPU/Common/GPUCommonDoubleBinary64.h @@ -0,0 +1,390 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUCommonDoubleBinary64.h +/// \brief IEEE-754 binary64 in software, for a device that has none + +#ifndef GPUCOMMONDOUBLEBINARY64_H +#define GPUCOMMONDOUBLEBINARY64_H + +#include "GPUCommonDef.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif + +// IEEE-754 binary64 in software, for a device that has no double at all. Round to +// nearest even only, with subnormals, infinities and NaNs; NaN propagation follows +// the ARM64 order, so an Apple host is a bit-exact reference down to the payload. +// There is no fused multiply-add, no square root and no other rounding mode. +// +// This is not meant for production: it costs of the order of a hundred times plain +// float on an M-series GPU, against roughly two for GPUdoubleCalcImpl. It is here so that a +// Metal build can be made to reproduce the CPU result exactly, which turns a +// disagreement between the two into a search over code rather than over numerics. + +namespace o2::gpu +{ + +namespace binary64_detail +{ + +#ifdef __METAL__ +typedef ulong u64; +typedef long i64; +typedef uint u32; +#else +typedef uint64_t u64; +typedef int64_t i64; +typedef uint32_t u32; +#endif + +#define GPUCA_B64_ALWAYS inline __attribute__((always_inline)) +#ifdef __METAL__ +// The arithmetic has to stay out of line: inlined into a kernel it drops the +// occupancy (maxTotalThreadsPerThreadgroup 832 -> 384) and runs two to five +// times slower than the call. +#define GPUCA_B64_OP __attribute__((noinline)) +#else +#define GPUCA_B64_OP inline +#endif + +#ifdef __METAL__ +GPUCA_B64_ALWAYS int32_t clz64(u64 x) { return (int32_t)metal::clz(x); } // clz(0) == 64 in MSL +GPUCA_B64_ALWAYS int32_t clz32(u32 x) { return (int32_t)metal::clz(x); } +GPUCA_B64_ALWAYS u64 mulhiu(u64 a, u64 b) { return metal::mulhi(a, b); } +GPUCA_B64_ALWAYS i64 mulhis(i64 a, i64 b) { return metal::mulhi(a, b); } +GPUCA_B64_ALWAYS u32 asu32(float f) { return as_type(f); } +GPUCA_B64_ALWAYS float asf32(u32 u) { return as_type(u); } +#else +GPUCA_B64_ALWAYS int32_t clz64(u64 x) { return x ? __builtin_clzll(x) : 64; } +GPUCA_B64_ALWAYS int32_t clz32(u32 x) { return x ? __builtin_clz(x) : 32; } +GPUCA_B64_ALWAYS u64 mulhiu(u64 a, u64 b) { return (u64)(((unsigned __int128)a * b) >> 64); } +GPUCA_B64_ALWAYS i64 mulhis(i64 a, i64 b) { return (i64)(((__int128)a * b) >> 64); } +GPUCA_B64_ALWAYS u32 asu32(float f) { return __builtin_bit_cast(uint32_t, f); } +GPUCA_B64_ALWAYS float asf32(u32 u) { return __builtin_bit_cast(float, u); } +#endif + +#define GPUCA_B64_SIGN 0x8000000000000000ULL +#define GPUCA_B64_FRAC 0x000fffffffffffffULL +#define GPUCA_B64_IMPL 0x0010000000000000ULL +#define GPUCA_B64_QUIET 0x0008000000000000ULL +#define GPUCA_B64_INF 0x7ff0000000000000ULL +#define GPUCA_B64_DNAN 0x7ff8000000000000ULL // ARM default NaN + +// right shift keeping a sticky bit; any count >= 0, no shift count ever reaches 64 +GPUCA_B64_ALWAYS u64 shrJam(u64 m, int32_t s) +{ + const int32_t sc = s > 63 ? 63 : s; + const u64 r = m >> sc; + const u64 lost = (m << (63 - sc)) << 1; + const bool big = s > 63; + const u64 rr = big ? 0ULL : r; + const u64 st = big ? m : lost; + return rr | (st != 0 ? 1ULL : 0ULL); +} + +// m: leading bit at 62 for a normal result, 10 guard bits below the 53-bit significand, +// value = m * 2^(e - 1085) with e the biased exponent. Packing (e-1)<<52 + m lets a +// rounding carry ripple into the exponent. +GPUCA_B64_ALWAYS u64 roundPack(u64 sign, int32_t e, u64 m) +{ + if (e >= 0x7ff) { + return sign | GPUCA_B64_INF; + } + if (e <= 0) { + m = shrJam(m, 1 - e); + e = 1; + } + const u64 r = m & 0x3ffULL; + m >>= 10; + if (r > 0x200ULL || (r == 0x200ULL && (m & 1ULL))) { + ++m; + } + return sign | (((u64)(e - 1) << 52) + m); +} + +// an sNaN operand wins over a qNaN one, among equals the first wins, the result is quietened +GPUCA_B64_ALWAYS u64 propNaN(u64 a, u64 b, bool an, bool bn) +{ + const bool as = an && !(a & GPUCA_B64_QUIET), bs = bn && !(b & GPUCA_B64_QUIET); + if (as) { + return a | GPUCA_B64_QUIET; + } + if (bs) { + return b | GPUCA_B64_QUIET; + } + return an ? a : b; +} + +GPUCA_B64_OP u64 addsub(u64 a, u64 b0, bool neg) +{ + const u64 b = neg ? (b0 ^ GPUCA_B64_SIGN) : b0; + const bool sw = (a & ~GPUCA_B64_SIGN) < (b & ~GPUCA_B64_SIGN); + const u64 x = sw ? b : a, y = sw ? a : b; // |x| >= |y| + int32_t ex = (int32_t)((x >> 52) & 0x7ff), ey = (int32_t)((y >> 52) & 0x7ff); + u64 mx = x & GPUCA_B64_FRAC, my = y & GPUCA_B64_FRAC; + if (ex == 0x7ff) { // y can only be inf/NaN if x is too + const bool an = ((a >> 52) & 0x7ff) == 0x7ff && (a & GPUCA_B64_FRAC) != 0, bn = ((b0 >> 52) & 0x7ff) == 0x7ff && (b0 & GPUCA_B64_FRAC) != 0; + if (an || bn) { + return propNaN(a, b0, an, bn); + } + if (ey == 0x7ff) { + return ((x ^ y) & GPUCA_B64_SIGN) ? GPUCA_B64_DNAN : x; + } + return x; + } + const u64 sx = x & GPUCA_B64_SIGN; + const bool sub = ((x ^ y) >> 63) != 0; + mx = (ex ? (mx | GPUCA_B64_IMPL) : mx) << 10; + ex = ex ? ex : 1; + my = (ey ? (my | GPUCA_B64_IMPL) : my) << 10; + ey = ey ? ey : 1; + my = shrJam(my, ex - ey); + const u64 s = sub ? mx - my : mx + my; + const bool carry = (s >> 63) != 0; + const int32_t lz = clz64(s) - 1; // -1 on carry, 63 on zero + const u64 sN = carry ? ((s >> 1) | (s & 1ULL)) : (s << (lz & 63)); + const int32_t eN = carry ? ex + 1 : ex - lz; + const bool zero = s == 0; + return roundPack((zero && sub) ? 0ULL : sx, zero ? -1 : eN, sN); +} + +GPUCA_B64_OP u64 mul(u64 a, u64 b) +{ + const u64 sign = (a ^ b) & GPUCA_B64_SIGN; + int32_t ea = (int32_t)((a >> 52) & 0x7ff), eb = (int32_t)((b >> 52) & 0x7ff); + u64 ma = a & GPUCA_B64_FRAC, mb = b & GPUCA_B64_FRAC; + if (ea == 0x7ff || eb == 0x7ff) { + const bool an = ea == 0x7ff && ma != 0, bn = eb == 0x7ff && mb != 0; + if (an || bn) { + return propNaN(a, b, an, bn); + } + if ((ea == 0 && ma == 0) || (eb == 0 && mb == 0)) { + return GPUCA_B64_DNAN; // inf * 0 + } + return sign | GPUCA_B64_INF; + } + if (ea == 0) { + if (ma == 0) { + return sign; + } + const int32_t lz = clz64(ma) - 11; + ma <<= lz; + ea = 1 - lz; + } else { + ma |= GPUCA_B64_IMPL; + } + if (eb == 0) { + if (mb == 0) { + return sign; + } + const int32_t lz = clz64(mb) - 11; + mb <<= lz; + eb = 1 - lz; + } else { + mb |= GPUCA_B64_IMPL; + } + const u64 lo = ma * mb, hi = mulhiu(ma, mb); // 106-bit product in [2^104, 2^106) + const bool top = (hi & (1ULL << 41)) != 0; + const u64 m1 = (hi << 21) | (lo >> 43), m0 = (hi << 22) | (lo >> 42); + const u64 st = top ? (lo & ((1ULL << 43) - 1)) : (lo & ((1ULL << 42) - 1)); + const u64 m = (top ? m1 : m0) | (st != 0 ? 1ULL : 0ULL); + return roundPack(sign, ea + eb - 1023 + (top ? 1 : 0), m); +} + +GPUCA_B64_OP u64 div(u64 a, u64 b) +{ + const u64 sign = (a ^ b) & GPUCA_B64_SIGN; + int32_t ea = (int32_t)((a >> 52) & 0x7ff), eb = (int32_t)((b >> 52) & 0x7ff); + u64 ma = a & GPUCA_B64_FRAC, mb = b & GPUCA_B64_FRAC; + if (ea == 0x7ff || eb == 0x7ff) { + const bool an = ea == 0x7ff && ma != 0, bn = eb == 0x7ff && mb != 0; + if (an || bn) { + return propNaN(a, b, an, bn); + } + if (ea == 0x7ff && eb == 0x7ff) { + return GPUCA_B64_DNAN; // inf / inf + } + return ea == 0x7ff ? (sign | GPUCA_B64_INF) : sign; // inf / x, x / inf + } + if (eb == 0 && mb == 0) { + return (ea == 0 && ma == 0) ? GPUCA_B64_DNAN : (sign | GPUCA_B64_INF); // x / 0 + } + if (ea == 0) { + if (ma == 0) { + return sign; + } + const int32_t lz = clz64(ma) - 11; + ma <<= lz; + ea = 1 - lz; + } else { + ma |= GPUCA_B64_IMPL; + } + if (eb == 0) { + const int32_t lz = clz64(mb) - 11; + mb <<= lz; + eb = 1 - lz; + } else { + mb |= GPUCA_B64_IMPL; + } + const bool lt = ma < mb; + const int32_t e = ea - eb + 1023 - (lt ? 1 : 0); + const u64 A2 = lt ? (ma << 1) : ma; // A2 / mb in [1, 2) + // reciprocal R ~ 2^114 / mb in (2^61, 2^62], seeded from a float division on the top 24 bits + const u32 rb = asu32(1.0f / (float)(u32)(mb >> 29)); + u64 R = (u64)((rb & 0x7fffffu) | 0x800000u) << ((int32_t)((rb >> 23) & 0xff) - 127 + 62); + const u64 Bn = mb << 11; + for (int32_t it = 0; it < 2; ++it) { + const i64 E = (i64)(1ULL << 61) - (i64)mulhiu(Bn, R); + R = (u64)((i64)R + mulhis((i64)R, E << 3)); + } + R = R > (1ULL << 62) ? (1ULL << 62) : R; + // Q ~ A2 * 2^62 / mb, then the exact remainder, which fits in a signed 64-bit + // word because Q is within a few units + u64 Q = mulhiu(A2 << 10, (R << 2) - 1); + i64 rem = (i64)((A2 << 62) - Q * mb); + const i64 adj = mulhis(rem, (i64)R) >> 50; + Q = (u64)((i64)Q + adj); + rem -= adj * (i64)mb; + // after adj the remainder is within one divisor of [0, mb): one predicated step each way + const bool ng = rem < 0; + Q = ng ? Q - 1 : Q; + rem = ng ? rem + (i64)mb : rem; + const bool bg = rem >= (i64)mb; + Q = bg ? Q + 1 : Q; + rem = bg ? rem - (i64)mb : rem; + return roundPack(sign, e, Q | (rem != 0 ? 1ULL : 0ULL)); +} + +GPUCA_B64_OP u64 fromFloat(float f) +{ + const u32 u = asu32(f); + const u64 sign = (u64)(u & 0x80000000u) << 32; + int32_t e = (int32_t)((u >> 23) & 0xff); + u32 m = u & 0x7fffffu; + if (e == 0xff) { + return sign | GPUCA_B64_INF | ((u64)m << 29) | (m ? GPUCA_B64_QUIET : 0ULL); + } + if (e == 0) { + if (m == 0) { + return sign; + } + const int32_t lz = clz32(m) - 8; + m <<= lz; + e = 1 - lz; + } + return sign | ((u64)(e - 127 + 1023) << 52) | ((u64)(m & 0x7fffffu) << 29); +} + +// binary64 -> binary32, round to nearest even, subnormals, inf, NaN (payload kept, quietened) +GPUCA_B64_OP float toFloat(u64 d) +{ + const u32 sign = (u32)(d >> 32) & 0x80000000u; + const int32_t be = (int32_t)((d >> 52) & 0x7ff); + const u32 man = (u32)((d & GPUCA_B64_FRAC) >> 29); + const u32 drop = (u32)d & 0x1fffffffu; + u32 bits; + if (be == 0x7ff) { + bits = sign | 0x7f800000u | man | ((d & GPUCA_B64_FRAC) ? 0x400000u : 0u); + } else if (be == 0) { + bits = sign; + } else { + const int32_t e = be - 1023 + 127; + if (e >= 0xff) { + bits = sign | 0x7f800000u; + } else if (e > 0) { + bits = sign | ((u32)e << 23) | man; + if ((drop & 0x10000000u) && ((drop & 0x0fffffffu) || (man & 1u))) { + bits += 1u; + } + } else if (e > -24) { + const u32 full = man | 0x800000u; + const u32 sh = (u32)(1 - e); + const u32 lost = full & ((1u << sh) - 1u); + const u32 halfb = 1u << (sh - 1); + u32 sub = full >> sh; + if (lost > halfb || (lost == halfb && ((sub & 1u) || drop))) { + sub += 1u; + } + bits = sign | sub; + } else { + bits = sign; + } + } + return asf32(bits); +} + +} // namespace binary64_detail + +class GPUdoubleBinary64 +{ + public: + GPUdDefault() GPUdoubleBinary64() = default; + GPUdi() GPUdoubleBinary64(float v) : mBits(binary64_detail::fromFloat(v)) {} + GPUdi() operator float() const { return binary64_detail::toFloat(mBits); } + + GPUdi() static GPUdoubleBinary64 fromBits(binary64_detail::u64 b) + { + GPUdoubleBinary64 r; + r.mBits = b; + return r; + } + GPUdi() binary64_detail::u64 bits() const { return mBits; } + + GPUdi() GPUdoubleBinary64 operator-() const { return fromBits(mBits ^ GPUCA_B64_SIGN); } + GPUdi() GPUdoubleBinary64 operator+(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::addsub(mBits, b.mBits, false)); } + GPUdi() GPUdoubleBinary64 operator-(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::addsub(mBits, b.mBits, true)); } + GPUdi() GPUdoubleBinary64 operator*(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::mul(mBits, b.mBits)); } + GPUdi() GPUdoubleBinary64 operator/(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::div(mBits, b.mBits)); } + + GPUdi() GPUdoubleBinary64 operator+(float b) const { return *this + GPUdoubleBinary64(b); } + GPUdi() GPUdoubleBinary64 operator-(float b) const { return *this - GPUdoubleBinary64(b); } + GPUdi() GPUdoubleBinary64 operator*(float b) const { return *this * GPUdoubleBinary64(b); } + GPUdi() GPUdoubleBinary64 operator/(float b) const { return *this / GPUdoubleBinary64(b); } +#ifndef __METAL__ + GPUdi() GPUdoubleBinary64 operator+(double b) const { return *this + (float)b; } + GPUdi() GPUdoubleBinary64 operator-(double b) const { return *this - (float)b; } + GPUdi() GPUdoubleBinary64 operator*(double b) const { return *this * (float)b; } + GPUdi() GPUdoubleBinary64 operator/(double b) const { return *this / (float)b; } +#endif + + GPUdi() GPUdoubleBinary64& operator+=(GPUdoubleBinary64 b) { return *this = *this + b; } + GPUdi() GPUdoubleBinary64& operator-=(GPUdoubleBinary64 b) { return *this = *this - b; } + GPUdi() GPUdoubleBinary64& operator*=(GPUdoubleBinary64 b) { return *this = *this * b; } + GPUdi() GPUdoubleBinary64& operator/=(GPUdoubleBinary64 b) { return *this = *this / b; } + + private: + binary64_detail::u64 mBits; +}; + +GPUdi() GPUdoubleBinary64 operator+(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) + b; } +GPUdi() GPUdoubleBinary64 operator-(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) - b; } +GPUdi() GPUdoubleBinary64 operator*(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) * b; } +GPUdi() GPUdoubleBinary64 operator/(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) / b; } +#ifndef __METAL__ +GPUdi() GPUdoubleBinary64 operator+(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) + b; } +GPUdi() GPUdoubleBinary64 operator-(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) - b; } +GPUdi() GPUdoubleBinary64 operator*(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) * b; } +GPUdi() GPUdoubleBinary64 operator/(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) / b; } +#endif + +// rounds once, as `someFloat += someDouble` does on the host +#ifdef __METAL__ +GPUdi() thread float& operator+=(thread float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +GPUdi() device float& operator+=(device float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +GPUdi() threadgroup float& operator+=(threadgroup float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +#else +GPUdi() float& operator+=(float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +#endif + +} // namespace o2::gpu + +#endif // GPUCOMMONDOUBLEBINARY64_H From 319055344cfaac79331b85022536c0e6e45b3f4e Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 28/41] GPUTracking: drop static from the function-scope constants MSL has no static storage duration inside a function. These are all scalar constexpr values used as compile-time constants, so removing static changes nothing for any backend: none of them is odr-used, and no storage was ever emitted for them. --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 ++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 12 ++++++------ GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 4 ++-- .../SectorTracker/GPUTPCNeighboursFinder.cxx | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 93308772dfb96..aa4a8b8e252b5 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1480,8 +1480,8 @@ struct GPUTPCGMMerger_CompareClusterIds { GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { - static constexpr int32_t kMaxParts = 16; - static constexpr int32_t kMaxClusters = constants::MERGER_MAX_TRACK_CLUSTERS; + constexpr int32_t kMaxParts = 16; + constexpr int32_t kMaxClusters = constants::MERGER_MAX_TRACK_CLUSTERS; GPUTPCGMSectorTrack* trackParts[kMaxParts]; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 23842d8a1f859..e418a8c7fecfa 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -49,7 +49,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) { - static constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); const GPUParam& GPUrestrict() param = merger->Param(); @@ -366,8 +366,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) { - static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; @@ -575,7 +575,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) { - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; if (Merger->Param().rec.tpc.disableRefitAttachment & 2) { return dodEdx; } @@ -678,7 +678,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float toX = mX; bool inFlyDirection = (Merger->MergedTracks()[iTrack].Leg() & 1) ^ up; - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = Merger->Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; @@ -744,7 +744,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) { - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; float Y = mP[2] > 0 ? -mX : mX; diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 4ef0f29eabdff..9e86e4627a2fb 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -398,8 +398,8 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov trk.NormalizeAlpha(alpha); prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { - static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX)) { if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(kSectAngle / 2.f)) { diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx index 5151427377b05..eb3038602283e 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx @@ -73,11 +73,11 @@ GPUdii() void GPUTPCNeighboursFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nTh return; } - static constexpr uint32_t UNROLL_GLOBAL = GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL > 1 ? GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL : 1; + constexpr uint32_t UNROLL_GLOBAL = GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL > 1 ? GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL : 1; static_assert(constants::NEIGHBOURS_MAX_N % UNROLL_GLOBAL == 0); - static constexpr uint32_t MAX_SHARED = GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP; - static constexpr uint32_t MAX_GLOBAL = (MAX_SHARED < constants::NEIGHBOURS_MAX_N) ? (((constants::NEIGHBOURS_MAX_N - MAX_SHARED - 1) / UNROLL_GLOBAL + 1) * UNROLL_GLOBAL) : 0; - static constexpr uint32_t MAX_TOTAL = MAX_SHARED + MAX_GLOBAL; + constexpr uint32_t MAX_SHARED = GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP; + constexpr uint32_t MAX_GLOBAL = (MAX_SHARED < constants::NEIGHBOURS_MAX_N) ? (((constants::NEIGHBOURS_MAX_N - MAX_SHARED - 1) / UNROLL_GLOBAL + 1) * UNROLL_GLOBAL) : 0; + constexpr uint32_t MAX_TOTAL = MAX_SHARED + MAX_GLOBAL; const float chi2Cut = 3.f * 3.f * 4 * (s.mUpDx * s.mUpDx + s.mDnDx * s.mDnDx); // float chi2Cut = 3.f*3.f*(s.mUpDx*s.mUpDx + s.mDnDx*s.mDnDx ); //SG From fdadd54e9fa72fb634a1c77eb35d2e29ac102eef Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 29/41] GPUTracking: declare the resolve kernel's shared memory as shared The definitions in GPUTPCGMMergerGPU.cxx qualify smem with GPUsharedref(), and so does every other declaration in the header; this one did not. It makes no difference where GPUsharedref() is empty, but on Metal the declaration then takes a generic reference and the definition a threadgroup one, which are different types. --- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 5d00451516aa8..d703f1b890443 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -81,7 +81,7 @@ class GPUTPCGMMergerResolve : public GPUTPCGMMergerGeneral }; template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, Args... args); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& clusterer, Args... args); }; class GPUTPCGMMergerClearLinks : public GPUTPCGMMergerGeneral From 098e41cd463898ece5bc4742a9a857ca80c50e42 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 30/41] GPUTracking: rename the cluster finder's fragment to frag fragment is a reserved word in MSL, where it qualifies a shader stage, so it cannot name a member, a parameter or a local. The cluster finder used it for all three, which was 35 of the errors left in the Metal compile. Purely a rename, in code positions only: the option descriptions in GPUSettingsList.h that mention fragments are strings and are untouched, as are the comments. --- GPU/GPUTracking/Global/GPUChainTracking.h | 8 +- .../Global/GPUChainTrackingClusterizer.cxx | 76 +++++++++---------- .../GPUTPCCFChargeMapFiller.cxx | 22 +++--- .../GPUTPCCFCheckPadBaseline.cxx | 20 ++--- .../TPCClusterFinder/GPUTPCCFClusterizer.cxx | 2 +- .../TPCClusterFinder/GPUTPCCFClusterizer.inc | 6 +- .../TPCClusterFinder/GPUTPCCFDecodeZS.cxx | 20 ++--- .../TPCClusterFinder/GPUTPCClusterFinder.h | 2 +- .../GPUTPCClusterFinderDump.cxx | 18 ++--- .../GPUTPCNNClusterizerKernels.cxx | 18 ++--- 10 files changed, 96 insertions(+), 96 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 759aaf818028e..868400401d7a1 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -300,11 +300,11 @@ class GPUChainTracking : public GPUChain int32_t RunTPCTrackingSectors_internal(); int32_t RunTPCClusterizer_prepare(bool restorePointers, const GPUTPCExtraADC& extraADCs); #ifndef GPUCA_RUN2 - std::pair RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& fragment, int32_t lane, const GPUTPCExtraADC& extraADCs); + std::pair RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& frag, int32_t lane, const GPUTPCExtraADC& extraADCs); void RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int32_t stage, bool doGPU, int32_t lane); - std::pair TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& fragment); - std::pair TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& fragment); - void TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& fragment); + std::pair TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& frag); + std::pair TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& frag); + void TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& frag); void TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs); void TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs); #endif diff --git a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx index 8c6534d74b31d..1557b2fab0934 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx @@ -70,7 +70,7 @@ using namespace o2::tpc::constants; using namespace o2::dataformats; #ifndef GPUCA_RUN2 -std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& fragment) +std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& frag) { bool doGPU = mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCClusterFinding; GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; @@ -78,7 +78,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdat uint32_t digits = 0; uint32_t pages = 0; for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) { - clusterer.mMinMaxCN[j] = mCFContext->fragmentData[fragment.index].minMaxCN[iSector][j]; + clusterer.mMinMaxCN[j] = mCFContext->fragmentData[frag.index].minMaxCN[iSector][j]; if (doGPU) { uint16_t posInEndpoint = 0; uint16_t pagesEndpoint = 0; @@ -86,7 +86,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdat const uint32_t pageFirst = (k == clusterer.mMinMaxCN[j].zsPtrFirst) ? clusterer.mMinMaxCN[j].zsPageFirst : 0; const uint32_t pageLast = (k + 1 == clusterer.mMinMaxCN[j].zsPtrLast) ? clusterer.mMinMaxCN[j].zsPageLast : mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]; for (uint32_t l = pageFirst; l < pageLast; l++) { - uint16_t pageDigits = mCFContext->fragmentData[fragment.index].pageDigits[iSector][j][posInEndpoint++]; + uint16_t pageDigits = mCFContext->fragmentData[frag.index].pageDigits[iSector][j][posInEndpoint++]; if (pageDigits) { *(o++) = GPUTPCClusterFinder::ZSOffset{digits, j, pagesEndpoint}; digits += pageDigits; @@ -94,35 +94,35 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdat pagesEndpoint++; } } - if (pagesEndpoint != mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()) { + if (pagesEndpoint != mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()) { if (GetProcessingSettings().ignoreNonFatalGPUErrors) { - GPUError("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()); + GPUError("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()); return {0, 0}; } else { - GPUFatal("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()); + GPUFatal("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()); } } } else { clusterer.mPzsOffsets[j] = GPUTPCClusterFinder::ZSOffset{digits, j, 0}; - digits += mCFContext->fragmentData[fragment.index].nDigits[iSector][j]; - pages += mCFContext->fragmentData[fragment.index].nPages[iSector][j]; + digits += mCFContext->fragmentData[frag.index].nDigits[iSector][j]; + pages += mCFContext->fragmentData[frag.index].nPages[iSector][j]; } } if (doGPU) { pages = o - processors()->tpcClusterer[iSector].mPzsOffsets; } if (GetProcessingSettings().clusterizerZSSanityCheck && mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) { - TPCClusterizerEnsureZSOffsets(iSector, fragment); + TPCClusterizerEnsureZSOffsets(iSector, frag); } return {digits, pages}; } -void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& fragment) +void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& frag) { GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; uint32_t nAdcs = 0; for (uint16_t endpoint = 0; endpoint < GPUTrackingInOutZS::NENDPOINTS; endpoint++) { - const auto& data = mCFContext->fragmentData[fragment.index]; + const auto& data = mCFContext->fragmentData[frag.index]; uint32_t pagesEndpoint = 0; const uint32_t nAdcsExpected = data.nDigits[iSector][endpoint]; const uint32_t nPagesExpected = data.nPages[iSector][endpoint]; @@ -144,15 +144,15 @@ void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfF } if (pagesEndpoint != nPagesExpected) { - GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector, endpoint, fragment.index, pagesEndpoint, nPagesExpected); + GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector, endpoint, frag.index, pagesEndpoint, nPagesExpected); } if (nAdcDecoded != nAdcsExpected) { - GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector, endpoint, fragment.index, nAdcsExpected, nAdcDecoded); + GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector, endpoint, frag.index, nAdcsExpected, nAdcDecoded); } if (nAdcs != clusterer.mPzsOffsets[endpoint].offset) { - GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC offset mismatch: expected %u, buffered %u", iSector, endpoint, fragment.index, nAdcs, clusterer.mPzsOffsets[endpoint].offset); + GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC offset mismatch: expected %u, buffered %u", iSector, endpoint, frag.index, nAdcs, clusterer.mPzsOffsets[endpoint].offset); } nAdcs += nAdcsExpected; @@ -162,10 +162,10 @@ void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfF void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs) { const int32_t iSector = clusterer.mISector; - const auto& fragment = clusterer.mPmemory->fragment; + const auto& frag = clusterer.mPmemory->frag; const auto& digits = extraADCs.digitsBySector[iSector]; - if (fragment.index != 0) { + if (frag.index != 0) { return; } @@ -188,11 +188,11 @@ void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clust SynchronizeStream(lane); for (const auto& d : digits) { - if (!fragment.contains(d.getTimeStamp())) { + if (!frag.contains(d.getTimeStamp())) { continue; } - CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - fragment.start)}; + CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - frag.start)}; chargeMapHost[pos] = PackedCharge(d.getChargeFloat()); extraPositions.push_back(pos); @@ -208,10 +208,10 @@ void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clust void GPUChainTracking::TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs) { const int32_t iSector = clusterer.mISector; - const auto& fragment = clusterer.mPmemory->fragment; + const auto& frag = clusterer.mPmemory->frag; const auto& digits = extraADCs.digitsBySector[iSector]; - if (fragment.index != 0) { + if (frag.index != 0) { return; } @@ -233,11 +233,11 @@ void GPUChainTracking::TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clu size_t nNonZeroADCs = 0; for (const auto& d : digits) { - if (!fragment.contains(d.getTimeStamp())) { + if (!frag.contains(d.getTimeStamp())) { continue; } - CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - fragment.start)}; + CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - frag.start)}; auto adc = chargeMapHost[pos].unpack(); @@ -326,7 +326,7 @@ GPUTPCExtraADC GenerateSaturatedSignals(size_t seed = 42) } // namespace -std::pair GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& fragment) +std::pair GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& frag) { mRec->getGeneralStepTimer(GeneralStep::Prepare).Start(); uint32_t nDigits = 0; @@ -349,7 +349,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCount(uint std::vector> fragments; fragments.reserve(mCFContext->nFragments); - fragments.emplace_back(std::pair{fragment, {0, 0, 0, 0, 0, -1}}); + fragments.emplace_back(std::pair{frag, {0, 0, 0, 0, 0, -1}}); for (uint32_t i = 1; i < mCFContext->nFragments; i++) { fragments.emplace_back(std::pair{fragments.back().first.next(), {0, 0, 0, 0, 0, -1}}); } @@ -601,14 +601,14 @@ void GPUChainTracking::RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clust } } -std::pair GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& fragment, int32_t lane, const GPUTPCExtraADC& extraADCs) +std::pair GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& frag, int32_t lane, const GPUTPCExtraADC& extraADCs) { bool doGPU = GetRecoStepsGPU() & RecoStep::TPCClusterFinding; if (mCFContext->abandonTimeframe) { return {0, 0}; } - auto retVal = TPCClusterizerDecodeZSCountUpdate(iSector, fragment); - if (fragment.index == 0) { + auto retVal = TPCClusterizerDecodeZSCountUpdate(iSector, frag); + if (frag.index == 0) { retVal.first += extraADCs.digitsBySector[iSector].size(); } if (doGPU) { @@ -994,12 +994,12 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) std::vector laneHasData(GetProcessingSettings().nTPCClustererLanes, false); static_assert(NSECTORS <= constants::GPU_MAX_STREAMS, "Stream events must be able to hold all sectors"); const int32_t maxLane = std::min(GetProcessingSettings().nTPCClustererLanes, NSECTORS - iSectorBase); - for (CfFragment fragment = mCFContext->fragmentFirst; !fragment.isEnd(); fragment = fragment.next()) { + for (CfFragment frag = mCFContext->fragmentFirst; !frag.isEnd(); frag = frag.next()) { if (GetProcessingSettings().debugLevel >= 3) { - GPUInfo("Processing time bins [%d, %d) for sectors %d to %d", fragment.start, fragment.last(), iSectorBase, iSectorBase + GetProcessingSettings().nTPCClustererLanes - 1); + GPUInfo("Processing time bins [%d, %d) for sectors %d to %d", frag.start, frag.last(), iSectorBase, iSectorBase + GetProcessingSettings().nTPCClustererLanes - 1); } mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) { - if (doGPU && fragment.index != 0) { + if (doGPU && frag.index != 0) { SynchronizeStream(lane); // Don't overwrite charge map from previous iteration until cluster computation is finished } @@ -1007,7 +1007,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer; clusterer.mPmemory->counters.nPeaks = clusterer.mPmemory->counters.nClusters = 0; - clusterer.mPmemory->fragment = fragment; + clusterer.mPmemory->frag = frag; if (mIOPtrs.tpcPackedDigits) { bool setDigitsOnGPU = doGPU && not mIOPtrs.tpcZS; @@ -1037,7 +1037,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) using PeakMapType = decltype(*clustererShadow.mPpeakMap); runKernel({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPchargeMap, TPCMapMemoryLayout::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(ChargeMapType)); runKernel({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpeakMap, TPCMapMemoryLayout::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(PeakMapType)); - if (fragment.index == 0) { + if (frag.index == 0) { runKernel({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpadIsNoisy, TPC_CLUSTERER_STRIDED_PAD_COUNT * sizeof(*clustererShadow.mPpadIsNoisy)); } DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererZeroedCharges, clusterer, &GPUTPCClusterFinder::DumpChargeMap, *mDebugFile, "Zeroed Charges"); @@ -1060,7 +1060,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) } if (propagateMCLabels) { - if (fragment.index == 0) { + if (frag.index == 0) { // Must be only called on the first fragment as some buffers are used across the whole timeframe clusterer.AllocMCBuffers(); } @@ -1112,7 +1112,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) SynchronizeStream(lane); } if (mIOPtrs.tpcZS) { - CfFragment f = fragment.next(); + CfFragment f = frag.next(); int32_t nextSector = iSector; if (f.isEnd()) { nextSector += GetProcessingSettings().nTPCClustererLanes; @@ -1138,7 +1138,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) } bool checkForNoisyPads = (rec()->GetParam().rec.tpc.maxTimeBinAboveThresholdIn1000Bin > 0) || (rec()->GetParam().rec.tpc.maxConsecTimeBinAboveThreshold > 0); - checkForNoisyPads &= (rec()->GetParam().rec.tpc.noisyPadsQuickCheck ? fragment.index == 0 : true); + checkForNoisyPads &= (rec()->GetParam().rec.tpc.noisyPadsQuickCheck ? frag.index == 0 : true); checkForNoisyPads &= !GetProcessingSettings().disableTPCNoisyPadFilter; // TODO Move hipTailFilter flag to ProcessingSettings? // TODO Add some warning when re enabling pad filter with this flag, so it's not just silently enabled when disabling was requested @@ -1152,7 +1152,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) const int32_t nBlocks = GPUTPCGeometry::NROWS; runKernel({GetGridBlk(nBlocks, lane), {iSector}}); - getKernelTimer(RecoStep::TPCClusterFinding, iSector, TPC_REAL_PADS_IN_SECTOR * fragment.lengthWithoutOverlap() * sizeof(PackedCharge), false); + getKernelTimer(RecoStep::TPCClusterFinding, iSector, TPC_REAL_PADS_IN_SECTOR * frag.lengthWithoutOverlap() * sizeof(PackedCharge), false); } DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererDigits, clusterer, &GPUTPCClusterFinder::DumpDigits, *mDebugFile); @@ -1197,7 +1197,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer; - const bool resetClusterCounters = fragment.index == 0; + const bool resetClusterCounters = frag.index == 0; // The reset must also run for an empty first fragment since later fragments can contain data. if (clusterer.mPmemory->counters.nPositions == 0 && !resetClusterCounters) { return; @@ -1403,7 +1403,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) } if (GetProcessingSettings().debugLevel >= 3) { - GPUInfo("Sector %02d Fragment %02d Lane %d: Found clusters: digits %u peaks %u clusters %u", iSector, fragment.index, lane, (int32_t)clusterer.mPmemory->counters.nPositions, (int32_t)clusterer.mPmemory->counters.nPeaks, (int32_t)clusterer.mPmemory->counters.nClusters); + GPUInfo("Sector %02d Fragment %02d Lane %d: Found clusters: digits %u peaks %u clusters %u", iSector, frag.index, lane, (int32_t)clusterer.mPmemory->counters.nPositions, (int32_t)clusterer.mPmemory->counters.nPeaks, (int32_t)clusterer.mPmemory->counters.nClusters); } TransferMemoryResourcesToHost(RecoStep::TPCClusterFinding, &clusterer, lane); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx index ed6d97dfd4e3c..95f78bd9890ee 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx @@ -24,11 +24,11 @@ template <> GPUdii() void GPUTPCCFChargeMapFiller::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D indexMap(clusterer.mPindexMap); - fillIndexMapImpl(nBlocks, nThreads, iBlock, iThread, clusterer.mPmemory->fragment, clusterer.mPdigits, indexMap, clusterer.mPmemory->counters.nDigitsInFragment); + fillIndexMapImpl(nBlocks, nThreads, iBlock, iThread, clusterer.mPmemory->frag, clusterer.mPdigits, indexMap, clusterer.mPmemory->counters.nDigitsInFragment); } GPUd() void GPUTPCCFChargeMapFiller::fillIndexMapImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, - const CfFragment& fragment, + const CfFragment& frag, const tpc::Digit* digits, CfArray2D& indexMap, size_t maxDigit) @@ -37,9 +37,9 @@ GPUd() void GPUTPCCFChargeMapFiller::fillIndexMapImpl(int32_t nBlocks, int32_t n if (idx >= maxDigit) { return; } - CPU_ONLY(idx += fragment.digitsStart); + CPU_ONLY(idx += frag.digitsStart); CPU_ONLY(tpc::Digit digit = digits[idx]); - CPU_ONLY(CfChargePos pos(digit.getRow(), digit.getPad(), fragment.toLocal(digit.getTimeStamp()))); + CPU_ONLY(CfChargePos pos(digit.getRow(), digit.getPad(), frag.toLocal(digit.getTimeStamp()))); CPU_ONLY(indexMap.safeWrite(pos, idx)); } @@ -47,10 +47,10 @@ template <> GPUdii() void GPUTPCCFChargeMapFiller::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); - fillFromDigitsImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->fragment, clusterer.mPmemory->counters.nPositions, clusterer.mPdigits, clusterer.mPpositions, chargeMap); + fillFromDigitsImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->frag, clusterer.mPmemory->counters.nPositions, clusterer.mPdigits, clusterer.mPpositions, chargeMap); } -GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, const CfFragment& fragment, size_t digitNum, +GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, const CfFragment& frag, size_t digitNum, const tpc::Digit* digits, CfChargePos* positions, CfArray2D& chargeMap) @@ -59,9 +59,9 @@ GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t if (idx >= digitNum) { return; } - tpc::Digit digit = digits[fragment.digitsStart + idx]; + tpc::Digit digit = digits[frag.digitsStart + idx]; - CfChargePos pos(digit.getRow(), digit.getPad(), fragment.toLocal(digit.getTimeStamp())); + CfChargePos pos(digit.getRow(), digit.getPad(), frag.toLocal(digit.getTimeStamp())); positions[idx] = pos; float q = digit.getChargeFloat(); q *= clusterer.GetConstantMem()->calibObjects.tpcPadGain->getGainCorrection(clusterer.mISector, digit.getRow(), digit.getPad()); @@ -77,10 +77,10 @@ GPUdii() void GPUTPCCFChargeMapFiller::Threadcounters.nDigits; const tpc::Digit* digits = clusterer.mPdigits; - size_t st = findTransition(clusterer.mPmemory->fragment.first(), digits, nDigits, 0); - size_t end = findTransition(clusterer.mPmemory->fragment.last(), digits, nDigits, st); + size_t st = findTransition(clusterer.mPmemory->frag.first(), digits, nDigits, 0); + size_t end = findTransition(clusterer.mPmemory->frag.last(), digits, nDigits, st); - clusterer.mPmemory->fragment.digitsStart = st; + clusterer.mPmemory->frag.digitsStart = st; size_t elems = end - st; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx index 4c48ed3e097f4..c1a459d138dbd 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx @@ -224,7 +224,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPU(int32_t nBlocks, int32_t return; } - const CfFragment& fragment = clusterer.mPmemory->fragment; + const CfFragment& frag = clusterer.mPmemory->frag; const bool hipFilterOn = clusterer.Param().rec.tpc.hipTailFilter; const Charge hipTailThreshold = clusterer.Param().rec.tpc.hipTailFilterThreshold; const Charge hipTailFilterAlpha = clusterer.Param().rec.tpc.hipTailFilterAlpha; @@ -253,7 +253,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPU(int32_t nBlocks, int32_t // saturated signal in overlap region can create tails in the next fragment // even when cleared in current fragment as they're decoded twice const TPCFragmentTime firstTB = 0; - const TPCFragmentTime lastTB = fragment.length; + const TPCFragmentTime lastTB = frag.length; for (uint16_t t = firstTB; t < lastTB; t += NumOfCachedTBs) { @@ -384,7 +384,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t const int32_t nPads = geo.NPads(row); const int32_t nVecPads = (nPads + PadsPerCacheline - 1) / PadsPerCacheline; - const CfFragment& fragment = clusterer.mPmemory->fragment; + const CfFragment& frag = clusterer.mPmemory->frag; const bool hipFilterOn = clusterer.Param().rec.tpc.hipTailFilter; const Charge hipTailThreshold = clusterer.Param().rec.tpc.hipTailFilterThreshold; const Charge hipTailFilterAlpha = clusterer.Param().rec.tpc.hipTailFilterAlpha; @@ -409,7 +409,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t std::vector activeHIPTailEndV(nVecPads, -1); std::vector tailFilterChargeV(nVecPads, Charge8{Vc::Zero}); - for (int16_t t = 0; t < fragment.length; t += NumOfCachedTBs) { + for (int16_t t = 0; t < frag.length; t += NumOfCachedTBs) { bool hasAnyTrigger = false; @@ -432,7 +432,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t for (tpccf::TPCFragmentTime localtime = 0; localtime < NumOfCachedTBs; localtime++) { const uint16_t* packedChargeStart = reinterpret_cast(&chargeMap[basePos.delta({0, localtime})]); - const UShort8 packedCharges = t + localtime < fragment.length + const UShort8 packedCharges = t + localtime < frag.length ? UShort8{packedChargeStart, Vc::Aligned} : UShort8{Vc::Zero}; const auto isCharge = packedCharges != 0; @@ -585,7 +585,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t auto activeHIPTailEnd = activeHIPTailEndV[iVecPad]; const auto shouldCloseTail = activeHIPTailStart > -1; - activeHIPTailEnd(shouldCloseTail && activeHIPTailEnd < 0) = fragment.length; + activeHIPTailEnd(shouldCloseTail && activeHIPTailEnd < 0) = frag.length; if (hipFilterOn && shouldCloseTail.isNotEmpty()) { for (int16_t p = 0; p < PadsPerCacheline; p++) { @@ -639,8 +639,8 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t GPUd() void GPUTPCCFCheckPadBaseline::updatePadBaseline(int32_t pad, const GPUTPCClusterFinder& clusterer, int32_t totalCharges, int32_t consecCharges, Charge maxCharge) { - const CfFragment& fragment = clusterer.mPmemory->fragment; - const int32_t totalChargesBaseline = clusterer.Param().rec.tpc.maxTimeBinAboveThresholdIn1000Bin * fragment.lengthWithoutOverlap() / 1000; + const CfFragment& frag = clusterer.mPmemory->frag; + const int32_t totalChargesBaseline = clusterer.Param().rec.tpc.maxTimeBinAboveThresholdIn1000Bin * frag.lengthWithoutOverlap() / 1000; const int32_t consecChargesBaseline = clusterer.Param().rec.tpc.maxConsecTimeBinAboveThreshold; const uint16_t saturationThreshold = clusterer.Param().rec.tpc.noisyPadSaturationThreshold; const bool isNoisy = (!saturationThreshold || maxCharge < saturationThreshold) && ((totalChargesBaseline > 0 && totalCharges >= totalChargesBaseline) || (consecChargesBaseline > 0 && consecCharges >= consecChargesBaseline)); @@ -728,7 +728,7 @@ GPUd() void GPUTPCCFHIPClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, nTails = CAMath::Min(nTails, (uint32_t)MaxHIPTailsPerRow - 1); const auto* tails = GetHIPTails(clusterer, row); - const auto& fragment = clusterer.mPmemory->fragment; + const auto& frag = clusterer.mPmemory->frag; auto* clusterPosInRow = clusterer.mPhipClusterPosInRow ? clusterer.mPhipClusterPosInRow + row * MaxHIPTailsPerRow @@ -776,7 +776,7 @@ GPUd() void GPUTPCCFHIPClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, cn.qMax = qMax; cn.setSaturatedQtot(qTot); cn.setSaturatedTailLength(tailEnd - tailStart); - float clusterTime = fragment.start + timeMean - clusterer.Param().rec.tpc.clustersShiftTimebinsClusterizer; + float clusterTime = frag.start + timeMean - clusterer.Param().rec.tpc.clustersShiftTimebinsClusterizer; cn.setTimeFlags(clusterTime, 0); cn.setPad(padMean); cn.setSigmaPad(padSigma); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx index 62e89ef4b7ddf..37db02d1b8559 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx @@ -35,5 +35,5 @@ GPUdii() void GPUTPCCFClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, tpc::ClusterNative* clusterOut = onlyMC ? nullptr : clusterer.mPclusterByRow; - GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, true); + GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->frag, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, true); } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc index 22cbeec9e86fb..f21f9b47480af 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc @@ -17,7 +17,7 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, - const CfFragment& fragment, + const CfFragment& frag, GPUSharedMemory& smem, const CfArray2D& chargeMap, const CfChargePos* filteredPeakPositions, @@ -55,14 +55,14 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t if (idx >= clusternum) { return; } - if (fragment.isOverlap(pos.time())) { + if (frag.isOverlap(pos.time())) { if (clusterPosInRow) { clusterPosInRow[idx] = maxClusterPerRow; } return; } tpc::ClusterNative myCluster; - pc.finalize(pos, charge, fragment.start); + pc.finalize(pos, charge, frag.start); bool rejectCluster = !pc.toNative(pos, charge, myCluster, clusterer.Param(), chargeMap); if (!isAccepted) { rejectCluster = true; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index aac5dbc0fc137..0f2d30362b4f0 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -172,13 +172,13 @@ GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUshared seqLen = rowData[(nSeq + 1) * 2] - rowData[nSeq * 2]; pad = rowData[nSeq++ * 2 + 1]; } - const CfFragment& fragment = clusterer.mPmemory->fragment; + const CfFragment& frag = clusterer.mPmemory->frag; TPCTime globalTime = timeBin + l; - bool discardTimeBin = not fragment.contains(globalTime); + bool discardTimeBin = not frag.contains(globalTime); discardTimeBin |= (tpcTimeBinCut > 0 && globalTime > tpcTimeBinCut); Row row = rowOffset + m; - CfChargePos pos(row, Pad(pad), discardTimeBin ? INVALID_TIME_BIN : fragment.toLocal(globalTime)); + CfChargePos pos(row, Pad(pad), discardTimeBin ? INVALID_TIME_BIN : frag.toLocal(globalTime)); positions[nDigitsTmp++] = pos; if (!discardTimeBin) { @@ -219,7 +219,7 @@ GPUdii() void GPUTPCCFDecodeZSLink::Thread<0>(int32_t nBlocks, int32_t nThreads, GPUd() size_t GPUTPCCFDecodeZSLink::DecodePage(GPUSharedMemory& smem, DecodeCtx& ctx) { - const CfFragment& fragment = ctx.clusterer.mPmemory->fragment; + const CfFragment& frag = ctx.clusterer.mPmemory->frag; const auto* rdHdr = ConsumeHeader(ctx.page); @@ -246,7 +246,7 @@ GPUd() size_t GPUTPCCFDecodeZSLink::DecodePage(GPUSharedMemory& smem, DecodeCtx& nDecoded += nAdc; - bool discardTimeBin = not fragment.contains(timeBin); + bool discardTimeBin = not frag.contains(timeBin); discardTimeBin |= (ctx.tpcTimeBinCut > 0 && timeBin > ctx.tpcTimeBinCut); if (discardTimeBin) { @@ -331,9 +331,9 @@ GPUd() void GPUTPCCFDecodeZSLink::DecodeTB( } o2::tpc::PadPos padAndRow = GetPadAndRowFromFEC(ctx.clusterer, cru, rawFECChannel, fecInPartition); - const CfFragment& fragment = ctx.clusterer.mPmemory->fragment; + const CfFragment& frag = ctx.clusterer.mPmemory->frag; float charge = ADCToFloat(adc, DECODE_MASK, DECODE_BITS_FACTOR); - WriteCharge(ctx.clusterer, charge, padAndRow, fragment.toLocal(timeBin), ctx.pageDigitOffset + myOffset); + WriteCharge(ctx.clusterer, charge, padAndRow, frag.toLocal(timeBin), ctx.pageDigitOffset + myOffset); } // for (uint8_t i = iThread; blockOffset < nAdc; i += NThreads) } @@ -651,7 +651,7 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( constexpr int32_t NTHREADS = GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCCFDecodeZSDenseLink); static_assert(NTHREADS == GPUCA_WARP_SIZE, "Decoding TB Headers in parallel assumes block size is a single warp."); - const CfFragment& fragment = ctx.clusterer.mPmemory->fragment; + const CfFragment& frag = ctx.clusterer.mPmemory->frag; // Read timebin block header uint16_t tbbHdr = ConsumeByte(ctx.page); @@ -721,7 +721,7 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( const uint8_t* adcData = ConsumeBytes(ctx.page, (nSamplesInTB * DECODE_BITS + 7) / 8); MAYBE_PAGE_OVERFLOW(ctx.page); - bool discardTimeBin = not fragment.contains(timeBin); + bool discardTimeBin = not frag.contains(timeBin); discardTimeBin |= (ctx.tpcTimeBinCut > 0 && timeBin > ctx.tpcTimeBinCut); if (discardTimeBin) { @@ -754,7 +754,7 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( o2::tpc::PadPos padAndRow = GetPadAndRowFromFEC(ctx.clusterer, cru, rawFECChannelLink, smem.linkIds[iLink]); float charge = ADCToFloat(adc, DECODE_MASK, DECODE_BITS_FACTOR); - WriteCharge(ctx.clusterer, charge, padAndRow, fragment.toLocal(timeBin), ctx.pageDigitOffset + sample); + WriteCharge(ctx.clusterer, charge, padAndRow, frag.toLocal(timeBin), ctx.pageDigitOffset + sample); } // for (uint16_t sample = iThread; sample < nSamplesInTB; sample += NTHREADS) diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h index d169440a8d972..4a790750c773b 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h @@ -63,7 +63,7 @@ class GPUTPCClusterFinder : public GPUProcessor uint32_t maxTimeBin = 0; uint32_t nPagesSubsector = 0; } counters; - CfFragment fragment; + CfFragment frag; }; struct ZSOffset { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx index 3b06db8efc1a3..d778dce37c4af 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx @@ -25,7 +25,7 @@ void GPUTPCClusterFinder::DumpDigits(std::ostream& out) { const auto nPositions = mPmemory->counters.nPositions; - out << "\nClusterer - Digits - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << ": " << nPositions << "\n"; + out << "\nClusterer - Digits - Sector " << mISector << " - Fragment " << mPmemory->frag.index << ": " << nPositions << "\n"; out << std::hex; for (size_t i = 0; i < mPmemory->counters.nPositions; i++) { @@ -37,7 +37,7 @@ void GPUTPCClusterFinder::DumpDigits(std::ostream& out) void GPUTPCClusterFinder::DumpChargeMap(std::ostream& out, std::string_view title) { - out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << "\n"; + out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->frag.index << "\n"; CfArray2D map(mPchargeMap); out << std::hex; @@ -70,7 +70,7 @@ void GPUTPCClusterFinder::DumpChargeMap(std::ostream& out, std::string_view titl void GPUTPCClusterFinder::DumpPeakMap(std::ostream& out, std::string_view title) { - out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << "\n"; + out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->frag.index << "\n"; CfArray2D map(mPpeakMap); @@ -106,7 +106,7 @@ void GPUTPCClusterFinder::DumpPeakMap(std::ostream& out, std::string_view title) void GPUTPCClusterFinder::DumpPeaks(std::ostream& out) { - out << "\nClusterer - Peaks - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << "\n"; + out << "\nClusterer - Peaks - Sector " << mISector << " - Fragment " << mPmemory->frag.index << "\n"; for (uint32_t i = 0; i < mPmemory->counters.nPositions; i++) { out << int32_t{mPisPeak[i]}; if ((i + 1) % 100 == 0) { @@ -119,7 +119,7 @@ void GPUTPCClusterFinder::DumpPeaksCompacted(std::ostream& out) { const auto nPeaks = mPmemory->counters.nPeaks; - out << "\nClusterer - Compacted Peaks - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << ": " << nPeaks << "\n"; + out << "\nClusterer - Compacted Peaks - Sector " << mISector << " - Fragment " << mPmemory->frag.index << ": " << nPeaks << "\n"; for (size_t i = 0; i < nPeaks; i++) { const auto& pos = mPpeakPositions[i]; out << pos.time() << " " << int32_t{pos.pad()} << " " << int32_t{pos.row()} << "\n"; @@ -128,10 +128,10 @@ void GPUTPCClusterFinder::DumpPeaksCompacted(std::ostream& out) void GPUTPCClusterFinder::DumpSuppressedPeaks(std::ostream& out) { - const auto& fragment = mPmemory->fragment; + const auto& frag = mPmemory->frag; const auto nPeaks = mPmemory->counters.nPeaks; - out << "\nClusterer - NoiseSuppression - Sector " << mISector << " - Fragment " << fragment.index << mISector << "\n"; + out << "\nClusterer - NoiseSuppression - Sector " << mISector << " - Fragment " << frag.index << mISector << "\n"; for (uint32_t i = 0; i < nPeaks; i++) { out << int32_t{mPisPeak[i]}; if ((i + 1) % 100 == 0) { @@ -142,10 +142,10 @@ void GPUTPCClusterFinder::DumpSuppressedPeaks(std::ostream& out) void GPUTPCClusterFinder::DumpSuppressedPeaksCompacted(std::ostream& out) { - const auto& fragment = mPmemory->fragment; + const auto& frag = mPmemory->frag; const auto nPeaks = mPmemory->counters.nClusters; - out << "\nClusterer - Noise Suppression Peaks Compacted - Sector " << mISector << " - Fragment " << fragment.index << ": " << nPeaks << "\n"; + out << "\nClusterer - Noise Suppression Peaks Compacted - Sector " << mISector << " - Fragment " << frag.index << ": " << nPeaks << "\n"; for (size_t i = 0; i < nPeaks; i++) { const auto& peak = mPfilteredPeakPositions[i]; out << peak.time() << " " << int32_t{peak.pad()} << " " << int32_t{peak.row()} << "\n"; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx index 549be70fa8a1e..e37ef5dbc454d 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx @@ -49,7 +49,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcounters.nClusters - 1)] > 0) : 1); - GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, isAccepted); + GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->frag, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, isAccepted); } template <> @@ -357,7 +357,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).isOverlap(peak.time())) { + if ((clusterer.mPmemory->frag).isOverlap(peak.time())) { if (clusterer.mPclusterPosInRow) { clusterer.mPclusterPosInRow[full_glo_idx] = clusterer.mNMaxClusterPerRow; } @@ -381,7 +381,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, notSingleTime ? clustererNN.mOutputDataReg1_32[model_output_index + 3] : 0.f, clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -392,7 +392,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, notSingleTime ? clustererNN.mOutputDataReg1_16[model_output_index + 3].ToFloat() : 0.f, clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -553,7 +553,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).isOverlap(peak.time())) { + if ((clusterer.mPmemory->frag).isOverlap(peak.time())) { if (clusterer.mPclusterPosInRow) { clusterer.mPclusterPosInRow[full_glo_idx] = clusterer.mNMaxClusterPerRow; } @@ -569,7 +569,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_32[model_output_index + 6], clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -580,7 +580,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_16[model_output_index + 6].ToFloat(), clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -623,7 +623,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_32[model_output_index + 7], clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -634,7 +634,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_16[model_output_index + 7].ToFloat(), clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); From 0cacef4ccebf2722e49ba44c9109f3c5b9a23ffc Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 31/41] GPU: let the constant-address-space constants be built on Metal A constructor's implicit `this` is generic in MSL 4.1, and a generic pointer does not reach the constant address space, so a class-type constant at program scope could not be constructed at all: gpustd::bitset for the DetID and GlobalTrackID masks, CfChargePos for INVALID_CHARGE_POS. MSL lets a member function be qualified with the address space of its `this`, so each of these gains a constant-qualified overload next to the existing one, alongside the copy constructor and the OpenCL __constant one that are already there for the same reason. Only the members actually called on a constant object need it. --- GPU/GPUTracking/TPCClusterFinder/CfChargePos.h | 8 ++++++++ GPU/Utils/GPUCommonBitSet.h | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h b/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h index 3f1265e6d0634..64aa0f6fcfe6d 100644 --- a/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h +++ b/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h @@ -34,6 +34,14 @@ struct CfChargePos { : gpad(tpcGlobalPadIdx(row, pad)), timePadded(t + GPUCF_PADDING_TIME) { } +#ifdef __METAL__ + // INVALID_CHARGE_POS below lives in the constant address space, which a + // generic `this` does not reach in MSL. + constexpr GPUhdi() CfChargePos(tpccf::Row row, tpccf::Pad pad, tpccf::TPCFragmentTime t) constant + : gpad(tpcGlobalPadIdx(row, pad)), timePadded(t + GPUCF_PADDING_TIME) + { + } +#endif GPUdi() CfChargePos(const tpccf::GlobalPad& p, const tpccf::TPCFragmentTime& t) : gpad(p), timePadded(t) {} diff --git a/GPU/Utils/GPUCommonBitSet.h b/GPU/Utils/GPUCommonBitSet.h index e35587ab60c7b..eebe00b2bcdcb 100644 --- a/GPU/Utils/GPUCommonBitSet.h +++ b/GPU/Utils/GPUCommonBitSet.h @@ -47,6 +47,11 @@ class bitset GPUdDefault() constexpr bitset(const __constant bitset&) = default; #endif // __OPENCL__ GPUd() constexpr bitset(uint32_t vv) : v(vv) {}; +#ifdef __METAL__ + // Objects in the constant address space are built and read through their own + // overloads: a generic `this` does not reach constant memory in MSL. + GPUd() constexpr bitset(uint32_t vv) constant : v(vv) {}; +#endif static GPUglobalconstexpr() uint32_t full_set = ((1ul << N) - 1ul); GPUd() constexpr bool all() const { return (v & full_set) == full_set; } @@ -83,6 +88,13 @@ class bitset GPUd() constexpr bool operator!=(const bitset b) const { return v != b.v; } GPUd() constexpr bool operator[](uint32_t i) const { return (v >> i) & 1u; } +#ifdef __METAL__ + GPUd() constexpr bitset operator|(const bitset b) constant { return v | b.v; } + GPUd() constexpr bitset operator&(const bitset b) constant { return v & b.v; } + GPUd() constexpr bool operator[](uint32_t i) constant { return (v >> i) & 1u; } + GPUd() constexpr bool any() constant { return v & full_set; } + GPUd() constexpr uint32_t to_ulong() constant { return v; } +#endif GPUd() constexpr uint32_t to_ulong() const { return v; } From 3c0884c35cebdae70e1b513b0a74f2c622198e90 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 32/41] GPU: make std::is_pointer address-space aware on Metal The shim's partial specialization on a bare T* does match a pointer type written out in full, but not one deduced from an argument, which carries its address space. RDHUtils uses std::is_pointer to keep its pointer overloads apart from its reference ones, so the reference template was instantiated for a pointer and dereferenced it as a struct. metal::is_pointer is address-space aware, so the Metal branch forwards to it. --- GPU/Common/GPUCommonTypeTraits.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/GPU/Common/GPUCommonTypeTraits.h b/GPU/Common/GPUCommonTypeTraits.h index 3f83e151b0f33..6d72f164771bc 100644 --- a/GPU/Common/GPUCommonTypeTraits.h +++ b/GPU/Common/GPUCommonTypeTraits.h @@ -114,7 +114,13 @@ struct is_pointer_t { }; template struct is_pointer { +#ifdef __METAL__ + // A bare T* partial specialization does not match a pointer type deduced from + // an argument, which carries its address space; metal::is_pointer does. + enum { value = metal::is_pointer::value }; +#else enum { value = is_pointer_t::type>::value }; +#endif }; template From 4838938ad13b206842d42da714c85be95e73a09d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:43 +0200 Subject: [PATCH 33/41] GPUTracking: take GPUdoubleValue in GetPadRowNumber It forwards straight to PadPlane::getPadRowNumber, which already takes the alias. --- GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h b/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h index 0867582fffa14..c0a0f897ca6a7 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h +++ b/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h @@ -41,7 +41,7 @@ class GPUTRDpadPlane : private o2::trd::PadPlane GPUd() float GetColPos(int32_t col) const { return getColPos(col); } GPUd() float GetNrows() const { return getNrows(); } GPUd() float GetNcols() const { return getNcols(); } - GPUd() int32_t GetPadRowNumber(double z) const { return getPadRowNumber(z); } + GPUd() int32_t GetPadRowNumber(o2::gpu::GPUdoubleValue z) const { return getPadRowNumber(z); } }; class GPUTRDGeometry : private o2::trd::GeometryFlat From f2c04e483664e4b2259e01cdec63012f03b4f5ef Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:45 +0200 Subject: [PATCH 34/41] GPUTracking: replace the merger's goto with a flag MSL supports neither goto nor labels. The label sat at the end of the loop body, so each jump was a continue for the outer loop that could not be written as one because it was issued from an inner loop. The flag is set there instead, breaks out of the loop it was raised in, and continues the outer one; where the jump came from two levels down it breaks twice. The k loop is left early exactly as before, and nothing between the old jumps and the old label ran then either. --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index aa4a8b8e252b5..4598e61993201 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1201,6 +1201,7 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in // PrintMergeGraph(track1, std::cout); // PrintMergeGraph(track2, std::cout); + bool nextTrack = false; while (track2->PrevSegmentNeighbour() >= 0) { track2 = &mSectorTrackInfos[track2->PrevSegmentNeighbour()]; } @@ -1211,26 +1212,41 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in while (track1->PrevSegmentNeighbour() >= 0) { track1 = &mSectorTrackInfos[track1->PrevSegmentNeighbour()]; if (track1 == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + continue; + } GPUCommonAlgorithm::swap(track1, track1Base); for (int32_t k = 0; k < 2; k++) { GPUTPCGMSectorTrack* tmp = track1Base; while (tmp->Neighbour(k) >= 0) { tmp = &mSectorTrackInfos[tmp->Neighbour(k)]; if (tmp == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + break; + } + } + if (nextTrack) { + continue; } while (track1->NextSegmentNeighbour() >= 0) { track1 = &mSectorTrackInfos[track1->NextSegmentNeighbour()]; if (track1 == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + continue; + } } else { while (track1->PrevSegmentNeighbour() >= 0) { track1 = &mSectorTrackInfos[track1->PrevSegmentNeighbour()]; @@ -1244,9 +1260,16 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in while (tmp->Neighbour(k) >= 0) { tmp = &mSectorTrackInfos[tmp->Neighbour(k)]; if (tmp == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + break; + } + } + if (nextTrack) { + continue; } float z1min, z1max, z2min, z2max; @@ -1318,7 +1341,6 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in } // GPUInfo("Result"); // PrintMergeGraph(track1, std::cout); - NextTrack:; } } } From 3e299bb89b906b133fcac467e153af53fa955d51 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:47 +0200 Subject: [PATCH 35/41] MathUtils: let bringTo* deduce the type of the call they forward to The parameter is a reference, so T is deduced with the address space attached, and forcing that same T on a by-value parameter is a substitution failure on Metal. Letting the inner call deduce its own type gives the same T everywhere else, where the address space is not part of it. --- Common/MathUtils/include/MathUtils/detail/trigonometric.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Common/MathUtils/include/MathUtils/detail/trigonometric.h b/Common/MathUtils/include/MathUtils/detail/trigonometric.h index 6e6c631f5197f..e0bf675d696cc 100644 --- a/Common/MathUtils/include/MathUtils/detail/trigonometric.h +++ b/Common/MathUtils/include/MathUtils/detail/trigonometric.h @@ -48,7 +48,7 @@ GPUhdi() T to02Pi(T phi) template GPUhdi() void bringTo02Pi(T& phi) { - phi = to02Pi(phi); + phi = to02Pi(phi); } template @@ -68,7 +68,7 @@ inline T to02PiGen(T phi) template inline void bringTo02PiGen(T& phi) { - phi = to02PiGen(phi); + phi = to02PiGen(phi); } template @@ -87,7 +87,7 @@ GPUhdi() T toPMPi(T phi) template GPUhdi() void bringToPMPi(T& phi) { - phi = toPMPi(phi); + phi = toPMPi(phi); } template @@ -107,7 +107,7 @@ inline T toPMPiGen(T phi) template inline void bringToPMPiGen(T& phi) { - phi = toPMPiGen(phi); + phi = toPMPiGen(phi); } #if defined(__OPENCL__) || defined(__METAL__) // TODO: get rid of that stupid workaround for OpenCL template address spaces From 5e1b1f9b407769d20f46bd12e0cb7f954f67a544 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:49 +0200 Subject: [PATCH 36/41] DetectorsBase: keep the Propagator's double off Metal The double overloads of getFieldXYZ and getBz were already guarded where they are declared, but not where they are defined. The explicit double in the crossing-point helper becomes GPUdoubleValue, and the differences of nearly equal crossing and centre coordinates that feed atan2 become GPUdoubleCalc, matching the phiCross and dphi next to them. Both aliases are double everywhere except Metal, so the preprocessed source is unchanged for every other backend and for the host. --- Detectors/Base/src/Propagator.cxx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Detectors/Base/src/Propagator.cxx b/Detectors/Base/src/Propagator.cxx index c86e2b202d58a..78163ed019eb7 100644 --- a/Detectors/Base/src/Propagator.cxx +++ b/Detectors/Base/src/Propagator.cxx @@ -586,7 +586,7 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, GPUdoubleCalc phiCross[2] = {}, dphi[2] = {}; auto curv = track.getCurvature(bz); bool clockwise = curv < 0; // q+ in B+ or q- in B- goes clockwise - auto phiLoc = math_utils::detail::asin(track.getSnp()); + auto phiLoc = math_utils::detail::asin(track.getSnp()); auto phi0 = phiLoc + track.getAlpha(); o2::math_utils::detail::bringTo02Pi(phi0); for (int i = 0; i < cross.nDCA; i++) { @@ -594,12 +594,12 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, // == angle of the tangential to track circle at the crossing point X,Y // == normal to the radial vector from the track circle center {X-cX, Y-cY} // i.e. the angle of the vector {Y-cY, -(X-cx)} - auto normX = double(cross.yDCA[i]) - double(traux.yC), normY = -(double(cross.xDCA[i]) - double(traux.xC)); + auto normX = o2::gpu::GPUdoubleCalc(cross.yDCA[i]) - o2::gpu::GPUdoubleCalc(traux.yC), normY = -(o2::gpu::GPUdoubleCalc(cross.xDCA[i]) - o2::gpu::GPUdoubleCalc(traux.xC)); if (!clockwise) { normX = -normX; normY = -normY; } - phiCross[i] = math_utils::detail::atan2(normY, normX); + phiCross[i] = math_utils::detail::atan2(normY, normX); o2::math_utils::detail::bringTo02Pi(phiCross[i]); dphi[i] = phiCross[i] - phi0; if (dphi[i] > o2::constants::math::PI) { @@ -615,7 +615,7 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, auto phiLocFin = phiLoc + deltaPhi; // case1 if (math_utils::detail::abs(phiLocFin) < MaxPhiLocSafe) { // just 1 step propagation - auto deltaX = (math_utils::detail::sin(phiLocFin) - track.getSnp()) / track.getCurvature(bz); + auto deltaX = (math_utils::detail::sin(phiLocFin) - track.getSnp()) / track.getCurvature(bz); if (!propagateTo(track, track.getX() + deltaX, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr)) { return false; } @@ -638,7 +638,7 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, // propagate to phiLoc = +-MaxPhiLocSafe auto tgtPhiLoc = deltaPhi > 0 ? MaxPhiLocSafe : -MaxPhiLocSafe; - auto deltaX = (math_utils::detail::sin(tgtPhiLoc) - track.getSnp()) / track.getCurvature(bz); + auto deltaX = (math_utils::detail::sin(tgtPhiLoc) - track.getSnp()) / track.getCurvature(bz); if (!propagateTo(track, track.getX() + deltaX, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr)) { return false; } @@ -1094,11 +1094,13 @@ GPUd() void PropagatorImpl::getFieldXYZ(const math_utils::Point3D(xyz, bxyz); } +#ifndef __METAL__ // MSL has no double; the float twin remains template GPUd() void PropagatorImpl::getFieldXYZ(const math_utils::Point3D xyz, double* bxyz) const { getFieldXYZImpl(xyz, bxyz); } +#endif template GPUd() float PropagatorImpl::getBz(const math_utils::Point3D xyz) const @@ -1106,11 +1108,13 @@ GPUd() float PropagatorImpl::getBz(const math_utils::Point3D xyz return getBzImpl(xyz); } +#ifndef __METAL__ // MSL has no double; the float twin remains template GPUd() double PropagatorImpl::getBz(const math_utils::Point3D xyz) const { return getBzImpl(xyz); } +#endif namespace o2::base { From 18c429abcea96322ea66bdbdedd5cb337b135eaa Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:04:51 +0200 Subject: [PATCH 37/41] MathUtils: materialise the product in SMatrixGPU's operator*= Unlike ROOT's SMatrix, where the product of two matrices is a matrix, SMatrixGPU returns a lazy expression, and every element of that expression reads the whole of the left operand. Assigning it back element by element therefore reads values that have already been overwritten. It also did not compile: the expression matched the generic operator= that copies the representation, which an expression does not have. This was unreachable until now, since nothing instantiated a matrix multiply assignment in device code. --- Common/MathUtils/include/MathUtils/SMatrixGPU.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Common/MathUtils/include/MathUtils/SMatrixGPU.h b/Common/MathUtils/include/MathUtils/SMatrixGPU.h index 21b3f3ca30406..497a142a1902c 100644 --- a/Common/MathUtils/include/MathUtils/SMatrixGPU.h +++ b/Common/MathUtils/include/MathUtils/SMatrixGPU.h @@ -1433,14 +1433,18 @@ template template GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const SMatrixGPU& rhs) { - return operator=(*this* rhs); + // the product is an expression evaluated element by element, and every element + // of it reads the whole of *this, so it has to be materialised first + const SMatrixGPU tmp(*this * rhs); + return operator=(tmp); } template template GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const Expr& rhs) { - return operator=(*this* rhs); + const SMatrixGPU tmp(*this * rhs); + return operator=(tmp); } template From c1c86e4135d5c2f93ee80ae860b9d7767ab62c7a Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 38/41] ReconstructionDataFormats: express the SMatrix aliases in GPUdoubleCalc MatrixDSym5 and MatrixD5 were kept off Metal because SMatrix cannot be named there, which left the track-to-track chi2 and update out of the device build. Spelling them in GPUdoubleCalc brings them back: it is double on every other backend and on the host, so the aliases and everything using them are unchanged there, and on Metal they become the compensated two-float type that the rest of this file already uses. The two remaining explicit doubles go the same way, and the SMatrix written out in full is just MatrixD5. --- .../TrackParametrizationWithError.h | 13 +++---------- .../src/TrackParametrizationWithError.cxx | 4 ++-- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h index 2af42953e43fe..d120e4c15077f 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h @@ -17,6 +17,7 @@ #ifndef INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKPARAMETRIZATIONWITHERROR_H_ #define INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKPARAMETRIZATIONWITHERROR_H_ +#include "GPUCommonDouble.h" #include "ReconstructionDataFormats/TrackParametrization.h" #include @@ -41,10 +42,8 @@ class TrackParametrizationWithError : public TrackParametrization #endif using covMat_t = std::array; -#ifndef __METAL__ - using MatrixDSym5 = o2::math_utils::SMatrix>; - using MatrixD5 = o2::math_utils::SMatrix>; -#endif + using MatrixDSym5 = o2::math_utils::SMatrix>; + using MatrixD5 = o2::math_utils::SMatrix>; GPUhd() TrackParametrizationWithError(); GPUd() TrackParametrizationWithError(value_t x, value_t alpha, const params_t& par, const covMat_t& cov, int charge = 1, const PID pid = PID::Pion); @@ -113,18 +112,12 @@ class TrackParametrizationWithError : public TrackParametrization template GPUd() value_t getPredictedChi2Quiet(const BaseCluster& p) const; -#ifndef __METAL__ GPUd() void buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const; -#endif -#ifndef __METAL__ GPUd() value_t getPredictedChi2(const TrackParametrizationWithError& rhs, MatrixDSym5& covToSet) const; -#endif GPUd() value_t getPredictedChi2(const TrackParametrizationWithError& rhs) const; GPUd() value_t getPredictedChi2Fast(const TrackParametrizationWithError& rhs) const; GPUd() value_t getPredictedChi2Quiet(const TrackParametrizationWithError& rhs) const; -#ifndef __METAL__ GPUd() bool update(const TrackParametrizationWithError& rhs, const MatrixDSym5& covInv); -#endif GPUd() bool update(const TrackParametrizationWithError& rhs); GPUd() bool update(const dim2_t& p, const dim3_t& cov); diff --git a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx index 284658bb7bb2b..7b19e7a83f867 100644 --- a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx +++ b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx @@ -1174,7 +1174,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T // chi2 = d^T C^-1 d = sum_i y_i^2 / D_i with y from the forward substitution L y = d GPUdoubleCalc chi2 = 0., y[kNParams]; for (int i = 0; i < kNParams; i++) { - GPUdoubleCalc s = double(this->getParam(i)) - double(rhs.getParam(i)); + GPUdoubleCalc s = GPUdoubleCalc(this->getParam(i)) - GPUdoubleCalc(rhs.getParam(i)); for (int k = 0; k < i; k++) { s -= lmat[i][k] * y[k]; } @@ -1287,7 +1287,7 @@ GPUd() bool TrackParametrizationWithError::update(const TrackParametriz } // updated covariance: Cov0 = Cov0 - K*Cov0 - matK *= o2::math_utils::SMatrix>(matC0); + matK *= MatrixD5(matC0); mC[kSigY2] -= matK(kY, kY); mC[kSigZY] -= matK(kZ, kY); mC[kSigZ2] -= matK(kZ, kZ); From da3986b2ad476e09ad2cce2450a134ef2bf5ca34 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 39/41] GPU: hand the kernel pointers to Thread() as generic pointers A kernel's buffers are device memory and the address arrives as an integer, but the Thread() entry points take their pointer arguments unannotated, which is the generic address space. Forwarding a device pointer made the call deduce a device pointer for Args..., which matched no explicit specialisation, so the kernels linked against a Thread() that is declared and never defined. The cast goes through device first rather than straight from the integer, so the generic pointer is formed by the normal conversion. --- GPU/GPUTracking/Definitions/GPUDef.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GPU/GPUTracking/Definitions/GPUDef.h b/GPU/GPUTracking/Definitions/GPUDef.h index b5b538e96a833..ece7cc57f7096 100644 --- a/GPU/GPUTracking/Definitions/GPUDef.h +++ b/GPU/GPUTracking/Definitions/GPUDef.h @@ -34,7 +34,9 @@ // As for OpenCL, pointers travel as a 64-bit address: a pointer to a derived // class is not a valid kernel argument type in MSL either. #define GPUPtr1(idx, a, b) constant uint64_t& b [[buffer(idx)]] - #define GPUPtr2(a, b) ((device a) b) + // through device and then to generic: the kernel's own buffers are device + // memory, but the Thread() entry points take the pointer unannotated + #define GPUPtr2(a, b) ((a)((device a)(b))) #define GPUArg1(idx, a, b) constant a& b [[buffer(idx)]] #else #define GPUPtr1(idx, a, b) a b From b53be27760a125dd055226126344c14dfe7e236a Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 40/41] GPU: check that the toolchain can compile MSL 4.1 before enabling Metal The Metal frameworks are present on every macOS, so finding them said nothing about whether the backend can be built. It needs MSL 4.1, which arrives with macOS 27 and its toolchain, and a library compiled as MSL 4.1 only loads on macOS 27 and later. The deployment target has no say in this: the -std= flag is what picks the target OS, and MACOSX_DEPLOYMENT_TARGET and -mmacosx-version-min are both ignored by the Metal compiler. Compiling a three-line kernel answers the question directly. AUTO now turns Metal off on an older toolchain instead of failing somewhere in the middle of the build, and an explicit ENABLE_METAL=ON says why it cannot be honoured. --- dependencies/FindO2GPU.cmake | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index 956272c4600ee..801bcd17b691d 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -10,7 +10,7 @@ # or submit itself to any jurisdiction. # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources -# FindO2GPU.cmake Version 20 +# FindO2GPU.cmake Version 21 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual) set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908) @@ -450,12 +450,29 @@ if(ENABLE_METAL) find_library(COREFOUNDATION_FRAMEWORK CoreFoundation) find_library(FOUNDATION_FRAMEWORK Foundation) find_library(QUARTZCORE_FRAMEWORK QuartzCore) - if(METAL_FRAMEWORK AND COREFOUNDATION_FRAMEWORK AND FOUNDATION_FRAMEWORK AND QUARTZCORE_FRAMEWORK) + # The frameworks are there on every macOS, but the backend needs MSL 4.1, which + # is macOS 27 and its toolchain, so ask the compiler instead of assuming. A + # library built as MSL 4.1 also only loads on macOS 27 and later. + if(NOT DEFINED GPUCA_METAL_MSL41) + set(GPUCA_METAL_PROBE "${CMAKE_CURRENT_BINARY_DIR}/metal_msl41_probe.metal") + file(WRITE "${GPUCA_METAL_PROBE}" "#include \nkernel void probe(device float* o [[buffer(0)]], uint i [[thread_position_in_grid]]) { o[i] = o[i] * 2.0f; }\n") + execute_process(COMMAND xcrun -sdk macosx metal -std=metal4.1 -c "${GPUCA_METAL_PROBE}" -o "${GPUCA_METAL_PROBE}.air" + RESULT_VARIABLE GPUCA_METAL_PROBE_RESULT OUTPUT_QUIET ERROR_QUIET) + if(GPUCA_METAL_PROBE_RESULT EQUAL 0) + set(GPUCA_METAL_MSL41 ON CACHE INTERNAL "Metal toolchain compiles MSL 4.1") + else() + set(GPUCA_METAL_MSL41 OFF CACHE INTERNAL "Metal toolchain compiles MSL 4.1") + endif() + endif() + if(METAL_FRAMEWORK AND COREFOUNDATION_FRAMEWORK AND FOUNDATION_FRAMEWORK AND QUARTZCORE_FRAMEWORK AND GPUCA_METAL_MSL41) set(METAL_ENABLED ON) set(METAL_FRAMEWORKS ${METAL_FRAMEWORK} ${COREFOUNDATION_FRAMEWORK} ${FOUNDATION_FRAMEWORK} ${QUARTZCORE_FRAMEWORK}) message(STATUS "Found Metal frameworks") elseif(NOT ENABLE_METAL STREQUAL "AUTO") + if(NOT GPUCA_METAL_MSL41) + message(FATAL_ERROR "The Metal backend needs MSL 4.1: this toolchain rejects -std=metal4.1, and the result would need macOS 27 to run") + endif() message(FATAL_ERROR "Metal frameworks not available") else() set(METAL_ENABLED OFF) From 424fb0762f4e89b727f6eecfb544c3ab2b33b27d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:01:45 +0200 Subject: [PATCH 41/41] GPUTracking: compile the real kernels into the Metal library GPUReconstructionMETAL.metal had the device headers and the kernel list behind #if 0, with a comment saying they do not compile as MSL yet. They do now, so the entry point includes them and the library it produces contains the 96 kernels rather than nothing. --- .../Base/metal/GPUReconstructionMETAL.metal | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal index 2bb0c7d9b042c..004220e7e268d 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal @@ -47,16 +47,8 @@ using namespace metal; #include "GPUCommonTypeTraits.h" #include "GPUCommonArray.h" -// The remaining headers do not compile as MSL yet, but nothing structural is in -// the way: with the pragma above and the untyped constant buffer below, the -// kernel list expands to all 104 entry points, with no derived-class and no -// kernel-argument-type errors left. What fails is the bodies, and it is bulk -// work rather than a missing language feature -- MSL has no double, and every -// namespace-scope constexpr needs GPUglobalconstexpr(). -#if 0 #include "GPUConstantMem.h" #include "GPUReconstructionIncludesDeviceAll.h" -#endif // --- Kernel list expansion --------------------------------------------------- #define GPUCA_KRNL(...) GPUCA_KRNLGPU(__VA_ARGS__) @@ -80,10 +72,7 @@ using namespace metal; , uint _metalTPerTg [[threads_per_threadgroup]] \ , uint _metalTgPerG [[threadgroups_per_grid]] -// Include the actual kernels, once the headers above compile as MSL. -#if 0 #include "GPUReconstructionKernelList.h" -#endif // clang-format on #pragma clang diagnostic pop