diff --git a/Common/Constants/include/CommonConstants/LHCConstants.h b/Common/Constants/include/CommonConstants/LHCConstants.h index 1582f2166ca0f..2cdd0b5118731 100644 --- a/Common/Constants/include/CommonConstants/LHCConstants.h +++ b/Common/Constants/include/CommonConstants/LHCConstants.h @@ -16,6 +16,8 @@ #ifndef ALICEO2_LHCCONSTANTS_H_ #define ALICEO2_LHCCONSTANTS_H_ +#include "GPUCommonDouble.h" + #include "GPUCommonDef.h" namespace o2 @@ -31,12 +33,12 @@ enum BeamDirection : int { BeamA, // beamA = beam 0, InteractingBC = -1 // as used in the BunchFilling class }; GPUglobalconstexpr() int LHCMaxBunches = 3564; // max N bunches -constexpr double LHCRFFreq = 400.789e6; // LHC RF frequency in Hz -constexpr double LHCBunchSpacingNS = 10 * 1.e9 / LHCRFFreq; // bunch spacing in ns (10 RFbuckets) -constexpr double LHCOrbitNS = LHCMaxBunches * LHCBunchSpacingNS; // orbit duration in ns -constexpr double LHCRevFreq = 1.e9 / LHCOrbitNS; // revolution frequency -constexpr double LHCBunchSpacingMUS = LHCBunchSpacingNS * 1e-3; // bunch spacing in \mus (10 RFbuckets) -constexpr double LHCOrbitMUS = LHCOrbitNS * 1e-3; // orbit duration in \mus +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCRFFreq = 400.789e6; // LHC RF frequency in Hz +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCBunchSpacingNS = 10 * 1.e9 / LHCRFFreq; // bunch spacing in ns (10 RFbuckets) +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCOrbitNS = LHCMaxBunches * LHCBunchSpacingNS; // orbit duration in ns +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCRevFreq = 1.e9 / LHCOrbitNS; // revolution frequency +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCBunchSpacingMUS = LHCBunchSpacingNS * 1e-3; // bunch spacing in \mus (10 RFbuckets) +GPUglobalconstexpr() o2::gpu::GPUdoubleValue LHCOrbitMUS = LHCOrbitNS * 1e-3; // orbit duration in \mus GPUglobalconstexpr() unsigned int MaxNOrbits = 0xffffffff; // Offsets of A, C beam bunches at P2 diff --git a/Common/Constants/include/CommonConstants/PhysicsConstants.h b/Common/Constants/include/CommonConstants/PhysicsConstants.h index 051fb6d2a6e89..86e5162e2b654 100644 --- a/Common/Constants/include/CommonConstants/PhysicsConstants.h +++ b/Common/Constants/include/CommonConstants/PhysicsConstants.h @@ -18,8 +18,19 @@ #ifndef ALICEO2_PHYSICSCONSTANTS_H_ #define ALICEO2_PHYSICSCONSTANTS_H_ +#include "GPUCommonDef.h" + namespace o2::constants::physics { +#ifdef __METAL__ +// MSL has no double. The masses only ever serve as compile-time initialisers, +// and the tables built from them (PID::sMasses) are float already, so nothing +// is lost. Every other backend, device included, keeps double. +using MassType = float; +#else +using MassType = double; +#endif + // particles masses // BEGINNING OF THE GENERATED BLOCK. @@ -102,153 +113,153 @@ enum Pdg { }; /// \brief Declarations of masses for additional particles -constexpr double MassEta = 0.547862; -constexpr double MassOmega = 0.78266; -constexpr double MassEtaPrime = 0.95778; -constexpr double MassB0 = 5.27966; -constexpr double MassB0Bar = 5.27966; -constexpr double MassBPlus = 5.27934; -constexpr double MassBCPlus = 6.27447; -constexpr double MassBS = 5.36692; -constexpr double MassBSBar = 5.36692; -constexpr double MassD0 = 1.86484; -constexpr double MassD0Bar = 1.86484; -constexpr double MassD0StarPlus = 2.272; -constexpr double MassD0Star0 = 2.343; -constexpr double MassD1Plus = 2.372; -constexpr double MassD10 = 2.412; -constexpr double MassD2StarPlus = 2.4601; -constexpr double MassD2Star0 = 2.4611; -constexpr double MassDMinus = 1.86966; -constexpr double MassDPlus = 1.86966; -constexpr double MassDS = 1.96835; -constexpr double MassDSBar = 1.96835; -constexpr double MassDSStar = 2.1122; -constexpr double MassDS1 = 2.53511; -constexpr double MassDS1Star2700 = 2.714; -constexpr double MassDS1Star2860 = 2.859; -constexpr double MassDS2Star = 2.5691; -constexpr double MassDS3Star2860 = 2.86; -constexpr double MassDStar = 2.01026; -constexpr double MassDStar0 = 2.00685; -constexpr double MassChiC1 = 3.51067; -constexpr double MassJPsi = 3.0969; -constexpr double MassLambdaB0 = 5.6196; -constexpr double MassLambdaCPlus = 2.28646; -constexpr double MassLambdaCPlus2860 = 2.8561; -constexpr double MassLambdaCPlus2880 = 2.8816; -constexpr double MassLambdaCPlus2940 = 2.9396; -constexpr double MassOmegaC0 = 2.6952; -constexpr double MassK0Star892 = 0.89555; -constexpr double MassKPlusStar892 = 0.89167; -constexpr double MassPhi = 1.019461; -constexpr double MassSigmaC0 = 2.45375; -constexpr double MassSigmaCPlusPlus = 2.45397; -constexpr double MassSigmaCStar0 = 2.51848; -constexpr double MassSigmaCStarPlusPlus = 2.51841; -constexpr double MassX3872 = 3.87165; -constexpr double MassXi0 = 1.31486; -constexpr double MassXiB0 = 5.7919; -constexpr double MassXiCCPlusPlus = 3.62155; -constexpr double MassXiCPlus = 2.46771; -constexpr double MassXiC0 = 2.47044; -constexpr double MassXiC3055Plus = 3.0559; -constexpr double MassXiC3080Plus = 3.0772; -constexpr double MassXiC3055_0 = 3.059; -constexpr double MassXiC3080_0 = 3.0799; -constexpr double MassDeuteron = 1.87561294257; -constexpr double MassTriton = 2.80892113298; -constexpr double MassHelium3 = 2.80839160743; -constexpr double MassAlpha = 3.7273794066; -constexpr double MassLithium4 = 3.7513; -constexpr double MassHyperTriton = 2.991134; -constexpr double MassHyperHydrogen4 = 3.922434; -constexpr double MassHyperHelium4 = 3.921728; -constexpr double MassHyperHelium5 = 4.839961; -constexpr double MassHyperHelium4Sigma = 3.995; -constexpr double MassLambda1520_Py = 1.5195; -constexpr double MassK1_1270_0 = 1.253; -constexpr double MassK1_1270Plus = 1.272; -constexpr double MassCDeuteron = 3.226; +GPUglobalconstexpr() MassType MassEta = 0.547862; +GPUglobalconstexpr() MassType MassOmega = 0.78266; +GPUglobalconstexpr() MassType MassEtaPrime = 0.95778; +GPUglobalconstexpr() MassType MassB0 = 5.27966; +GPUglobalconstexpr() MassType MassB0Bar = 5.27966; +GPUglobalconstexpr() MassType MassBPlus = 5.27934; +GPUglobalconstexpr() MassType MassBCPlus = 6.27447; +GPUglobalconstexpr() MassType MassBS = 5.36692; +GPUglobalconstexpr() MassType MassBSBar = 5.36692; +GPUglobalconstexpr() MassType MassD0 = 1.86484; +GPUglobalconstexpr() MassType MassD0Bar = 1.86484; +GPUglobalconstexpr() MassType MassD0StarPlus = 2.272; +GPUglobalconstexpr() MassType MassD0Star0 = 2.343; +GPUglobalconstexpr() MassType MassD1Plus = 2.372; +GPUglobalconstexpr() MassType MassD10 = 2.412; +GPUglobalconstexpr() MassType MassD2StarPlus = 2.4601; +GPUglobalconstexpr() MassType MassD2Star0 = 2.4611; +GPUglobalconstexpr() MassType MassDMinus = 1.86966; +GPUglobalconstexpr() MassType MassDPlus = 1.86966; +GPUglobalconstexpr() MassType MassDS = 1.96835; +GPUglobalconstexpr() MassType MassDSBar = 1.96835; +GPUglobalconstexpr() MassType MassDSStar = 2.1122; +GPUglobalconstexpr() MassType MassDS1 = 2.53511; +GPUglobalconstexpr() MassType MassDS1Star2700 = 2.714; +GPUglobalconstexpr() MassType MassDS1Star2860 = 2.859; +GPUglobalconstexpr() MassType MassDS2Star = 2.5691; +GPUglobalconstexpr() MassType MassDS3Star2860 = 2.86; +GPUglobalconstexpr() MassType MassDStar = 2.01026; +GPUglobalconstexpr() MassType MassDStar0 = 2.00685; +GPUglobalconstexpr() MassType MassChiC1 = 3.51067; +GPUglobalconstexpr() MassType MassJPsi = 3.0969; +GPUglobalconstexpr() MassType MassLambdaB0 = 5.6196; +GPUglobalconstexpr() MassType MassLambdaCPlus = 2.28646; +GPUglobalconstexpr() MassType MassLambdaCPlus2860 = 2.8561; +GPUglobalconstexpr() MassType MassLambdaCPlus2880 = 2.8816; +GPUglobalconstexpr() MassType MassLambdaCPlus2940 = 2.9396; +GPUglobalconstexpr() MassType MassOmegaC0 = 2.6952; +GPUglobalconstexpr() MassType MassK0Star892 = 0.89555; +GPUglobalconstexpr() MassType MassKPlusStar892 = 0.89167; +GPUglobalconstexpr() MassType MassPhi = 1.019461; +GPUglobalconstexpr() MassType MassSigmaC0 = 2.45375; +GPUglobalconstexpr() MassType MassSigmaCPlusPlus = 2.45397; +GPUglobalconstexpr() MassType MassSigmaCStar0 = 2.51848; +GPUglobalconstexpr() MassType MassSigmaCStarPlusPlus = 2.51841; +GPUglobalconstexpr() MassType MassX3872 = 3.87165; +GPUglobalconstexpr() MassType MassXi0 = 1.31486; +GPUglobalconstexpr() MassType MassXiB0 = 5.7919; +GPUglobalconstexpr() MassType MassXiCCPlusPlus = 3.62155; +GPUglobalconstexpr() MassType MassXiCPlus = 2.46771; +GPUglobalconstexpr() MassType MassXiC0 = 2.47044; +GPUglobalconstexpr() MassType MassXiC3055Plus = 3.0559; +GPUglobalconstexpr() MassType MassXiC3080Plus = 3.0772; +GPUglobalconstexpr() MassType MassXiC3055_0 = 3.059; +GPUglobalconstexpr() MassType MassXiC3080_0 = 3.0799; +GPUglobalconstexpr() MassType MassDeuteron = 1.87561294257; +GPUglobalconstexpr() MassType MassTriton = 2.80892113298; +GPUglobalconstexpr() MassType MassHelium3 = 2.80839160743; +GPUglobalconstexpr() MassType MassAlpha = 3.7273794066; +GPUglobalconstexpr() MassType MassLithium4 = 3.7513; +GPUglobalconstexpr() MassType MassHyperTriton = 2.991134; +GPUglobalconstexpr() MassType MassHyperHydrogen4 = 3.922434; +GPUglobalconstexpr() MassType MassHyperHelium4 = 3.921728; +GPUglobalconstexpr() MassType MassHyperHelium5 = 4.839961; +GPUglobalconstexpr() MassType MassHyperHelium4Sigma = 3.995; +GPUglobalconstexpr() MassType MassLambda1520_Py = 1.5195; +GPUglobalconstexpr() MassType MassK1_1270_0 = 1.253; +GPUglobalconstexpr() MassType MassK1_1270Plus = 1.272; +GPUglobalconstexpr() MassType MassCDeuteron = 3.226; /// \brief Declarations of masses for particles in ROOT PDG_t -constexpr double MassDown = 0.00467; -constexpr double MassDownBar = 0.00467; -constexpr double MassUp = 0.00216; -constexpr double MassUpBar = 0.00216; -constexpr double MassStrange = 0.0934; -constexpr double MassStrangeBar = 0.0934; -constexpr double MassCharm = 1.27; -constexpr double MassCharmBar = 1.27; -constexpr double MassBottom = 4.18; -constexpr double MassBottomBar = 4.18; -constexpr double MassTop = 172.5; -constexpr double MassTopBar = 172.5; -constexpr double MassGluon = 0.0; -constexpr double MassElectron = 0.000510999; -constexpr double MassPositron = 0.000510999; -constexpr double MassNuE = 0.0; -constexpr double MassNuEBar = 0.0; -constexpr double MassMuonMinus = 0.1056584; -constexpr double MassMuonPlus = 0.1056584; -constexpr double MassNuMu = 0.0; -constexpr double MassNuMuBar = 0.0; -constexpr double MassTauMinus = 1.77686; -constexpr double MassTauPlus = 1.77686; -constexpr double MassNuTau = 0.0; -constexpr double MassNuTauBar = 0.0; -constexpr double MassGamma = 0.0; -constexpr double MassZ0 = 91.1876; -constexpr double MassWPlus = 80.377; -constexpr double MassWMinus = 80.377; -constexpr double MassPi0 = 0.1349768; -constexpr double MassK0Long = 0.497611; -constexpr double MassPiPlus = 0.1395704; -constexpr double MassPiMinus = 0.1395704; -constexpr double MassProton = 0.9382721; -constexpr double MassProtonBar = 0.9382721; -constexpr double MassNeutron = 0.9395654; -constexpr double MassNeutronBar = 0.9395654; -constexpr double MassK0Short = 0.497611; -constexpr double MassK0 = 0.497611; -constexpr double MassK0Bar = 0.497611; -constexpr double MassKPlus = 0.493677; -constexpr double MassKMinus = 0.493677; -constexpr double MassLambda0 = 1.115683; -constexpr double MassLambda0Bar = 1.115683; -constexpr double MassLambda1520 = 1.519; -constexpr double MassSigmaMinus = 1.197449; -constexpr double MassSigmaBarPlus = 1.197449; -constexpr double MassSigmaPlus = 1.18937; -constexpr double MassSigmaBarMinus = 1.18937; -constexpr double MassSigma0 = 1.192642; -constexpr double MassSigma0Bar = 1.192642; -constexpr double MassXiMinus = 1.32171; -constexpr double MassXiPlusBar = 1.32171; -constexpr double MassOmegaMinus = 1.67245; -constexpr double MassOmegaPlusBar = 1.67245; +GPUglobalconstexpr() MassType MassDown = 0.00467; +GPUglobalconstexpr() MassType MassDownBar = 0.00467; +GPUglobalconstexpr() MassType MassUp = 0.00216; +GPUglobalconstexpr() MassType MassUpBar = 0.00216; +GPUglobalconstexpr() MassType MassStrange = 0.0934; +GPUglobalconstexpr() MassType MassStrangeBar = 0.0934; +GPUglobalconstexpr() MassType MassCharm = 1.27; +GPUglobalconstexpr() MassType MassCharmBar = 1.27; +GPUglobalconstexpr() MassType MassBottom = 4.18; +GPUglobalconstexpr() MassType MassBottomBar = 4.18; +GPUglobalconstexpr() MassType MassTop = 172.5; +GPUglobalconstexpr() MassType MassTopBar = 172.5; +GPUglobalconstexpr() MassType MassGluon = 0.0; +GPUglobalconstexpr() MassType MassElectron = 0.000510999; +GPUglobalconstexpr() MassType MassPositron = 0.000510999; +GPUglobalconstexpr() MassType MassNuE = 0.0; +GPUglobalconstexpr() MassType MassNuEBar = 0.0; +GPUglobalconstexpr() MassType MassMuonMinus = 0.1056584; +GPUglobalconstexpr() MassType MassMuonPlus = 0.1056584; +GPUglobalconstexpr() MassType MassNuMu = 0.0; +GPUglobalconstexpr() MassType MassNuMuBar = 0.0; +GPUglobalconstexpr() MassType MassTauMinus = 1.77686; +GPUglobalconstexpr() MassType MassTauPlus = 1.77686; +GPUglobalconstexpr() MassType MassNuTau = 0.0; +GPUglobalconstexpr() MassType MassNuTauBar = 0.0; +GPUglobalconstexpr() MassType MassGamma = 0.0; +GPUglobalconstexpr() MassType MassZ0 = 91.1876; +GPUglobalconstexpr() MassType MassWPlus = 80.377; +GPUglobalconstexpr() MassType MassWMinus = 80.377; +GPUglobalconstexpr() MassType MassPi0 = 0.1349768; +GPUglobalconstexpr() MassType MassK0Long = 0.497611; +GPUglobalconstexpr() MassType MassPiPlus = 0.1395704; +GPUglobalconstexpr() MassType MassPiMinus = 0.1395704; +GPUglobalconstexpr() MassType MassProton = 0.9382721; +GPUglobalconstexpr() MassType MassProtonBar = 0.9382721; +GPUglobalconstexpr() MassType MassNeutron = 0.9395654; +GPUglobalconstexpr() MassType MassNeutronBar = 0.9395654; +GPUglobalconstexpr() MassType MassK0Short = 0.497611; +GPUglobalconstexpr() MassType MassK0 = 0.497611; +GPUglobalconstexpr() MassType MassK0Bar = 0.497611; +GPUglobalconstexpr() MassType MassKPlus = 0.493677; +GPUglobalconstexpr() MassType MassKMinus = 0.493677; +GPUglobalconstexpr() MassType MassLambda0 = 1.115683; +GPUglobalconstexpr() MassType MassLambda0Bar = 1.115683; +GPUglobalconstexpr() MassType MassLambda1520 = 1.519; +GPUglobalconstexpr() MassType MassSigmaMinus = 1.197449; +GPUglobalconstexpr() MassType MassSigmaBarPlus = 1.197449; +GPUglobalconstexpr() MassType MassSigmaPlus = 1.18937; +GPUglobalconstexpr() MassType MassSigmaBarMinus = 1.18937; +GPUglobalconstexpr() MassType MassSigma0 = 1.192642; +GPUglobalconstexpr() MassType MassSigma0Bar = 1.192642; +GPUglobalconstexpr() MassType MassXiMinus = 1.32171; +GPUglobalconstexpr() MassType MassXiPlusBar = 1.32171; +GPUglobalconstexpr() MassType MassOmegaMinus = 1.67245; +GPUglobalconstexpr() MassType MassOmegaPlusBar = 1.67245; // END OF THE GENERATED BLOCK // legacy names -constexpr double MassPhoton = MassGamma; -constexpr double MassMuon = MassMuonMinus; -constexpr double MassPionCharged = MassPiPlus; -constexpr double MassPionNeutral = MassPi0; -constexpr double MassKaonCharged = MassKPlus; -constexpr double MassKaonNeutral = MassK0; -constexpr double MassLambda = MassLambda0; -constexpr double MassHyperhydrog4 = MassHyperHydrogen4; -constexpr double MassHyperhelium4 = MassHyperHelium4; -constexpr double MassHyperhelium4sigma = MassHyperHelium4Sigma; +GPUglobalconstexpr() MassType MassPhoton = MassGamma; +GPUglobalconstexpr() MassType MassMuon = MassMuonMinus; +GPUglobalconstexpr() MassType MassPionCharged = MassPiPlus; +GPUglobalconstexpr() MassType MassPionNeutral = MassPi0; +GPUglobalconstexpr() MassType MassKaonCharged = MassKPlus; +GPUglobalconstexpr() MassType MassKaonNeutral = MassK0; +GPUglobalconstexpr() MassType MassLambda = MassLambda0; +GPUglobalconstexpr() MassType MassHyperhydrog4 = MassHyperHydrogen4; +GPUglobalconstexpr() MassType MassHyperhelium4 = MassHyperHelium4; +GPUglobalconstexpr() MassType MassHyperhelium4sigma = MassHyperHelium4Sigma; // Light speed -constexpr float LightSpeedCm2S = 299792458.e2; // C in cm/s -constexpr float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns -constexpr float LightSpeedCm2PS = LightSpeedCm2S * 1e-12; // C in cm/ps +GPUglobalconstexpr() float LightSpeedCm2S = 299792458.e2; // C in cm/s +GPUglobalconstexpr() float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns +GPUglobalconstexpr() float LightSpeedCm2PS = LightSpeedCm2S * 1e-12; // C in cm/ps // Light speed inverse -constexpr float invLightSpeedCm2PS = 1. / LightSpeedCm2PS; // 1/C in ps/cm +GPUglobalconstexpr() float invLightSpeedCm2PS = 1. / LightSpeedCm2PS; // 1/C in ps/cm } // namespace o2::constants::physics diff --git a/Common/Constants/include/CommonConstants/make_pdg_header.py b/Common/Constants/include/CommonConstants/make_pdg_header.py index b2dac688fd098..512f3f9223a52 100755 --- a/Common/Constants/include/CommonConstants/make_pdg_header.py +++ b/Common/Constants/include/CommonConstants/make_pdg_header.py @@ -169,9 +169,9 @@ def mass(code): return dbPdg.Mass(code, success) -def declare_mass(pdg, mass_type="double") -> str: +def declare_mass(pdg, mass_type="MassType") -> str: """Returns a C++ declaration of a particle mass constant.""" - return f"constexpr {mass_type} Mass{pdg.name[1:]} = {mass(pdg.value)};" + return f"GPUglobalconstexpr() {mass_type} Mass{pdg.name[1:]} = {mass(pdg.value)};" def main(): diff --git a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h index 75e146d872cd1..38ee5f6f7b5ba 100644 --- a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h +++ b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h @@ -58,19 +58,19 @@ struct Float16Impl { /// /// /// - GPUd() constexpr static uint16_t ToUint16Impl(float v) noexcept; + GPUd() constexpr static uint16_t ToUint16Impl(float v) GPUnoexcept(); /// /// Converts float16 to float /// /// float representation of float16 value - GPUd() float ToFloatImpl() const noexcept; + GPUd() float ToFloatImpl() const GPUnoexcept(); /// /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() uint16_t AbsImpl() const noexcept + GPUd() uint16_t AbsImpl() const GPUnoexcept() { return static_cast(val & ~kSignMask); } @@ -79,24 +79,24 @@ struct Float16Impl { /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() uint16_t NegateImpl() const noexcept + GPUd() uint16_t NegateImpl() const GPUnoexcept() { return IsNaN() ? val : static_cast(val ^ kSignMask); } public: // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7C00U; - static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; - static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; - static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; - static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; - static constexpr uint16_t kEpsilonBits = 0x4170U; - static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number - static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number - static constexpr uint16_t kOneBits = 0x3C00U; - static constexpr uint16_t kMinusOneBits = 0xBC00U; + static GPUglobalconstexpr() uint16_t kSignMask = 0x8000U; + static GPUglobalconstexpr() uint16_t kBiasedExponentMask = 0x7C00U; + static GPUglobalconstexpr() uint16_t kPositiveInfinityBits = 0x7C00U; + static GPUglobalconstexpr() uint16_t kNegativeInfinityBits = 0xFC00U; + static GPUglobalconstexpr() uint16_t kPositiveQNaNBits = 0x7E00U; + static GPUglobalconstexpr() uint16_t kNegativeQNaNBits = 0xFE00U; + static GPUglobalconstexpr() uint16_t kEpsilonBits = 0x4170U; + static GPUglobalconstexpr() uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number + static GPUglobalconstexpr() uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number + static GPUglobalconstexpr() uint16_t kOneBits = 0x3C00U; + static GPUglobalconstexpr() uint16_t kMinusOneBits = 0xBC00U; uint16_t val{0}; @@ -106,7 +106,7 @@ struct Float16Impl { /// Checks if the value is negative /// /// true if negative - GPUd() bool IsNegative() const noexcept + GPUd() bool IsNegative() const GPUnoexcept() { return static_cast(val) < 0; } @@ -115,7 +115,7 @@ struct Float16Impl { /// Tests if the value is NaN /// /// true if NaN - GPUd() bool IsNaN() const noexcept + GPUd() bool IsNaN() const GPUnoexcept() { return AbsImpl() > kPositiveInfinityBits; } @@ -124,7 +124,7 @@ struct Float16Impl { /// Tests if the value is finite /// /// true if finite - GPUd() bool IsFinite() const noexcept + GPUd() bool IsFinite() const GPUnoexcept() { return AbsImpl() < kPositiveInfinityBits; } @@ -133,7 +133,7 @@ struct Float16Impl { /// Tests if the value represents positive infinity. /// /// true if positive infinity - GPUd() bool IsPositiveInfinity() const noexcept + GPUd() bool IsPositiveInfinity() const GPUnoexcept() { return val == kPositiveInfinityBits; } @@ -142,7 +142,7 @@ struct Float16Impl { /// Tests if the value represents negative infinity /// /// true if negative infinity - GPUd() bool IsNegativeInfinity() const noexcept + GPUd() bool IsNegativeInfinity() const GPUnoexcept() { return val == kNegativeInfinityBits; } @@ -151,7 +151,7 @@ struct Float16Impl { /// Tests if the value is either positive or negative infinity. /// /// True if absolute value is infinity - GPUd() bool IsInfinity() const noexcept + GPUd() bool IsInfinity() const GPUnoexcept() { return AbsImpl() == kPositiveInfinityBits; } @@ -160,7 +160,7 @@ struct Float16Impl { /// Tests if the value is NaN or zero. Useful for comparisons. /// /// True if NaN or zero. - GPUd() bool IsNaNOrZero() const noexcept + GPUd() bool IsNaNOrZero() const GPUnoexcept() { auto abs = AbsImpl(); return (abs == 0 || abs > kPositiveInfinityBits); @@ -170,7 +170,7 @@ struct Float16Impl { /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). /// /// True if so - GPUd() bool IsNormal() const noexcept + GPUd() bool IsNormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -182,7 +182,7 @@ struct Float16Impl { /// Tests if the value is subnormal (denormal). /// /// True if so - GPUd() bool IsSubnormal() const noexcept + GPUd() bool IsSubnormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -194,13 +194,13 @@ struct Float16Impl { /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + GPUd() Derived Abs() const GPUnoexcept() { return Derived::FromBits(AbsImpl()); } /// /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + GPUd() Derived Negate() const GPUnoexcept() { return Derived::FromBits(NegateImpl()); } /// /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check @@ -210,12 +210,12 @@ struct Float16Impl { /// first value /// second value /// True if both arguments represent zero - GPUd() static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept + GPUd() static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) GPUnoexcept() { return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; } - GPUd() bool operator==(const Float16Impl& rhs) const noexcept + GPUd() bool operator==(const Float16Impl& rhs) const GPUnoexcept() { if (IsNaN() || rhs.IsNaN()) { // IEEE defines that NaN is not equal to anything, including itself. @@ -224,9 +224,9 @@ struct Float16Impl { return val == rhs.val; } - GPUd() bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } + GPUd() bool operator!=(const Float16Impl& rhs) const GPUnoexcept() { return !(*this == rhs); } - GPUd() bool operator<(const Float16Impl& rhs) const noexcept + GPUd() bool operator<(const Float16Impl& rhs) const GPUnoexcept() { if (IsNaN() || rhs.IsNaN()) { // IEEE defines that NaN is unordered with respect to everything, including itself. @@ -275,7 +275,7 @@ union float32_bits { }; // namespace detail template -GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept +GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) GPUnoexcept() { detail::float32_bits f{}; f.f = v; @@ -324,7 +324,7 @@ GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept } template -GPUdi() float Float16Impl::ToFloatImpl() const noexcept +GPUdi() float Float16Impl::ToFloatImpl() const GPUnoexcept() { constexpr detail::float32_bits magic = {113 << 23}; constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift @@ -364,19 +364,19 @@ struct BFloat16Impl { /// /// /// - GPUd() static uint16_t ToUint16Impl(float v) noexcept; + GPUd() static uint16_t ToUint16Impl(float v) GPUnoexcept(); /// /// Converts bfloat16 to float /// /// float representation of bfloat16 value - GPUd() float ToFloatImpl() const noexcept; + GPUd() float ToFloatImpl() const GPUnoexcept(); /// /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() uint16_t AbsImpl() const noexcept + GPUd() uint16_t AbsImpl() const GPUnoexcept() { return static_cast(val & ~kSignMask); } @@ -385,26 +385,26 @@ struct BFloat16Impl { /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() uint16_t NegateImpl() const noexcept + GPUd() uint16_t NegateImpl() const GPUnoexcept() { return IsNaN() ? val : static_cast(val ^ kSignMask); } public: // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7F80U; - static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; - static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; - static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; - static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; - static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; - static constexpr uint16_t kEpsilonBits = 0x0080U; - static constexpr uint16_t kMinValueBits = 0xFF7FU; - static constexpr uint16_t kMaxValueBits = 0x7F7FU; - static constexpr uint16_t kRoundToNearest = 0x7FFFU; - static constexpr uint16_t kOneBits = 0x3F80U; - static constexpr uint16_t kMinusOneBits = 0xBF80U; + static GPUglobalconstexpr() uint16_t kSignMask = 0x8000U; + static GPUglobalconstexpr() uint16_t kBiasedExponentMask = 0x7F80U; + static GPUglobalconstexpr() uint16_t kPositiveInfinityBits = 0x7F80U; + static GPUglobalconstexpr() uint16_t kNegativeInfinityBits = 0xFF80U; + static GPUglobalconstexpr() uint16_t kPositiveQNaNBits = 0x7FC1U; + static GPUglobalconstexpr() uint16_t kNegativeQNaNBits = 0xFFC1U; + static GPUglobalconstexpr() uint16_t kSignaling_NaNBits = 0x7F80U; + static GPUglobalconstexpr() uint16_t kEpsilonBits = 0x0080U; + static GPUglobalconstexpr() uint16_t kMinValueBits = 0xFF7FU; + static GPUglobalconstexpr() uint16_t kMaxValueBits = 0x7F7FU; + static GPUglobalconstexpr() uint16_t kRoundToNearest = 0x7FFFU; + static GPUglobalconstexpr() uint16_t kOneBits = 0x3F80U; + static GPUglobalconstexpr() uint16_t kMinusOneBits = 0xBF80U; uint16_t val{0}; @@ -414,7 +414,7 @@ struct BFloat16Impl { /// Checks if the value is negative /// /// true if negative - GPUd() bool IsNegative() const noexcept + GPUd() bool IsNegative() const GPUnoexcept() { return static_cast(val) < 0; } @@ -423,7 +423,7 @@ struct BFloat16Impl { /// Tests if the value is NaN /// /// true if NaN - GPUd() bool IsNaN() const noexcept + GPUd() bool IsNaN() const GPUnoexcept() { return AbsImpl() > kPositiveInfinityBits; } @@ -432,7 +432,7 @@ struct BFloat16Impl { /// Tests if the value is finite /// /// true if finite - GPUd() bool IsFinite() const noexcept + GPUd() bool IsFinite() const GPUnoexcept() { return AbsImpl() < kPositiveInfinityBits; } @@ -441,7 +441,7 @@ struct BFloat16Impl { /// Tests if the value represents positive infinity. /// /// true if positive infinity - GPUd() bool IsPositiveInfinity() const noexcept + GPUd() bool IsPositiveInfinity() const GPUnoexcept() { return val == kPositiveInfinityBits; } @@ -450,7 +450,7 @@ struct BFloat16Impl { /// Tests if the value represents negative infinity /// /// true if negative infinity - GPUd() bool IsNegativeInfinity() const noexcept + GPUd() bool IsNegativeInfinity() const GPUnoexcept() { return val == kNegativeInfinityBits; } @@ -459,7 +459,7 @@ struct BFloat16Impl { /// Tests if the value is either positive or negative infinity. /// /// True if absolute value is infinity - GPUd() bool IsInfinity() const noexcept + GPUd() bool IsInfinity() const GPUnoexcept() { return AbsImpl() == kPositiveInfinityBits; } @@ -468,7 +468,7 @@ struct BFloat16Impl { /// Tests if the value is NaN or zero. Useful for comparisons. /// /// True if NaN or zero. - GPUd() bool IsNaNOrZero() const noexcept + GPUd() bool IsNaNOrZero() const GPUnoexcept() { auto abs = AbsImpl(); return (abs == 0 || abs > kPositiveInfinityBits); @@ -478,7 +478,7 @@ struct BFloat16Impl { /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). /// /// True if so - GPUd() bool IsNormal() const noexcept + GPUd() bool IsNormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -490,7 +490,7 @@ struct BFloat16Impl { /// Tests if the value is subnormal (denormal). /// /// True if so - GPUd() bool IsSubnormal() const noexcept + GPUd() bool IsSubnormal() const GPUnoexcept() { auto abs = AbsImpl(); return (abs < kPositiveInfinityBits) // is finite @@ -502,13 +502,13 @@ struct BFloat16Impl { /// Creates an instance that represents absolute value. /// /// Absolute value - GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + GPUd() Derived Abs() const GPUnoexcept() { return Derived::FromBits(AbsImpl()); } /// /// Creates a new instance with the sign flipped. /// /// Flipped sign instance - GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + GPUd() Derived Negate() const GPUnoexcept() { return Derived::FromBits(NegateImpl()); } /// /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check @@ -518,7 +518,7 @@ struct BFloat16Impl { /// first value /// second value /// True if both arguments represent zero - GPUd() static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept + GPUd() static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) GPUnoexcept() { // IEEE defines that positive and negative zero are equal, this gives us a quick equality check // for two values by or'ing the private bits together and stripping the sign. They are both zero, @@ -528,7 +528,7 @@ struct BFloat16Impl { }; template -GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept +GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) GPUnoexcept() { uint16_t result; if (o2::gpu::CAMath::IsNaN(v)) { @@ -566,7 +566,7 @@ GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept } template -GPUdi() float BFloat16Impl::ToFloatImpl() const noexcept +GPUdi() float BFloat16Impl::ToFloatImpl() const GPUnoexcept() { #ifndef __FAST_MATH__ if (IsNaN()) { @@ -621,7 +621,7 @@ struct Float16_t : OrtDataType::Float16Impl { /// No conversion is done here. /// /// 16-bit representation - constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } + constexpr explicit Float16_t(uint16_t v) GPUnoexcept() { val = v; } public: using Base = OrtDataType::Float16Impl; @@ -636,19 +636,19 @@ struct Float16_t : OrtDataType::Float16Impl { /// /// uint16_t bit representation of float16 /// new instance of Float16_t - GPUd() constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } + GPUd() constexpr static Float16_t FromBits(uint16_t v) GPUnoexcept() { return Float16_t(v); } /// /// __ctor from float. Float is converted into float16 16-bit representation. /// /// float value - GPUd() explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + GPUd() explicit Float16_t(float v) GPUnoexcept() { val = Base::ToUint16Impl(v); } /// /// Converts float16 to float /// /// float representation of float16 value - GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } + GPUd() float ToFloat() const GPUnoexcept() { return Base::ToFloatImpl(); } /// /// Checks if the value is negative @@ -729,7 +729,7 @@ struct Float16_t : OrtDataType::Float16Impl { /// /// User defined conversion operator. Converts Float16_t to float. /// - GPUdi() explicit operator float() const noexcept { return ToFloat(); } + GPUdi() explicit operator float() const GPUnoexcept() { return ToFloat(); } using Base::operator==; using Base::operator!=; @@ -765,7 +765,7 @@ struct BFloat16_t : OrtDataType::BFloat16Impl { /// No conversion is done. /// /// 16-bit bfloat16 value - constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } + constexpr explicit BFloat16_t(uint16_t v) GPUnoexcept() { val = v; } public: using Base = OrtDataType::BFloat16Impl; @@ -777,19 +777,19 @@ struct BFloat16_t : OrtDataType::BFloat16Impl { /// /// uint16_t bit representation of bfloat16 /// new instance of BFloat16_t - GPUd() static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } + GPUd() static constexpr BFloat16_t FromBits(uint16_t v) GPUnoexcept() { return BFloat16_t(v); } /// /// __ctor from float. Float is converted into bfloat16 16-bit representation. /// /// float value - GPUd() explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + GPUd() explicit BFloat16_t(float v) GPUnoexcept() { val = Base::ToUint16Impl(v); } /// /// Converts bfloat16 to float /// /// float representation of bfloat16 value - GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } + GPUd() float ToFloat() const GPUnoexcept() { return Base::ToFloatImpl(); } /// /// Checks if the value is negative @@ -870,13 +870,13 @@ struct BFloat16_t : OrtDataType::BFloat16Impl { /// /// User defined conversion operator. Converts BFloat16_t to float. /// - GPUdi() explicit operator float() const noexcept { return ToFloat(); } + GPUdi() explicit operator float() const GPUnoexcept() { return ToFloat(); } // We do not have an inherited impl for the below operators // as the internal class implements them a little differently - bool operator==(const BFloat16_t& rhs) const noexcept; - bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } - bool operator<(const BFloat16_t& rhs) const noexcept; + bool operator==(const BFloat16_t& rhs) const GPUnoexcept(); + bool operator!=(const BFloat16_t& rhs) const GPUnoexcept() { return !(*this == rhs); } + bool operator<(const BFloat16_t& rhs) const GPUnoexcept(); }; static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); diff --git a/Common/MathUtils/include/MathUtils/Cartesian.h b/Common/MathUtils/include/MathUtils/Cartesian.h index e61b10a7caee9..99b946069f089 100644 --- a/Common/MathUtils/include/MathUtils/Cartesian.h +++ b/Common/MathUtils/include/MathUtils/Cartesian.h @@ -152,7 +152,9 @@ class Rotation2D }; using Rotation2Df_t = Rotation2D; +#ifndef __METAL__ using Rotation2Dd_t = Rotation2D; +#endif #if (!defined(GPUCA_STANDALONE) || !defined(DGPUCA_NO_ROOT)) && !defined(GPUCA_GPUCODE) && !defined(GPUCOMMONRTYPES_H_ACTIVE) diff --git a/Common/MathUtils/include/MathUtils/Primitive2D.h b/Common/MathUtils/include/MathUtils/Primitive2D.h index 052926d111594..e654b9cfb1f13 100644 --- a/Common/MathUtils/include/MathUtils/Primitive2D.h +++ b/Common/MathUtils/include/MathUtils/Primitive2D.h @@ -28,17 +28,23 @@ namespace math_utils template using CircleXY = detail::CircleXY; using CircleXYf_t = detail::CircleXY; +#ifndef __METAL__ using CircleXYd_t = detail::CircleXY; +#endif template using IntervalXY = detail::IntervalXY; using IntervalXYf_t = detail::IntervalXY; +#ifndef __METAL__ using IntervalXYd_t = detail::IntervalXY; +#endif template using Bracket = detail::Bracket; using Bracketf_t = detail::Bracket; +#ifndef __METAL__ using Bracketd_t = detail::Bracket; +#endif } // namespace math_utils } // namespace o2 diff --git a/Common/MathUtils/include/MathUtils/SMatrixGPU.h b/Common/MathUtils/include/MathUtils/SMatrixGPU.h index 8158a93666a92..497a142a1902c 100644 --- a/Common/MathUtils/include/MathUtils/SMatrixGPU.h +++ b/Common/MathUtils/include/MathUtils/SMatrixGPU.h @@ -340,7 +340,11 @@ class MatRepSymGPU static GPUdi() int off(int i) { +#ifdef __METAL__ // MSL rejects variables declared static at function scope + constexpr auto v = row_offsets_utils::make(off1); +#else static constexpr auto v = row_offsets_utils::make(off1); +#endif return v[i]; } @@ -518,7 +522,7 @@ class SMatrixGPU R mRep; }; -#ifndef __OPENCL__ // TODO: current C++ for OpenCL 2021 is at C++17, so no concepts. But we don't need this trick for OpenCL anyway, so we can just hide it. +#if !defined(__OPENCL__) && !defined(__METAL__) // TODO: current C++ for OpenCL 2021 and MSL 4.1 are both at C++17, so no concepts. But we don't need this trick there anyway, so we can just hide it. template requires(sizeof(typename X::traits_type::pos_type) != 0) // do not provide a template to fair::Logger, etc... (pos_type is a member type of all std::ostream classes) GPUd() X& operator<<(Y& y, const SMatrixGPU&) @@ -1429,14 +1433,18 @@ template template GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const SMatrixGPU& rhs) { - return operator=(*this* rhs); + // the product is an expression evaluated element by element, and every element + // of it reads the whole of *this, so it has to be materialised first + const SMatrixGPU tmp(*this * rhs); + return operator=(tmp); } template template GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const Expr& rhs) { - return operator=(*this* rhs); + const SMatrixGPU tmp(*this * rhs); + return operator=(tmp); } template diff --git a/Common/MathUtils/include/MathUtils/Utils.h b/Common/MathUtils/include/MathUtils/Utils.h index 3c51245fc6c29..4d17efb0fe6c2 100644 --- a/Common/MathUtils/include/MathUtils/Utils.h +++ b/Common/MathUtils/include/MathUtils/Utils.h @@ -32,111 +32,133 @@ GPUdi() float to02Pi(float phi) return detail::to02Pi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double to02Pid(double phi) { return detail::to02Pi(phi); } +#endif GPUdi() void bringTo02Pi(float& phi) { detail::bringTo02Pi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void bringTo02Pid(double& phi) { detail::bringTo02Pi(phi); } +#endif inline float toPMPiGen(float phi) { return detail::toPMPiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double toPMPiGend(double phi) { return detail::toPMPiGen(phi); } +#endif inline void bringToPMPiGen(float& phi) { detail::bringToPMPiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline void bringToPMPiGend(double& phi) { detail::bringToPMPiGen(phi); } +#endif inline float to02PiGen(float phi) { return detail::to02PiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double to02PiGend(double phi) { return detail::to02PiGen(phi); } +#endif inline void bringTo02PiGen(float& phi) { detail::bringTo02PiGen(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline void bringTo02PiGend(double& phi) { detail::bringTo02PiGen(phi); } +#endif inline float toPMPi(float phi) { return detail::toPMPi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double toPMPid(double phi) { return detail::toPMPi(phi); } +#endif inline void bringToPMPi(float& phi) { return detail::bringToPMPi(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline void bringToPMPid(double& phi) { return detail::bringToPMPi(phi); } +#endif GPUdi() void sincos(float ang, float& s, float& c) { detail::sincos(ang, s, c); } #ifndef __OPENCL__ +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void sincosd(double ang, double& s, double& c) { detail::sincos(ang, s, c); } #endif +#endif GPUdi() void rotateZ(float xL, float yL, float& xG, float& yG, float snAlp, float csAlp) { return detail::rotateZ(xL, yL, xG, yG, snAlp, csAlp); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void rotateZd(double xL, double yL, double& xG, double& yG, double snAlp, double csAlp) { return detail::rotateZ(xL, yL, xG, yG, snAlp, csAlp); } +#endif GPUdi() void rotateZInv(float xG, float yG, float& xL, float& yL, float snAlp, float csAlp) { detail::rotateZInv(xG, yG, xL, yL, snAlp, csAlp); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() void rotateZInvd(double xG, double yG, double& xL, double& yL, double snAlp, double csAlp) { detail::rotateZInv(xG, yG, xL, yL, snAlp, csAlp); } +#endif #ifndef GPUCA_GPUCODE_DEVICE inline std::tuple rotateZInv(float xG, float yG, float snAlp, float csAlp) @@ -185,40 +207,48 @@ inline int angle2Sector(float phi) return detail::angle2Sector(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline int angle2Sectord(double phi) { return detail::angle2Sector(phi); } +#endif inline float sector2Angle(int sect) { return detail::sector2Angle(sect); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double sector2Angled(int sect) { return detail::sector2Angle(sect); } +#endif inline float angle2Alpha(float phi) { return detail::angle2Alpha(phi); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these inline double angle2Alphad(double phi) { return detail::angle2Alpha(phi); } +#endif GPUhdi() constexpr float fastATan2(float y, float x) { return detail::fastATan2(y, x); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() constexpr double fastATan2d(double y, double x) { return detail::fastATan2(y, x); } +#endif template GPUhdi() T min(const T x, const T y) @@ -226,10 +256,12 @@ GPUhdi() T min(const T x, const T y) return detail::min(x, y); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double mind(const double x, const double y) { return detail::min(x, y); }; +#endif template GPUhdi() T max(const T x, const T y) @@ -237,130 +269,156 @@ GPUhdi() T max(const T x, const T y) return detail::max(x, y); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double maxd(const double x, const double y) { return detail::max(x, y); }; +#endif GPUhdi() float sqrt(float x) { return detail::sqrt(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double sqrtd(double x) { return detail::sqrt(x); }; +#endif GPUhdi() float abs(float x) { return detail::abs(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUhdi() double absd(double x) { return detail::abs(x); }; +#endif GPUdi() float asin(float x) { return detail::asin(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double asind(double x) { return detail::asin(x); }; +#endif GPUdi() float atan(float x) { return detail::atan(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double atand(double x) { return detail::atan(x); }; +#endif GPUdi() float atan2(float y, float x) { return detail::atan2(y, x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double atan2d(double y, double x) { return detail::atan2(y, x); }; +#endif GPUdi() float sin(float x) { return detail::sin(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double sind(double x) { return detail::sin(x); }; +#endif GPUdi() float cos(float x) { return detail::cos(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double cosd(double x) { return detail::cos(x); }; +#endif GPUdi() float tan(float x) { return detail::tan(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double tand(double x) { return detail::tan(x); }; +#endif GPUdi() float twoPi() { return detail::twoPi(); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double twoPid() { return detail::twoPi(); }; +#endif GPUdi() float pi() { return detail::pi(); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double pid() { return detail::pi(); } +#endif GPUdi() int nint(float x) { return detail::nint(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() int nintd(double x) { return detail::nint(x); }; +#endif GPUdi() bool finite(float x) { return detail::finite(x); } +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() bool finited(double x) { return detail::finite(x); } +#endif GPUdi() unsigned int clz(unsigned int val) { @@ -377,12 +435,16 @@ GPUdi() float log(float x) return detail::log(x); }; +#ifndef __METAL__ // MSL has no double; every other backend keeps these GPUdi() double logd(double x) { return detail::log(x); }; +#endif +#ifndef __METAL__ using detail::StatAccumulator; +#endif using detail::bit2Mask; using detail::numberOfBitsSet; diff --git a/Common/MathUtils/include/MathUtils/detail/Bracket.h b/Common/MathUtils/include/MathUtils/detail/Bracket.h index 2da6949c4a6f8..450174dc9737f 100644 --- a/Common/MathUtils/include/MathUtils/detail/Bracket.h +++ b/Common/MathUtils/include/MathUtils/detail/Bracket.h @@ -16,6 +16,8 @@ #ifndef ALICEO2_BRACKET_H #define ALICEO2_BRACKET_H +#include "GPUCommonDef.h" + #include #ifndef GPUCA_GPUCODE_DEVICE #include @@ -53,9 +55,9 @@ class Bracket bool operator==(const Bracket& other) const; bool operator!=(const Bracket& other) const; - void setMax(T v) noexcept; - void setMin(T v) noexcept; - void set(T minv, T maxv) noexcept; + void setMax(T v) GPUnoexcept(); + void setMin(T v) GPUnoexcept(); + void set(T minv, T maxv) GPUnoexcept(); T& getMax(); T& getMin(); @@ -129,19 +131,19 @@ inline bool Bracket::operator!=(const Bracket& rhs) const } template -inline void Bracket::setMax(T v) noexcept +inline void Bracket::setMax(T v) GPUnoexcept() { mMax = v; } template -inline void Bracket::setMin(T v) noexcept +inline void Bracket::setMin(T v) GPUnoexcept() { mMin = v; } template -inline void Bracket::set(T minv, T maxv) noexcept +inline void Bracket::set(T minv, T maxv) GPUnoexcept() { this->setMin(minv); this->setMax(maxv); diff --git a/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h b/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h index abb8a716cc5ee..2d0eb94fd951a 100644 --- a/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h +++ b/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h @@ -27,6 +27,7 @@ namespace math_utils namespace detail { +#ifndef __METAL__ // host-only accumulator, and MSL has no double struct StatAccumulator { // mean / RMS accumulator double sum = 0.; @@ -83,6 +84,7 @@ struct StatAccumulator { n = 0; } }; +#endif } // namespace detail } // namespace math_utils diff --git a/Common/MathUtils/include/MathUtils/detail/trigonometric.h b/Common/MathUtils/include/MathUtils/detail/trigonometric.h index e13d965663dc9..e0bf675d696cc 100644 --- a/Common/MathUtils/include/MathUtils/detail/trigonometric.h +++ b/Common/MathUtils/include/MathUtils/detail/trigonometric.h @@ -48,7 +48,7 @@ GPUhdi() T to02Pi(T phi) template GPUhdi() void bringTo02Pi(T& phi) { - phi = to02Pi(phi); + phi = to02Pi(phi); } template @@ -68,7 +68,7 @@ inline T to02PiGen(T phi) template inline void bringTo02PiGen(T& phi) { - phi = to02PiGen(phi); + phi = to02PiGen(phi); } template @@ -87,7 +87,7 @@ GPUhdi() T toPMPi(T phi) template GPUhdi() void bringToPMPi(T& phi) { - phi = toPMPi(phi); + phi = toPMPi(phi); } template @@ -107,10 +107,10 @@ inline T toPMPiGen(T phi) template inline void bringToPMPiGen(T& phi) { - phi = toPMPiGen(phi); + phi = toPMPiGen(phi); } -#ifdef __OPENCL__ // TODO: get rid of that stupid workaround for OpenCL template address spaces +#if defined(__OPENCL__) || defined(__METAL__) // TODO: get rid of that stupid workaround for OpenCL template address spaces template GPUhdi() void sincos(T ang, S& s, U& c) { @@ -122,12 +122,14 @@ GPUhdi() void sincos(T ang, T& s, T& c) { return o2::gpu::GPUCommonMath::SinCos(ang, s, c); } +#ifndef __METAL__ // MSL has no double; the primary template still serves float template <> GPUhdi() void sincos(double ang, double& s, double& c) { return o2::gpu::GPUCommonMath::SinCosd(ang, s, c); } #endif +#endif #ifndef GPUCA_GPUCODE_DEVICE @@ -358,11 +360,13 @@ GPUdi() T twoPi() return o2::gpu::GPUCommonMath::TwoPi(); }; +#ifndef __METAL__ // MSL has no double; the primary template still serves float template <> GPUdi() double twoPi() { return o2::constants::math::TwoPI; }; +#endif template GPUdi() T pi() @@ -370,11 +374,13 @@ GPUdi() T pi() return o2::gpu::GPUCommonMath::Pi(); } +#ifndef __METAL__ // MSL has no double; the primary template still serves float template <> GPUdi() double pi() { return o2::constants::math::PI; } +#endif #ifndef GPUCA_GPUCODE_DEVICE template <> diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h index a5be0da32f641..15f72d6e2f68e 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/Defs.h @@ -19,6 +19,8 @@ #ifndef AliceO2_TPC_Defs_H #define AliceO2_TPC_Defs_H +#include "GPUCommonDouble.h" + #include "GPUCommonDef.h" #ifndef GPUCA_GPUCODE_DEVICE @@ -42,9 +44,9 @@ enum Side { A = 0, GPUglobalconstexpr() unsigned char SECTORSPERSIDE = 18; GPUglobalconstexpr() unsigned char SIDES = 2; -constexpr double PI = 3.14159265358979323846; -constexpr double TWOPI = 2. * PI; -constexpr double SECPHIWIDTH = TWOPI / 18.; +GPUglobalconstexpr() o2::gpu::GPUdoubleValue PI = 3.14159265358979323846; +GPUglobalconstexpr() o2::gpu::GPUdoubleValue TWOPI = 2. * PI; +GPUglobalconstexpr() o2::gpu::GPUdoubleValue SECPHIWIDTH = TWOPI / 18.; /// TPC ROC types enum RocType { IROC = 0, diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h index 47de5457cea16..0785afa7553ac 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h @@ -17,6 +17,7 @@ #define _ALICEO2_HELIX_HELPER_ #include "CommonConstants/MathConstants.h" +#include "GPUCommonDouble.h" #include "MathUtils/Utils.h" #include "MathUtils/Primitive2D.h" @@ -247,8 +248,8 @@ struct CrossInfo { auto tgp = trcL.getSnp() * cspi; float kx = traxL.c - traxL.s * tgp; float ky = traxL.s + traxL.c * tgp; - double dk = dx * kx + dy * ky; - double det = dk * dk - cspi2 * (dx * dx + dy * dy - traxH.rC * traxH.rC); + o2::gpu::GPUdoubleCalc dk = dx * kx + dy * ky; + o2::gpu::GPUdoubleCalc det = dk * dk - cspi2 * (dx * dx + dy * dy - traxH.rC * traxH.rC); if (det > 0) { // 2 crossings det = o2::gpu::GPUCommonMath::Sqrt(det); float t0 = (-dk + det) * cspi2; diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h index c4b17158f7dc5..2d4d67574a12a 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Track.h @@ -25,11 +25,15 @@ namespace track { using TrackParF = TrackParametrization; +#ifndef __METAL__ using TrackParD = TrackParametrization; +#endif using TrackPar = TrackParF; using TrackParCovF = TrackParametrizationWithError; +#ifndef __METAL__ using TrackParCovD = TrackParametrizationWithError; +#endif using TrackParCov = TrackParCovF; } // namespace track diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h index 81280d090be71..d120e4c15077f 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h @@ -17,6 +17,7 @@ #ifndef INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKPARAMETRIZATIONWITHERROR_H_ #define INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKPARAMETRIZATIONWITHERROR_H_ +#include "GPUCommonDouble.h" #include "ReconstructionDataFormats/TrackParametrization.h" #include @@ -41,8 +42,8 @@ class TrackParametrizationWithError : public TrackParametrization #endif using covMat_t = std::array; - using MatrixDSym5 = o2::math_utils::SMatrix>; - using MatrixD5 = o2::math_utils::SMatrix>; + using MatrixDSym5 = o2::math_utils::SMatrix>; + using MatrixD5 = o2::math_utils::SMatrix>; GPUhd() TrackParametrizationWithError(); GPUd() TrackParametrizationWithError(value_t x, value_t alpha, const params_t& par, const covMat_t& cov, int charge = 1, const PID pid = PID::Pion); diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h index 6befcd0dfc898..1e96ab224a308 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h @@ -138,7 +138,11 @@ GPUd() value_T BetheBlochSolid(value_T bg, value_T rho, value_T kp1, value_T kp2 if (x > kp2) { d2 = lhwI + x - value_T(0.5); } else if (x > kp1) { +#ifdef __METAL__ // MSL has no double + float r = (kp2 - x) / (kp2 - kp1); +#else double r = (kp2 - x) / (kp2 - kp1); +#endif d2 = lhwI + x - value_T(0.5) + (value_T(0.5) - lhwI - kp1) * r * r * r; } auto dedx = mK * meanZA / beta2 * (value_T(0.5) * gpu::CAMath::Log(value_T(2) * me * bg2 * maxT / (meanI * meanI)) - beta2 - d2); diff --git a/DataFormats/Reconstruction/src/TrackParametrization.cxx b/DataFormats/Reconstruction/src/TrackParametrization.cxx index fb398bbcf07cf..cdf8f35b7cfb0 100644 --- a/DataFormats/Reconstruction/src/TrackParametrization.cxx +++ b/DataFormats/Reconstruction/src/TrackParametrization.cxx @@ -15,6 +15,7 @@ /// @brief #include "ReconstructionDataFormats/TrackParametrization.h" +#include "GPUCommonDouble.h" #include "ReconstructionDataFormats/Vertex.h" #include "ReconstructionDataFormats/DCA.h" #include @@ -477,7 +478,7 @@ GPUd() bool TrackParametrization::getYZAt(value_t xk, value_t b, value_ if (gpu::CAMath::Abs(r2) < constants::math::Almost0) { return false; } - double dy2dx = (f1 + f2) / (r1 + r2); + GPUdoubleCalc dy2dx = (f1 + f2) / (r1 + r2); y += dx * dy2dx; if (gpu::CAMath::Abs(x2r) < 0.05f) { z += dx * (r2 + f2 * dy2dx) * getTgl(); @@ -692,7 +693,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val // DirOutward (==1) - go along the track (increasing mX) // DirInward (==-1) - go backward (decreasing mX) // - const double fy = mP[0], sn = mP[2]; + const GPUdoubleCalc fy = mP[0], sn = mP[2]; const value_t kEps = 1.e-6; // if (gpu::CAMath::Abs(getSnp()) > constants::math::Almost1) { @@ -711,7 +712,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val if (r0 <= constants::math::Almost0) { return false; // the track is concentric to circle } - double tR2r0 = 1., g = 0., tmp = 0.; + GPUdoubleCalc tR2r0 = 1., g = 0., tmp = 0.; if (gpu::CAMath::Abs(circle.rC - r0) > kEps) { tR2r0 = circle.rC / r0; g = 0.5f * (r * r / (r0 * circle.rC) - tR2r0 - 1.f / tR2r0); @@ -786,7 +787,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val } // this is a straight track if (gpu::CAMath::Abs(sn) >= constants::math::Almost1) { // || to Y axis - double det = (r - mX) * (r + mX); + GPUdoubleCalc det = (r - mX) * (r + mX); if (det < 0.f) { return false; // does not reach raduis r } @@ -815,7 +816,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val } } } else if (gpu::CAMath::Abs(sn) <= constants::math::Almost0) { // || to X axis - double det = (r - fy) * (r + fy); + GPUdoubleCalc det = (r - fy) * (r + fy); if (det < 0.) { return false; // does not reach raduis r } diff --git a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx index 748cb47094d26..7b19e7a83f867 100644 --- a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx +++ b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "ReconstructionDataFormats/TrackParametrizationWithError.h" +#include "GPUCommonDouble.h" #include "ReconstructionDataFormats/Vertex.h" #include "ReconstructionDataFormats/DCA.h" #include "CommonConstants/MathConstants.h" @@ -67,9 +68,9 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu if (gpu::CAMath::Abs(r2) < constants::math::Almost0) { return false; } - double r1pr2Inv = 1. / (r1 + r2); - double dy2dx = (f1 + f2) * r1pr2Inv; - const auto dy2dxF = static_cast(dy2dx); // the parameter update does not need the double + GPUdoubleCalc r1pr2Inv = 1. / (r1 + r2); + GPUdoubleCalc dy2dx = (f1 + f2) * r1pr2Inv; + const auto dy2dxF = static_cast(dy2dx); // the parameter update does not need the GPUdoubleCalc bool arcz = gpu::CAMath::Abs(x2r) > 0.05f; params_t dP{0.f}; if (arcz) { @@ -110,33 +111,33 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu // evaluate matrix in double prec. value_t kb = bz * constants::math::B2C; - double r2inv = 1. / r2, r1inv = 1. / r1; - double dx2r1pr2 = dx * r1pr2Inv; + GPUdoubleCalc r2inv = 1. / r2, r1inv = 1. / r1; + GPUdoubleCalc dx2r1pr2 = dx * r1pr2Inv; - double hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); - double f02 = hh * r1inv; - double f04 = hh * dx2r1pr2 * kb; - double f24 = dx * kb; // x2r/mP[kQ2Pt]; - double f12 = this->getTgl() * (f02 * f2 + jj); - double f13 = dx * (r2 + f2 * dy2dx); - double f14 = this->getTgl() * (f04 * f2 + jj * f24); + GPUdoubleCalc hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); + GPUdoubleCalc f02 = hh * r1inv; + GPUdoubleCalc f04 = hh * dx2r1pr2 * kb; + GPUdoubleCalc f24 = dx * kb; // x2r/mP[kQ2Pt]; + GPUdoubleCalc f12 = this->getTgl() * (f02 * f2 + jj); + GPUdoubleCalc f13 = dx * (r2 + f2 * dy2dx); + GPUdoubleCalc f14 = this->getTgl() * (f04 * f2 + jj * f24); // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -180,17 +181,17 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac } value_t kb = bz * constants::math::B2C; // evaluate in double prec. - double snpRef0 = linRef0.getSnp(), cspRef0 = gpu::CAMath::Sqrt((1 - snpRef0) * (1 + snpRef0)); - double snpRef1 = linRef1.getSnp(), cspRef1 = gpu::CAMath::Sqrt((1 - snpRef1) * (1 + snpRef1)); - double cspRef0Inv = 1 / cspRef0, cspRef1Inv = 1 / cspRef1, cc = cspRef0 + cspRef1, ccInv = 1 / cc, dy2dx = (snpRef0 + snpRef1) * ccInv; - double dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); - - double f02 = hh * cspRef0Inv; - double f04 = hh * dxccInv * kb; - double f24 = dx * kb; - double f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); - double f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS - double f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); + GPUdoubleCalc snpRef0 = linRef0.getSnp(), cspRef0 = gpu::CAMath::Sqrt((1.f - snpRef0) * (1.f + snpRef0)); + GPUdoubleCalc snpRef1 = linRef1.getSnp(), cspRef1 = gpu::CAMath::Sqrt((1.f - snpRef1) * (1.f + snpRef1)); + GPUdoubleCalc cspRef0Inv = 1.f / cspRef0, cspRef1Inv = 1.f / cspRef1, cc = cspRef0 + cspRef1, ccInv = 1.f / cc, dy2dx = (snpRef0 + snpRef1) * ccInv; + GPUdoubleCalc dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1.f + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); + + GPUdoubleCalc f02 = hh * cspRef0Inv; + GPUdoubleCalc f04 = hh * dxccInv * kb; + GPUdoubleCalc f24 = dx * kb; + GPUdoubleCalc f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); + GPUdoubleCalc f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS + GPUdoubleCalc f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); // difference between the current and reference state value_t diff[5]; @@ -215,21 +216,21 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac &c44 = mC[kSigQ2Pt2]; // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -636,9 +637,9 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons if (gpu::CAMath::Abs(r2) < constants::math::Almost0) { return false; } - double r1pr2Inv = 1. / (r1 + r2), r2inv = 1. / r2, r1inv = 1. / r1; - double dy2dx = (f1 + f2) * r1pr2Inv, dx2r1pr2 = dx * r1pr2Inv; - value_t step = (gpu::CAMath::Abs(x2r) < 0.05f) ? dx * gpu::CAMath::Abs(r2 + f2 * dy2dx) // chord + GPUdoubleCalc r1pr2Inv = 1. / (r1 + r2), r2inv = 1. / r2, r1inv = 1. / r1; + GPUdoubleCalc dy2dx = (f1 + f2) * r1pr2Inv, dx2r1pr2 = dx * r1pr2Inv; + value_t step = (gpu::CAMath::Abs(x2r) < 0.05f) ? value_t(dx * gpu::CAMath::Abs(r2 + f2 * dy2dx)) // chord : 2.f * gpu::CAMath::ASin(0.5f * dx * gpu::CAMath::Sqrt(1.f + dy2dx * dy2dx) * crv) / crv; // arc step *= gpu::CAMath::Sqrt(1.f + this->getTgl() * this->getTgl()); // @@ -656,30 +657,30 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons // evaluate matrix in double prec. value_t kb = b[2] * constants::math::B2C; - double hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); - double f02 = hh * r1inv; - double f04 = hh * dx2r1pr2 * kb; - double f24 = dx * kb; // x2r/mP[kQ2Pt]; - double f12 = this->getTgl() * (f02 * f2 + jj); - double f13 = dx * (r2 + f2 * dy2dx); - double f14 = this->getTgl() * (f04 * f2 + jj * f24); + GPUdoubleCalc hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv); + GPUdoubleCalc f02 = hh * r1inv; + GPUdoubleCalc f04 = hh * dx2r1pr2 * kb; + GPUdoubleCalc f24 = dx * kb; // x2r/mP[kQ2Pt]; + GPUdoubleCalc f12 = this->getTgl() * (f02 * f2 + jj); + GPUdoubleCalc f13 = dx * (r2 + f2 * dy2dx); + GPUdoubleCalc f14 = this->getTgl() * (f04 * f2 + jj * f24); // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -889,13 +890,13 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac cc = cspRef0 + cspRef1; ccInv = value_t(1) / cc; dy2dx = (snpRef0 + snpRef1) * ccInv; - double dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); - double f02 = hh * cspRef0Inv; - double f04 = hh * dxccInv * kb; - double f24 = dx * kb; - double f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); - double f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS - double f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); + GPUdoubleCalc dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv); + GPUdoubleCalc f02 = hh * cspRef0Inv; + GPUdoubleCalc f04 = hh * dxccInv * kb; + GPUdoubleCalc f24 = dx * kb; + GPUdoubleCalc f12 = linRef0.getTgl() * (f02 * snpRef1 + jj); + GPUdoubleCalc f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS + GPUdoubleCalc f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24); // difference between the current and reference state value_t diff[5]; @@ -922,21 +923,21 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, Trac &c44 = mC[kSigQ2Pt2]; // b = C*ft - double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; - double b02 = f24 * c40; - double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; - double b12 = f24 * c41; - double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; - double b22 = f24 * c42; - double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; - double b42 = f24 * c44; - double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; - double b32 = f24 * c43; + GPUdoubleCalc b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30; + GPUdoubleCalc b02 = f24 * c40; + GPUdoubleCalc b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31; + GPUdoubleCalc b12 = f24 * c41; + GPUdoubleCalc b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32; + GPUdoubleCalc b22 = f24 * c42; + GPUdoubleCalc b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43; + GPUdoubleCalc b42 = f24 * c44; + GPUdoubleCalc b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33; + GPUdoubleCalc b32 = f24 * c43; // a = f*b = f*C*ft - double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; - double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; - double a22 = f24 * b42; + GPUdoubleCalc a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42; + GPUdoubleCalc a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32; + GPUdoubleCalc a22 = f24 * b42; // F*C*Ft = C + (b + bt + a) c00 += b00 + b00 + a00; @@ -1034,7 +1035,7 @@ template GPUd() void TrackParametrizationWithError::resetCovariance(value_t s2) { // Reset the covarince matrix to "something big" - double d0(kCY2max), d1(kCZ2max), d2(kCSnp2max), d3(kCTgl2max), d4(kC1Pt2max); + GPUdoubleCalc d0(kCY2max), d1(kCZ2max), d2(kCSnp2max), d3(kCTgl2max), d4(kC1Pt2max); if (s2 > constants::math::Almost0) { d0 = getSigmaY2() * s2; d1 = getSigmaZ2() * s2; @@ -1072,9 +1073,9 @@ template GPUd() auto TrackParametrizationWithError::getPredictedChi2(const value_t* p, const value_t* cov) const -> value_t { // Estimate the chi2 of the space point "p" with the cov. matrix "cov" - auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); - auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); - auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); + auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); + auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); + auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); auto det = sdd * szz - sdz * sdz; if (gpu::CAMath::Abs(det) < constants::math::Almost0) { @@ -1086,7 +1087,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2(const value auto chi2 = (d * (szz * d - sdz * z) + z * (sdd * z - d * sdz)) / det; if (chi2 < 0.) { #ifndef GPUCA_ALIGPUCODE - LOGP(warning, "Negative chi2={}, Cluster: {} {} {} Dy:{} Dz:{} | sdd:{} sdz:{} szz:{} det:{}", chi2, cov[0], cov[1], cov[2], d, z, sdd, sdz, szz, det); + LOGP(warning, "Negative chi2={}, Cluster: {} {} {} Dy:{} Dz:{} | sdd:{} sdz:{} szz:{} det:{}", double(chi2), cov[0], cov[1], cov[2], d, z, double(sdd), double(sdz), double(szz), double(det)); LOGP(warning, "Track: {}", asString()); #endif } @@ -1098,9 +1099,9 @@ template GPUd() auto TrackParametrizationWithError::getPredictedChi2Quiet(const value_t* p, const value_t* cov) const -> value_t { // Estimate the chi2 of the space point "p" with the cov. matrix "cov" - auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); - auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); - auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); + auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]); + auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]); + auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]); auto det = sdd * szz - sdz * sdz; if (gpu::CAMath::Abs(det) < constants::math::Almost0) { @@ -1150,9 +1151,9 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T // Factorize cov = L * D * L^T with L unit lower triangular. The strictly lower triangle of // lmat holds L, its strictly upper triangle holds the transpose of L * D, so that the inner // products below need no extra multiplication by D. dInv holds the inverted diagonal of D. - double lmat[kNParams][kNParams], dInv[kNParams]; + GPUdoubleCalc lmat[kNParams][kNParams], dInv[kNParams]; for (int j = 0; j < kNParams; j++) { - double djj = cov(j, j); + GPUdoubleCalc djj = cov(j, j); for (int k = 0; k < j; k++) { djj -= lmat[j][k] * lmat[k][j]; } @@ -1161,7 +1162,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T } dInv[j] = 1. / djj; for (int i = j + 1; i < kNParams; i++) { - double s = cov(i, j); + GPUdoubleCalc s = cov(i, j); for (int k = 0; k < j; k++) { s -= lmat[i][k] * lmat[k][j]; } @@ -1171,9 +1172,9 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2Fast(const T } // chi2 = d^T C^-1 d = sum_i y_i^2 / D_i with y from the forward substitution L y = d - double chi2 = 0., y[kNParams]; + GPUdoubleCalc chi2 = 0., y[kNParams]; for (int i = 0; i < kNParams; i++) { - double s = double(this->getParam(i)) - double(rhs.getParam(i)); + GPUdoubleCalc s = GPUdoubleCalc(this->getParam(i)) - GPUdoubleCalc(rhs.getParam(i)); for (int k = 0; k < i; k++) { s -= lmat[i][k] * y[k]; } @@ -1188,21 +1189,21 @@ template GPUd() void TrackParametrizationWithError::buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const { // fill combined cov.matrix (NOT inverted) - cov(kY, kY) = static_cast(getSigmaY2()) + static_cast(rhs.getSigmaY2()); - cov(kZ, kY) = static_cast(getSigmaZY()) + static_cast(rhs.getSigmaZY()); - cov(kZ, kZ) = static_cast(getSigmaZ2()) + static_cast(rhs.getSigmaZ2()); - cov(kSnp, kY) = static_cast(getSigmaSnpY()) + static_cast(rhs.getSigmaSnpY()); - cov(kSnp, kZ) = static_cast(getSigmaSnpZ()) + static_cast(rhs.getSigmaSnpZ()); - cov(kSnp, kSnp) = static_cast(getSigmaSnp2()) + static_cast(rhs.getSigmaSnp2()); - cov(kTgl, kY) = static_cast(getSigmaTglY()) + static_cast(rhs.getSigmaTglY()); - cov(kTgl, kZ) = static_cast(getSigmaTglZ()) + static_cast(rhs.getSigmaTglZ()); - cov(kTgl, kSnp) = static_cast(getSigmaTglSnp()) + static_cast(rhs.getSigmaTglSnp()); - cov(kTgl, kTgl) = static_cast(getSigmaTgl2()) + static_cast(rhs.getSigmaTgl2()); - cov(kQ2Pt, kY) = static_cast(getSigma1PtY()) + static_cast(rhs.getSigma1PtY()); - cov(kQ2Pt, kZ) = static_cast(getSigma1PtZ()) + static_cast(rhs.getSigma1PtZ()); - cov(kQ2Pt, kSnp) = static_cast(getSigma1PtSnp()) + static_cast(rhs.getSigma1PtSnp()); - cov(kQ2Pt, kTgl) = static_cast(getSigma1PtTgl()) + static_cast(rhs.getSigma1PtTgl()); - cov(kQ2Pt, kQ2Pt) = static_cast(getSigma1Pt2()) + static_cast(rhs.getSigma1Pt2()); + cov(kY, kY) = static_cast(getSigmaY2()) + static_cast(rhs.getSigmaY2()); + cov(kZ, kY) = static_cast(getSigmaZY()) + static_cast(rhs.getSigmaZY()); + cov(kZ, kZ) = static_cast(getSigmaZ2()) + static_cast(rhs.getSigmaZ2()); + cov(kSnp, kY) = static_cast(getSigmaSnpY()) + static_cast(rhs.getSigmaSnpY()); + cov(kSnp, kZ) = static_cast(getSigmaSnpZ()) + static_cast(rhs.getSigmaSnpZ()); + cov(kSnp, kSnp) = static_cast(getSigmaSnp2()) + static_cast(rhs.getSigmaSnp2()); + cov(kTgl, kY) = static_cast(getSigmaTglY()) + static_cast(rhs.getSigmaTglY()); + cov(kTgl, kZ) = static_cast(getSigmaTglZ()) + static_cast(rhs.getSigmaTglZ()); + cov(kTgl, kSnp) = static_cast(getSigmaTglSnp()) + static_cast(rhs.getSigmaTglSnp()); + cov(kTgl, kTgl) = static_cast(getSigmaTgl2()) + static_cast(rhs.getSigmaTgl2()); + cov(kQ2Pt, kY) = static_cast(getSigma1PtY()) + static_cast(rhs.getSigma1PtY()); + cov(kQ2Pt, kZ) = static_cast(getSigma1PtZ()) + static_cast(rhs.getSigma1PtZ()); + cov(kQ2Pt, kSnp) = static_cast(getSigma1PtSnp()) + static_cast(rhs.getSigma1PtSnp()); + cov(kQ2Pt, kTgl) = static_cast(getSigma1PtTgl()) + static_cast(rhs.getSigma1PtTgl()); + cov(kQ2Pt, kQ2Pt) = static_cast(getSigma1Pt2()) + static_cast(rhs.getSigma1Pt2()); } //______________________________________________ @@ -1225,7 +1226,7 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2(const Track LOG(warning) << "Cov.matrix inversion failed: " << covToSet; return 2.f * HugeF; } - double chi2diag = 0., chi2ndiag = 0., diff[kNParams]; + GPUdoubleCalc chi2diag = 0., chi2ndiag = 0., diff[kNParams]; for (int i = kNParams; i--;) { diff[i] = this->getParam(i) - rhs.getParam(i); chi2diag += diff[i] * diff[i] * covToSet(i, i); @@ -1275,7 +1276,7 @@ GPUd() bool TrackParametrizationWithError::update(const TrackParametriz // updated state vector: x = K*(x1-x0) // RS: why SMatix, SVector does not provide multiplication operators ??? - double diff[kNParams]; + GPUdoubleCalc diff[kNParams]; for (int i = kNParams; i--;) { diff[i] = rhs.getParam(i) - this->getParam(i); } @@ -1286,7 +1287,7 @@ GPUd() bool TrackParametrizationWithError::update(const TrackParametriz } // updated covariance: Cov0 = Cov0 - K*Cov0 - matK *= o2::math_utils::SMatrix>(matC0); + matK *= MatrixD5(matC0); mC[kSigY2] -= matK(kY, kY); mC[kSigZY] -= matK(kZ, kY); mC[kSigZ2] -= matK(kZ, kZ); @@ -1333,25 +1334,25 @@ GPUd() bool TrackParametrizationWithError::update(const value_t* p, con &cm44 = mC[kSigQ2Pt2]; // use double precision? - double r00 = static_cast(cov[0]) + static_cast(cm00); - double r01 = static_cast(cov[1]) + static_cast(cm10); - double r11 = static_cast(cov[2]) + static_cast(cm11); - double det = r00 * r11 - r01 * r01; + GPUdoubleCalc r00 = static_cast(cov[0]) + static_cast(cm00); + GPUdoubleCalc r01 = static_cast(cov[1]) + static_cast(cm10); + GPUdoubleCalc r11 = static_cast(cov[2]) + static_cast(cm11); + GPUdoubleCalc det = r00 * r11 - r01 * r01; if (gpu::CAMath::Abs(det) < constants::math::Almost0) { return false; } - double detI = 1. / det; - double tmp = r00; + GPUdoubleCalc detI = 1. / det; + GPUdoubleCalc tmp = r00; r00 = r11 * detI; r11 = tmp * detI; r01 = -r01 * detI; - double k00 = cm00 * r00 + cm10 * r01, k01 = cm00 * r01 + cm10 * r11; - double k10 = cm10 * r00 + cm11 * r01, k11 = cm10 * r01 + cm11 * r11; - double k20 = cm20 * r00 + cm21 * r01, k21 = cm20 * r01 + cm21 * r11; - double k30 = cm30 * r00 + cm31 * r01, k31 = cm30 * r01 + cm31 * r11; - double k40 = cm40 * r00 + cm41 * r01, k41 = cm40 * r01 + cm41 * r11; + GPUdoubleCalc k00 = cm00 * r00 + cm10 * r01, k01 = cm00 * r01 + cm10 * r11; + GPUdoubleCalc k10 = cm10 * r00 + cm11 * r01, k11 = cm10 * r01 + cm11 * r11; + GPUdoubleCalc k20 = cm20 * r00 + cm21 * r01, k21 = cm20 * r01 + cm21 * r11; + GPUdoubleCalc k30 = cm30 * r00 + cm31 * r01, k31 = cm30 * r01 + cm31 * r11; + GPUdoubleCalc k40 = cm40 * r00 + cm41 * r01, k41 = cm40 * r01 + cm41 * r11; value_t dy = p[kY] - this->getY(), dz = p[kZ] - this->getZ(); value_t dsnp = k20 * dy + k21 * dz; @@ -1363,8 +1364,8 @@ GPUd() bool TrackParametrizationWithError::update(const value_t* p, con value_t(k40 * dy + k41 * dz)}; this->updateParams(dP); - double c01 = cm10, c02 = cm20, c03 = cm30, c04 = cm40; - double c12 = cm21, c13 = cm31, c14 = cm41; + GPUdoubleCalc c01 = cm10, c02 = cm20, c03 = cm30, c04 = cm40; + GPUdoubleCalc c12 = cm21, c13 = cm31, c14 = cm41; cm00 -= k00 * cm00 + k01 * cm10; cm10 -= k00 * c01 + k01 * cm11; @@ -1709,7 +1710,7 @@ GPUd() bool TrackParametrizationWithError::getCovXYZPxPyPzGlo(std::arra } } - double jac[6][5] = {}; + GPUdoubleCalc jac[6][5] = {}; jac[0][kY] = -sn; jac[1][kY] = cs; jac[2][kZ] = 1.f; @@ -1727,7 +1728,7 @@ GPUd() bool TrackParametrizationWithError::getCovXYZPxPyPzGlo(std::arra int idx = 0; for (int i = 0; i < 6; ++i) { for (int j = 0; j <= i; ++j) { - double cij = 0.f; + GPUdoubleCalc cij = 0.f; for (int k = 0; k < kNParams; ++k) { for (int l = 0; l < kNParams; ++l) { cij += jac[i][k] * cTr[k][l] * jac[j][l]; diff --git a/Detectors/Base/include/DetectorsBase/Propagator.h b/Detectors/Base/include/DetectorsBase/Propagator.h index 38f4e089a1a1a..789d167ea1c70 100644 --- a/Detectors/Base/include/DetectorsBase/Propagator.h +++ b/Detectors/Base/include/DetectorsBase/Propagator.h @@ -192,11 +192,15 @@ class PropagatorImpl GPUd() void getFieldXYZ(const math_utils::Point3D xyz, float* bxyz) const; +#ifndef __METAL__ // MSL has no double; the float twin remains GPUd() void getFieldXYZ(const math_utils::Point3D xyz, double* bxyz) const; +#endif GPUd() float getBz(const math_utils::Point3D xyz) const; +#ifndef __METAL__ // MSL has no double; the float twin remains GPUd() double getBz(const math_utils::Point3D xyz) const; +#endif private: #ifndef GPUCA_GPUCODE @@ -221,7 +225,9 @@ class PropagatorImpl }; using PropagatorF = PropagatorImpl; +#ifndef __METAL__ // MSL has no double; the float twin remains using PropagatorD = PropagatorImpl; +#endif using Propagator = PropagatorF; } // namespace base diff --git a/Detectors/Base/src/Propagator.cxx b/Detectors/Base/src/Propagator.cxx index a465bbc312c77..78163ed019eb7 100644 --- a/Detectors/Base/src/Propagator.cxx +++ b/Detectors/Base/src/Propagator.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "DetectorsBase/Propagator.h" +#include "GPUCommonDouble.h" #include "GPUCommonLogger.h" #include "GPUCommonConstants.h" #include "GPUCommonMath.h" @@ -582,10 +583,10 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, if (cross.nDCA < 1) { return false; } - double phiCross[2] = {}, dphi[2] = {}; + GPUdoubleCalc phiCross[2] = {}, dphi[2] = {}; auto curv = track.getCurvature(bz); bool clockwise = curv < 0; // q+ in B+ or q- in B- goes clockwise - auto phiLoc = math_utils::detail::asin(track.getSnp()); + auto phiLoc = math_utils::detail::asin(track.getSnp()); auto phi0 = phiLoc + track.getAlpha(); o2::math_utils::detail::bringTo02Pi(phi0); for (int i = 0; i < cross.nDCA; i++) { @@ -593,12 +594,12 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, // == angle of the tangential to track circle at the crossing point X,Y // == normal to the radial vector from the track circle center {X-cX, Y-cY} // i.e. the angle of the vector {Y-cY, -(X-cx)} - auto normX = double(cross.yDCA[i]) - double(traux.yC), normY = -(double(cross.xDCA[i]) - double(traux.xC)); + auto normX = o2::gpu::GPUdoubleCalc(cross.yDCA[i]) - o2::gpu::GPUdoubleCalc(traux.yC), normY = -(o2::gpu::GPUdoubleCalc(cross.xDCA[i]) - o2::gpu::GPUdoubleCalc(traux.xC)); if (!clockwise) { normX = -normX; normY = -normY; } - phiCross[i] = math_utils::detail::atan2(normY, normX); + phiCross[i] = math_utils::detail::atan2(normY, normX); o2::math_utils::detail::bringTo02Pi(phiCross[i]); dphi[i] = phiCross[i] - phi0; if (dphi[i] > o2::constants::math::PI) { @@ -614,7 +615,7 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, auto phiLocFin = phiLoc + deltaPhi; // case1 if (math_utils::detail::abs(phiLocFin) < MaxPhiLocSafe) { // just 1 step propagation - auto deltaX = (math_utils::detail::sin(phiLocFin) - track.getSnp()) / track.getCurvature(bz); + auto deltaX = (math_utils::detail::sin(phiLocFin) - track.getSnp()) / track.getCurvature(bz); if (!propagateTo(track, track.getX() + deltaX, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr)) { return false; } @@ -637,7 +638,7 @@ GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, // propagate to phiLoc = +-MaxPhiLocSafe auto tgtPhiLoc = deltaPhi > 0 ? MaxPhiLocSafe : -MaxPhiLocSafe; - auto deltaX = (math_utils::detail::sin(tgtPhiLoc) - track.getSnp()) / track.getCurvature(bz); + auto deltaX = (math_utils::detail::sin(tgtPhiLoc) - track.getSnp()) / track.getCurvature(bz); if (!propagateTo(track, track.getX() + deltaX, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr)) { return false; } @@ -1093,11 +1094,13 @@ GPUd() void PropagatorImpl::getFieldXYZ(const math_utils::Point3D(xyz, bxyz); } +#ifndef __METAL__ // MSL has no double; the float twin remains template GPUd() void PropagatorImpl::getFieldXYZ(const math_utils::Point3D xyz, double* bxyz) const { getFieldXYZImpl(xyz, bxyz); } +#endif template GPUd() float PropagatorImpl::getBz(const math_utils::Point3D xyz) const @@ -1105,11 +1108,13 @@ GPUd() float PropagatorImpl::getBz(const math_utils::Point3D xyz return getBzImpl(xyz); } +#ifndef __METAL__ // MSL has no double; the float twin remains template GPUd() double PropagatorImpl::getBz(const math_utils::Point3D xyz) const { return getBzImpl(xyz); } +#endif namespace o2::base { diff --git a/Detectors/TRD/base/include/TRDBase/PadPlane.h b/Detectors/TRD/base/include/TRDBase/PadPlane.h index 24c67a8453b48..39132de72d404 100644 --- a/Detectors/TRD/base/include/TRDBase/PadPlane.h +++ b/Detectors/TRD/base/include/TRDBase/PadPlane.h @@ -15,6 +15,7 @@ // Forwards to standard header with protection for GPU compilation #include "GPUCommonRtypes.h" // for ClassDef #include "GPUCommonDef.h" +#include "GPUCommonDouble.h" //////////////////////////////////////////////////////////////////////////// // // @@ -39,53 +40,55 @@ class PadPlane PadPlane& operator=(const PadPlane& p) = delete; ~PadPlane() = default; +#ifndef GPUCA_GPUCODE_DEVICE void setLayer(int l) { mLayer = l; }; void setStack(int s) { mStack = s; }; - void setRowSpacing(double s) { mRowSpacing = s; }; - void setColSpacing(double s) { mColSpacing = s; }; - void setLengthRim(double l) { mLengthRim = l; }; - void setWidthRim(double w) { mWidthRim = w; }; + void setRowSpacing(o2::gpu::GPUdoubleValue s) { mRowSpacing = s; }; + void setColSpacing(o2::gpu::GPUdoubleValue s) { mColSpacing = s; }; + void setLengthRim(o2::gpu::GPUdoubleValue l) { mLengthRim = l; }; + void setWidthRim(o2::gpu::GPUdoubleValue w) { mWidthRim = w; }; void setNcols(int n); void setNrows(int n); - void setPadCol(int ic, double c) + void setPadCol(int ic, o2::gpu::GPUdoubleValue c) { if (ic < mNcols) { mPadCol[ic] = c; } }; - void setPadRow(int ir, double r) + void setPadRow(int ir, o2::gpu::GPUdoubleValue r) { if (ir < mNrows) { mPadRow[ir] = r; } }; - void setLength(double l) { mLength = l; }; - void setWidth(double w) { mWidth = w; }; - void setLengthOPad(double l) + void setLength(o2::gpu::GPUdoubleValue l) { mLength = l; }; + void setWidth(o2::gpu::GPUdoubleValue w) { mWidth = w; }; + void setLengthOPad(o2::gpu::GPUdoubleValue l) { mLengthOPad = l; mInverseLengthOPad = 1.0 / l; }; - void setWidthOPad(double w) + void setWidthOPad(o2::gpu::GPUdoubleValue w) { mWidthOPad = w; mInverseWidthOPad = 1.0 / w; }; - void setLengthIPad(double l) + void setLengthIPad(o2::gpu::GPUdoubleValue l) { mLengthIPad = l; mInverseLengthIPad = 1.0 / l; }; - void setWidthIPad(double w) + void setWidthIPad(o2::gpu::GPUdoubleValue w) { mWidthIPad = w; mInverseWidthIPad = 1.0 / w; }; - void setPadRowSMOffset(double o) { mPadRowSMOffset = o; }; + void setPadRowSMOffset(o2::gpu::GPUdoubleValue o) { mPadRowSMOffset = o; }; void setAnodeWireOffset(float o) { mAnodeWireOffset = o; }; - void setTiltingAngle(double t); + void setTiltingAngle(o2::gpu::GPUdoubleValue t); +#endif - GPUd() int getPadRowNumber(double z) const + GPUd() int getPadRowNumber(o2::gpu::GPUdoubleValue z) const { // // Finds the pad row number for a given z-position in local supermodule system @@ -103,10 +106,10 @@ class PadPlane nbelow = 0; while (nabove - nbelow > 1) { middle = (nabove + nbelow) / 2; - if (z == (mPadRow[middle - 1] + mPadRowSMOffset)) { + if (z == (o2::gpu::GPUdoubleGet(mPadRow[middle - 1]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset))) { row = middle; } - if (z > (mPadRow[middle - 1] + mPadRowSMOffset)) { + if (z > (o2::gpu::GPUdoubleGet(mPadRow[middle - 1]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset))) { nabove = middle; } else { nbelow = middle; @@ -118,124 +121,124 @@ class PadPlane return row; }; - GPUd() int getPadRowNumberROC(double z) const; - GPUd() double getPadRow(double z) const; - GPUd() int getPadColNumber(double rphi) const; - GPUd() double getPad(double y, double z) const; + GPUd() int getPadRowNumberROC(o2::gpu::GPUdoubleValue z) const; + GPUd() o2::gpu::GPUdoubleValue getPadRow(o2::gpu::GPUdoubleValue z) const; + GPUd() int getPadColNumber(o2::gpu::GPUdoubleValue rphi) const; + GPUd() o2::gpu::GPUdoubleValue getPad(o2::gpu::GPUdoubleValue y, o2::gpu::GPUdoubleValue z) const; - GPUd() double getTiltOffset(int row, double rowOffset) const + GPUd() o2::gpu::GPUdoubleValue getTiltOffset(int row, o2::gpu::GPUdoubleValue rowOffset) const { if (row == 0 || row == mNrows - 1) { - return mTiltingTan * (rowOffset - 0.5 * mLengthOPad); + return o2::gpu::GPUdoubleGet(mTiltingTan) * (rowOffset - 0.5 * o2::gpu::GPUdoubleGet(mLengthOPad)); } else { - return mTiltingTan * (rowOffset - 0.5 * mLengthIPad); + return o2::gpu::GPUdoubleGet(mTiltingTan) * (rowOffset - 0.5 * o2::gpu::GPUdoubleGet(mLengthIPad)); } }; - GPUd() double getPadRowOffset(int row, double z) const + GPUd() o2::gpu::GPUdoubleValue getPadRowOffset(int row, o2::gpu::GPUdoubleValue z) const { if ((row < 0) || (row >= mNrows)) { return -1.0; } else { - return mPadRow[row] + mPadRowSMOffset - z; + return o2::gpu::GPUdoubleGet(mPadRow[row]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset) - z; } }; - GPUd() double getPadRowOffsetROC(int row, double z) const + GPUd() o2::gpu::GPUdoubleValue getPadRowOffsetROC(int row, o2::gpu::GPUdoubleValue z) const { if ((row < 0) || (row >= mNrows)) { return -1.0; } else { - return mPadRow[row] - z; + return o2::gpu::GPUdoubleGet(mPadRow[row]) - z; } }; - GPUd() double getPadColOffset(int col, double rphi) const + GPUd() o2::gpu::GPUdoubleValue getPadColOffset(int col, o2::gpu::GPUdoubleValue rphi) const { if ((col < 0) || (col >= mNcols)) { return -1.0; } else { - return rphi - mPadCol[col]; + return rphi - o2::gpu::GPUdoubleGet(mPadCol[col]); } }; - GPUd() double getTiltingAngle() const { return mTiltingAngle; }; + GPUd() o2::gpu::GPUdoubleValue getTiltingAngle() const { return o2::gpu::GPUdoubleGet(mTiltingAngle); }; GPUd() int getNrows() const { return mNrows; }; GPUd() int getNcols() const { return mNcols; }; - GPUd() double getRow0() const { return mPadRow[0] + mPadRowSMOffset; }; - GPUd() double getRow0ROC() const { return mPadRow[0]; }; - GPUd() double getCol0() const { return mPadCol[0]; }; - GPUd() double getRowEnd() const { return mPadRow[mNrows - 1] - mLengthOPad + mPadRowSMOffset; }; - GPUd() double getRowEndROC() const { return mPadRow[mNrows - 1] - mLengthOPad; }; - GPUd() double getColEnd() const { return mPadCol[mNcols - 1] + mWidthOPad; }; - GPUd() double getRowPos(int row) const { return mPadRow[row] + mPadRowSMOffset; }; - GPUd() double getRowPosROC(int row) const { return mPadRow[row]; }; - GPUd() double getColPos(int col) const { return mPadCol[col]; }; - GPUd() double getRowSize(int row) const + GPUd() o2::gpu::GPUdoubleValue getRow0() const { return o2::gpu::GPUdoubleGet(mPadRow[0]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset); }; + GPUd() o2::gpu::GPUdoubleValue getRow0ROC() const { return o2::gpu::GPUdoubleGet(mPadRow[0]); }; + GPUd() o2::gpu::GPUdoubleValue getCol0() const { return o2::gpu::GPUdoubleGet(mPadCol[0]); }; + GPUd() o2::gpu::GPUdoubleValue getRowEnd() const { return o2::gpu::GPUdoubleGet(mPadRow[mNrows - 1]) - o2::gpu::GPUdoubleGet(mLengthOPad) + o2::gpu::GPUdoubleGet(mPadRowSMOffset); }; + GPUd() o2::gpu::GPUdoubleValue getRowEndROC() const { return o2::gpu::GPUdoubleGet(mPadRow[mNrows - 1]) - o2::gpu::GPUdoubleGet(mLengthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getColEnd() const { return o2::gpu::GPUdoubleGet(mPadCol[mNcols - 1]) + o2::gpu::GPUdoubleGet(mWidthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getRowPos(int row) const { return o2::gpu::GPUdoubleGet(mPadRow[row]) + o2::gpu::GPUdoubleGet(mPadRowSMOffset); }; + GPUd() o2::gpu::GPUdoubleValue getRowPosROC(int row) const { return o2::gpu::GPUdoubleGet(mPadRow[row]); }; + GPUd() o2::gpu::GPUdoubleValue getColPos(int col) const { return o2::gpu::GPUdoubleGet(mPadCol[col]); }; + GPUd() o2::gpu::GPUdoubleValue getRowSize(int row) const { if ((row == 0) || (row == mNrows - 1)) { - return mLengthOPad; + return o2::gpu::GPUdoubleGet(mLengthOPad); } else { - return mLengthIPad; + return o2::gpu::GPUdoubleGet(mLengthIPad); } }; - GPUd() double getColSize(int col) const + GPUd() o2::gpu::GPUdoubleValue getColSize(int col) const { if ((col == 0) || (col == mNcols - 1)) { - return mWidthOPad; + return o2::gpu::GPUdoubleGet(mWidthOPad); } else { - return mWidthIPad; + return o2::gpu::GPUdoubleGet(mWidthIPad); } }; - GPUd() double getLengthRim() const { return mLengthRim; }; - GPUd() double getWidthRim() const { return mWidthRim; }; - GPUd() double getRowSpacing() const { return mRowSpacing; }; - GPUd() double getColSpacing() const { return mColSpacing; }; - GPUd() double getLengthOPad() const { return mLengthOPad; }; - GPUd() double getLengthIPad() const { return mLengthIPad; }; - GPUd() double getWidthOPad() const { return mWidthOPad; }; - GPUd() double getWidthIPad() const { return mWidthIPad; }; - GPUd() double getAnodeWireOffset() const { return mAnodeWireOffset; }; + GPUd() o2::gpu::GPUdoubleValue getLengthRim() const { return o2::gpu::GPUdoubleGet(mLengthRim); }; + GPUd() o2::gpu::GPUdoubleValue getWidthRim() const { return o2::gpu::GPUdoubleGet(mWidthRim); }; + GPUd() o2::gpu::GPUdoubleValue getRowSpacing() const { return o2::gpu::GPUdoubleGet(mRowSpacing); }; + GPUd() o2::gpu::GPUdoubleValue getColSpacing() const { return o2::gpu::GPUdoubleGet(mColSpacing); }; + GPUd() o2::gpu::GPUdoubleValue getLengthOPad() const { return o2::gpu::GPUdoubleGet(mLengthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getLengthIPad() const { return o2::gpu::GPUdoubleGet(mLengthIPad); }; + GPUd() o2::gpu::GPUdoubleValue getWidthOPad() const { return o2::gpu::GPUdoubleGet(mWidthOPad); }; + GPUd() o2::gpu::GPUdoubleValue getWidthIPad() const { return o2::gpu::GPUdoubleGet(mWidthIPad); }; + GPUd() o2::gpu::GPUdoubleValue getAnodeWireOffset() const { return o2::gpu::GPUdoubleGet(mAnodeWireOffset); }; private: - static constexpr int MAXCOLS = 144; - static constexpr int MAXROWS = 16; + static GPUglobalconstexpr() int MAXCOLS = 144; + static GPUglobalconstexpr() int MAXROWS = 16; int mLayer; // Layer number int mStack; // Stack number - double mLength; // Length of pad plane in z-direction (row) - double mWidth; // Width of pad plane in rphi-direction (col) + o2::gpu::GPUdoubleStore mLength; // Length of pad plane in z-direction (row) + o2::gpu::GPUdoubleStore mWidth; // Width of pad plane in rphi-direction (col) - double mLengthRim; // Length of the rim in z-direction (row) - double mWidthRim; // Width of the rim in rphi-direction (col) + o2::gpu::GPUdoubleStore mLengthRim; // Length of the rim in z-direction (row) + o2::gpu::GPUdoubleStore mWidthRim; // Width of the rim in rphi-direction (col) - double mLengthOPad; // Length of an outer pad in z-direction (row) - double mWidthOPad; // Width of an outer pad in rphi-direction (col) + o2::gpu::GPUdoubleStore mLengthOPad; // Length of an outer pad in z-direction (row) + o2::gpu::GPUdoubleStore mWidthOPad; // Width of an outer pad in rphi-direction (col) - double mLengthIPad; // Length of an inner pad in z-direction (row) - double mWidthIPad; // Width of an inner pad in rphi-direction (col) + o2::gpu::GPUdoubleStore mLengthIPad; // Length of an inner pad in z-direction (row) + o2::gpu::GPUdoubleStore mWidthIPad; // Width of an inner pad in rphi-direction (col) - double mRowSpacing; // Spacing between the pad rows - double mColSpacing; // Spacing between the pad columns + o2::gpu::GPUdoubleStore mRowSpacing; // Spacing between the pad rows + o2::gpu::GPUdoubleStore mColSpacing; // Spacing between the pad columns int mNrows; // Number of rows int mNcols; // Number of columns - double mTiltingAngle; // Pad tilting angle - double mTiltingTan; // Tangens of pad tilting angle + o2::gpu::GPUdoubleStore mTiltingAngle; // Pad tilting angle + o2::gpu::GPUdoubleStore mTiltingTan; // Tangens of pad tilting angle - double mPadRow[MAXROWS]; // Pad border positions in row direction - double mPadCol[MAXCOLS]; // Pad border positions in column direction + o2::gpu::GPUdoubleStore mPadRow[MAXROWS]; // Pad border positions in row direction + o2::gpu::GPUdoubleStore mPadCol[MAXCOLS]; // Pad border positions in column direction - double mPadRowSMOffset; // To be added to translate local ROC system to local SM system + o2::gpu::GPUdoubleStore mPadRowSMOffset; // To be added to translate local ROC system to local SM system - double mAnodeWireOffset; // Distance of first anode wire from pad edge + o2::gpu::GPUdoubleStore mAnodeWireOffset; // Distance of first anode wire from pad edge - double mInverseLengthIPad; // 1 / mLengthIPad - double mInverseLengthOPad; // 1 / mLengthOPad + o2::gpu::GPUdoubleStore mInverseLengthIPad; // 1 / mLengthIPad + o2::gpu::GPUdoubleStore mInverseLengthOPad; // 1 / mLengthOPad - double mInverseWidthIPad; // 1 / mWidthIPad - double mInverseWidthOPad; // 1 / mWidthOPad + o2::gpu::GPUdoubleStore mInverseWidthIPad; // 1 / mWidthIPad + o2::gpu::GPUdoubleStore mInverseWidthOPad; // 1 / mWidthOPad ClassDefNV(PadPlane, 2); // TRD ROC pad plane }; diff --git a/GPU/Common/GPUCommonAlgorithm.h b/GPU/Common/GPUCommonAlgorithm.h index be88973561e0a..24d025cebfa56 100644 --- a/GPU/Common/GPUCommonAlgorithm.h +++ b/GPU/Common/GPUCommonAlgorithm.h @@ -32,16 +32,16 @@ class GPUCommonAlgorithm template GPUd() static void sort(T* begin, T* end); template - GPUd() static void sortInBlock(T* begin, T* end); + GPUd() static void sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end); template GPUd() static void sortDeviceDynamic(T* begin, T* end); template GPUd() static void sort(T* begin, T* end, const S& comp); template - GPUd() static void sortInBlock(T* begin, T* end, const S& comp); + GPUd() static void sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end, const S& comp); template GPUd() static void sortDeviceDynamic(T* begin, T* end, const S& comp); -#ifndef __OPENCL__ +#if !defined(__OPENCL__) && !defined(__METAL__) // auto parameters are C++20; both are C++17 template GPUh() static void sortOnDevice(auto* rec, int32_t stream, T* begin, size_t N, const S& comp); #endif @@ -51,32 +51,32 @@ class GPUCommonAlgorithm private: // Quicksort implementation template - GPUd() static void QuickSort(I f, I l) noexcept; + GPUd() static void QuickSort(I f, I l) GPUnoexcept(); // Quicksort implementation template - GPUd() static void QuickSort(I f, I l, Cmp cmp) noexcept; + GPUd() static void QuickSort(I f, I l, Cmp cmp) GPUnoexcept(); // Insertionsort implementation template - GPUd() static void InsertionSort(I f, I l, Cmp cmp) noexcept; + GPUd() static void InsertionSort(I f, I l, Cmp cmp) GPUnoexcept(); // Helper for Quicksort implementation template - GPUd() static I MedianOf3Select(I f, I l, Cmp cmp) noexcept; + GPUd() static I MedianOf3Select(I f, I l, Cmp cmp) GPUnoexcept(); // Helper for Quicksort implementation template - GPUd() static I UnguardedPartition(I f, I l, T piv, Cmp cmp) noexcept; + GPUd() static I UnguardedPartition(I f, I l, T piv, Cmp cmp) GPUnoexcept(); // Helper template - GPUd() static void IterSwap(I a, I b) noexcept; + GPUd() static void IterSwap(I a, I b) GPUnoexcept(); }; #ifndef GPUCA_ALGORITHM_STD template -GPUdi() void GPUCommonAlgorithm::IterSwap(I a, I b) noexcept +GPUdi() void GPUCommonAlgorithm::IterSwap(I a, I b) GPUnoexcept() { auto tmp = *a; *a = *b; @@ -84,7 +84,7 @@ GPUdi() void GPUCommonAlgorithm::IterSwap(I a, I b) noexcept } template -GPUdi() void GPUCommonAlgorithm::InsertionSort(I f, I l, Cmp cmp) noexcept +GPUdi() void GPUCommonAlgorithm::InsertionSort(I f, I l, Cmp cmp) GPUnoexcept() { auto it0{f}; while (it0 != l) { @@ -102,7 +102,7 @@ GPUdi() void GPUCommonAlgorithm::InsertionSort(I f, I l, Cmp cmp) noexcept } template -GPUdi() I GPUCommonAlgorithm::MedianOf3Select(I f, I l, Cmp cmp) noexcept +GPUdi() I GPUCommonAlgorithm::MedianOf3Select(I f, I l, Cmp cmp) GPUnoexcept() { auto m = f + (l - f) / 2; @@ -126,7 +126,7 @@ GPUdi() I GPUCommonAlgorithm::MedianOf3Select(I f, I l, Cmp cmp) noexcept } template -GPUdi() I GPUCommonAlgorithm::UnguardedPartition(I f, I l, T piv, Cmp cmp) noexcept +GPUdi() I GPUCommonAlgorithm::UnguardedPartition(I f, I l, T piv, Cmp cmp) GPUnoexcept() { do { while (cmp(*f, piv)) { @@ -146,7 +146,7 @@ GPUdi() I GPUCommonAlgorithm::UnguardedPartition(I f, I l, T piv, Cmp cmp) noexc } template -GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l, Cmp cmp) noexcept +GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l, Cmp cmp) GPUnoexcept() { if (f == l) { return; @@ -204,7 +204,7 @@ GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l, Cmp cmp) noexcept } template -GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l) noexcept +GPUdi() void GPUCommonAlgorithm::QuickSort(I f, I l) GPUnoexcept() { QuickSort(f, l, [](auto&& x, auto&& y) { return x < y; }); } @@ -268,29 +268,29 @@ GPUdi() void GPUCommonAlgorithm::sort(T* begin, T* end, const S& comp) } template -GPUdi() void GPUCommonAlgorithm::sortInBlock(T* begin, T* end) +GPUdi() void GPUCommonAlgorithm::sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end) { #ifndef GPUCA_GPUCODE GPUCommonAlgorithm::sort(begin, end); #else - GPUCommonAlgorithm::sortInBlock(begin, end, [](auto&& x, auto&& y) { return x < y; }); + GPUCommonAlgorithm::sortInBlock(nThreads, iThread, begin, end, [](auto&& x, auto&& y) { return x < y; }); #endif } template -GPUdi() void GPUCommonAlgorithm::sortInBlock(T* begin, T* end, const S& comp) +GPUdi() void GPUCommonAlgorithm::sortInBlock(int32_t nThreads, int32_t iThread, T* begin, T* end, const S& comp) { #ifndef GPUCA_GPUCODE GPUCommonAlgorithm::sort(begin, end, comp); #elif defined(GPUCA_DETERMINISTIC_MODE) // Not using GPUCA_DETERMINISTIC_CODE, which is enforced in TPC compression - if (get_local_id(0) == 0) { + if (iThread == 0) { GPUCommonAlgorithm::sort(begin, end, comp); } GPUbarrier(); #else int32_t n = end - begin; for (int32_t i = 0; i < n; i++) { - for (int32_t tIdx = get_local_id(0); tIdx < n; tIdx += get_local_size(0)) { + for (int32_t tIdx = iThread; tIdx < n; tIdx += nThreads) { int32_t offset = i % 2; int32_t curPos = 2 * tIdx + offset; int32_t nextPos = curPos + 1; diff --git a/GPU/Common/GPUCommonDef.h b/GPU/Common/GPUCommonDef.h index 6e2269650f576..38e2ac35c5b8d 100644 --- a/GPU/Common/GPUCommonDef.h +++ b/GPU/Common/GPUCommonDef.h @@ -58,7 +58,10 @@ #define GPUCA_ALIGPUCODE // Part of GPUTracking library but not of interface #endif -#if (defined(__CUDACC__) && defined(GPUCA_CUDA_NO_CONSTANT_MEMORY)) || (defined(__HIPCC__) && defined(GPUCA_HIP_NO_CONSTANT_MEMORY)) || (defined(__OPENCL__) && defined(GPUCA_OPENCL_NO_CONSTANT_MEMORY)) +// __METAL__ unconditionally: the MSL generic address space does not span +// `constant`, so a generic member function cannot be called on an object living +// there, and the shared code is generic throughout. +#if (defined(__CUDACC__) && defined(GPUCA_CUDA_NO_CONSTANT_MEMORY)) || (defined(__HIPCC__) && defined(GPUCA_HIP_NO_CONSTANT_MEMORY)) || (defined(__OPENCL__) && defined(GPUCA_OPENCL_NO_CONSTANT_MEMORY)) || defined(__METAL__) #define GPUCA_NO_CONSTANT_MEMORY #elif (defined(__CUDACC__) || defined(__HIPCC__)) && !defined(GPUCA_GPUCODE_HOSTONLY) #define GPUCA_HAS_GLOBAL_SYMBOL_CONSTANT_MEM @@ -78,6 +81,12 @@ #define GPUCA_RTC_CONSTEXPR #endif +#if defined(GPUCA_DETERMINISTIC_MODE) && defined(__METAL__) + // The deterministic paths compute in double (see GPUCommonMath::SinCos) and + // MSL has no double, so the results could not match the other backends. + #error "GPUCA_DETERMINISTIC_MODE is not supported on Metal" +#endif + #ifndef GPUCA_DETERMINISTIC_CODE #ifdef GPUCA_DETERMINISTIC_MODE #define GPUCA_DETERMINISTIC_CODE(det, indet) det // In deterministic mode, take deterministic code path diff --git a/GPU/Common/GPUCommonDefAPI.h b/GPU/Common/GPUCommonDefAPI.h index a04934304d525..d3cdf5cf2db33 100644 --- a/GPU/Common/GPUCommonDefAPI.h +++ b/GPU/Common/GPUCommonDefAPI.h @@ -49,6 +49,7 @@ #define GPUconstant() // constant memory variable declaraion #define GPUconstexpr() static constexpr // constexpr on GPU that needs to be instantiated for dynamic access (e.g. arrays), becomes __constant on GPU #define GPUglobalconstexpr() constexpr // constexpr variable at program scope, needs the constant address space in MSL + #define GPUnoexcept() noexcept // noexcept where the backend supports it #define GPUprivate() // private memory variable declaration #define GPUgeneric() // reference / ptr to generic address space #define GPUbarrier() // synchronize all GPU threads in block @@ -162,17 +163,20 @@ #define GPUconstant() constant // TODO: possibly add const __restrict where possible later! #define GPUconstexpr() constant #define GPUglobalconstexpr() constant constexpr + #define GPUnoexcept() #define GPUprivate() thread #define GPUgeneric() - #define GPUglobalref() device + #define GPUglobalref() #define GPUsharedref() threadgroup #define GPUprivateref() thread - #define GPUconstantref() constant + #if !defined(GPUCA_NO_CONSTANT_MEMORY) + #define GPUconstantref() constant + #endif #define GPUconstexprref() GPUconstexpr() #define GPUdouble() float #define GPUbarrier() threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup) #define GPUbarrierWarp() simdgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup) - #define GPUAtomic(type) atomic // atomic variable type + #define GPUAtomic(type) type // atomic variable type #elif defined(__HIPCC__) //Defines for HIP #define GPUd() __device__ #define GPUdDefault() __device__ @@ -260,6 +264,9 @@ #ifndef GPUglobalconstexpr #define GPUglobalconstexpr() constexpr #endif +#ifndef GPUnoexcept +#define GPUnoexcept() noexcept +#endif #define GPUrestrict() __restrict__ @@ -273,6 +280,15 @@ #define get_group_id(dim) (blockIdx.x) #elif defined(__OPENCL__) // Using OpenCL defaults +#elif defined(__METAL__) + // MSL has no work-item builtins; these come in as kernel attributes, declared + // by GPUCA_KRNL_GRID_ARGS on every entry point. + #define get_global_id(dim) (_metalTgIg * _metalTPerTg + _metalTiTg) + #define get_global_size(dim) (_metalTPerTg * _metalTgPerG) + #define get_num_groups(dim) (_metalTgPerG) + #define get_local_id(dim) (_metalTiTg) + #define get_local_size(dim) (_metalTPerTg) + #define get_group_id(dim) (_metalTgIg) #else #define get_global_id(dim) iBlock #define get_global_size(dim) nBlocks diff --git a/GPU/Common/GPUCommonDouble.h b/GPU/Common/GPUCommonDouble.h new file mode 100644 index 0000000000000..1241d97e90c09 --- /dev/null +++ b/GPU/Common/GPUCommonDouble.h @@ -0,0 +1,252 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUCommonDouble.h +/// \brief Storage for double precision members shared with a device that has none + +#ifndef GPUCOMMONDOUBLE_H +#define GPUCOMMONDOUBLE_H + +#include "GPUCommonDef.h" +#include "GPUCommonMath.h" +#include "GPUCommonDoubleBinary64.h" + +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif + +namespace o2::gpu +{ + +#ifdef __METAL__ + +// MSL has no double, and rejects it at the declaration, so a struct holding one +// cannot even be declared. GPUdoubleStore occupies the same eight bytes with the +// same alignment, which keeps the object layout identical to the host's, and the +// stored bits are decoded on read. Storage is therefore untouched: the host still +// writes a double and the same bytes are uploaded. +struct alignas(8) GPUdoubleStore { + uint32_t mLo; + uint32_t mHi; +}; +typedef float GPUdoubleValue; + +// IEEE-754 binary64 -> binary32, round to nearest even, including the subnormal, +// infinity and NaN cases. +GPUdi() GPUdoubleValue GPUdoubleGet(GPUdoubleStore d) +{ + const uint32_t sign = d.mHi & 0x80000000u; + const int32_t be = int32_t((d.mHi >> 20) & 0x7ffu); + const uint32_t man = ((d.mHi & 0x000fffffu) << 3) | (d.mLo >> 29); + const uint32_t drop = d.mLo & 0x1fffffffu; + uint32_t bits; + if (be == 0x7ff) { + bits = sign | 0x7f800000u | (((d.mHi & 0x000fffffu) | d.mLo) ? 0x400000u : 0u); + } else if (be == 0) { + bits = sign; + } else { + const int32_t e = be - 1023 + 127; + if (e >= 0xff) { + bits = sign | 0x7f800000u; + } else if (e > 0) { + bits = sign | (uint32_t(e) << 23) | man; + if ((drop & 0x10000000u) && ((drop & 0x0fffffffu) || (man & 1u))) { + bits += 1u; + } + } else if (e > -24) { + const uint32_t full = man | 0x800000u; + const uint32_t sh = uint32_t(1 - e); + const uint32_t lost = full & ((1u << sh) - 1u); + const uint32_t halfb = 1u << (sh - 1); + uint32_t sub = full >> sh; + if (lost > halfb || (lost == halfb && ((sub & 1u) || drop))) { + sub += 1u; + } + bits = sign | sub; + } else { + bits = sign; + } + } + return __builtin_bit_cast(float, bits); +} + +#else + +typedef double GPUdoubleStore; +typedef double GPUdoubleValue; +GPUhdi() GPUdoubleValue GPUdoubleGet(GPUdoubleStore d) { return d; } + +#endif + +static_assert(sizeof(GPUdoubleStore) == 8, "GPUdoubleStore must match the size of a double"); +static_assert(alignof(GPUdoubleStore) == 8, "GPUdoubleStore must match the alignment of a double"); + + +// Compensated two-float arithmetic, value = mHi + mLo, for the intermediates that +// are deliberately computed in double even when the track itself is float -- the +// Jacobian and covariance terms in TrackParametrizationWithError::propagateTo and +// friends, where differences of nearly equal quantities cancel. +// +// Every operation here depends on the compiler not reassociating the compensation +// terms away, which is why the Metal device code is built without fast math; when +// it is built with fast math, GPUdoubleCalc below is a plain float instead. +// +// mLo is not renormalised after each operation: the value is still mHi + mLo and +// nothing downstream requires |mLo| <= ulp(mHi)/2. +// +// Defined for every backend so it can be tested on the host, but only Metal uses +// it: everywhere else GPUdoubleCalc is a plain double. +class GPUdoubleCalcImpl +{ + public: + GPUdDefault() GPUdoubleCalcImpl() = default; + GPUdi() GPUdoubleCalcImpl(float v) : mHi(v), mLo(0.f) {} + GPUdi() GPUdoubleCalcImpl(float hi, float lo) : mHi(hi), mLo(lo) {} + // implicit, exactly as double narrows to float: the call sites assign these + // intermediates straight back into value_t covariance elements + GPUdi() operator float() const { return mHi + mLo; } + + GPUdi() GPUdoubleCalcImpl operator-() const { return GPUdoubleCalcImpl(-mHi, -mLo); } + GPUdi() GPUdoubleCalcImpl operator+(GPUdoubleCalcImpl b) const + { + const float s = mHi + b.mHi, bb = s - mHi; + return GPUdoubleCalcImpl(s, ((mHi - (s - bb)) + (b.mHi - bb)) + (mLo + b.mLo)); + } + GPUdi() GPUdoubleCalcImpl operator-(GPUdoubleCalcImpl b) const { return *this + (-b); } + GPUdi() GPUdoubleCalcImpl operator*(GPUdoubleCalcImpl b) const + { + const float p = mHi * b.mHi; + return GPUdoubleCalcImpl(p, __builtin_fmaf(mHi, b.mHi, -p) + (mHi * b.mLo + mLo * b.mHi)); + } + GPUdi() GPUdoubleCalcImpl operator/(GPUdoubleCalcImpl b) const + { + const float q = mHi / b.mHi; + const float r = (__builtin_fmaf(-q, b.mHi, mHi) + mLo) - q * b.mLo; + return GPUdoubleCalcImpl(q, r / b.mHi); + } + + // exact matches for the mixed forms, so `a * someFloat` does not sit ambiguously + // between converting the float up and converting *this down. They also skip the + // mLo terms that are zero for a float operand, which no reassociation is allowed + // to fold away here. + GPUdi() GPUdoubleCalcImpl operator+(float b) const + { + const float s = mHi + b, bb = s - mHi; + return GPUdoubleCalcImpl(s, ((mHi - (s - bb)) + (b - bb)) + mLo); + } + GPUdi() GPUdoubleCalcImpl operator-(float b) const { return *this + (-b); } + GPUdi() GPUdoubleCalcImpl operator*(float b) const + { + const float p = mHi * b; + return GPUdoubleCalcImpl(p, __builtin_fmaf(mHi, b, -p) + mLo * b); + } + GPUdi() GPUdoubleCalcImpl operator/(float b) const + { + const float q = mHi / b; + const float r = __builtin_fmaf(-q, b, mHi) + mLo; + return GPUdoubleCalcImpl(q, r / b); + } + // MSL has no double, so on Metal a literal like `1.` is already float and only + // the float forms are ever selected. The double forms exist so the type can be + // compiled and tested on the host, where such literals really are double. +#ifndef __METAL__ + GPUdi() GPUdoubleCalcImpl operator+(double b) const { return *this + (float)b; } + GPUdi() GPUdoubleCalcImpl operator-(double b) const { return *this - (float)b; } + GPUdi() GPUdoubleCalcImpl operator*(double b) const { return *this * (float)b; } + GPUdi() GPUdoubleCalcImpl operator/(double b) const { return *this / (float)b; } +#endif + + GPUdi() GPUdoubleCalcImpl& operator+=(GPUdoubleCalcImpl b) { return *this = *this + b; } + GPUdi() GPUdoubleCalcImpl& operator-=(GPUdoubleCalcImpl b) { return *this = *this - b; } + GPUdi() GPUdoubleCalcImpl& operator*=(GPUdoubleCalcImpl b) { return *this = *this * b; } + GPUdi() GPUdoubleCalcImpl& operator/=(GPUdoubleCalcImpl b) { return *this = *this / b; } + + private: + float mHi, mLo; +}; + +#ifndef __METAL__ +GPUdi() GPUdoubleCalcImpl operator+(double a, GPUdoubleCalcImpl b) { return b + (float)a; } +GPUdi() GPUdoubleCalcImpl operator-(double a, GPUdoubleCalcImpl b) { return (-b) + (float)a; } +GPUdi() GPUdoubleCalcImpl operator*(double a, GPUdoubleCalcImpl b) { return b * (float)a; } +GPUdi() GPUdoubleCalcImpl operator/(double a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl((float)a) / b; } +#endif +GPUdi() GPUdoubleCalcImpl operator+(float a, GPUdoubleCalcImpl b) { return b + a; } +GPUdi() GPUdoubleCalcImpl operator-(float a, GPUdoubleCalcImpl b) { return (-b) + a; } +GPUdi() GPUdoubleCalcImpl operator*(float a, GPUdoubleCalcImpl b) { return b * a; } +GPUdi() GPUdoubleCalcImpl operator/(float a, GPUdoubleCalcImpl b) { return GPUdoubleCalcImpl(a) / b; } + +// rounds once, as `someFloat += someDouble` does on the host. MSL resolves the +// address spaces separately, and a generic reference would tie with the built-in +// float += float rather than beat it. +#ifdef __METAL__ +GPUdi() thread float& operator+=(thread float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +GPUdi() device float& operator+=(device float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +GPUdi() threadgroup float& operator+=(threadgroup float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +#else +GPUdi() float& operator+=(float& a, GPUdoubleCalcImpl b) { return a = (float)(GPUdoubleCalcImpl(a) + b); } +#endif + +// CAMath::Abs is a template that deduces its parameter, so a call on one of these +// types picks the primary template, which has no definition. The rest of CAMath +// takes float and is reached through the implicit conversion. +template <> +GPUhdi() GPUdoubleCalcImpl GPUCommonMath::Abs(GPUdoubleCalcImpl x) +{ + return (float)x < 0.f ? -x : x; +} + +template <> +GPUhdi() GPUdoubleBinary64 GPUCommonMath::Abs(GPUdoubleBinary64 x) +{ + return GPUdoubleBinary64::fromBits(x.bits() & ~GPUCA_B64_SIGN); +} + +// What GPUdoubleCalc is. Define GPUCA_DOUBLECALC to one of these to override the +// default, which is hardware double everywhere except Metal, and the compensated +// two-float type there. A host build can be set to any of them to compare the +// representations against each other; a Metal build can be set to BINARY64 to +// reproduce the CPU result bit for bit, at two orders of magnitude the cost. +#define GPUCA_DOUBLECALC_DOUBLE 1 +#define GPUCA_DOUBLECALC_FLOAT 2 +#define GPUCA_DOUBLECALC_TWOFLOAT 3 +#define GPUCA_DOUBLECALC_BINARY64 4 + +#ifndef GPUCA_DOUBLECALC + #if defined(__METAL__) && defined(__FAST_MATH__) + // Fast math reassociates the compensation terms away, so the two-float type + // would cost 1.5x for the accuracy of a plain float. + #define GPUCA_DOUBLECALC GPUCA_DOUBLECALC_FLOAT + #elif defined(__METAL__) + #define GPUCA_DOUBLECALC GPUCA_DOUBLECALC_TWOFLOAT + #else + #define GPUCA_DOUBLECALC GPUCA_DOUBLECALC_DOUBLE + #endif +#endif + +#if GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_DOUBLE + #ifdef __METAL__ + #error "MSL has no double; GPUCA_DOUBLECALC_DOUBLE cannot be selected for Metal" + #endif +typedef double GPUdoubleCalc; +#elif GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_FLOAT +typedef float GPUdoubleCalc; +#elif GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_TWOFLOAT +typedef GPUdoubleCalcImpl GPUdoubleCalc; +#elif GPUCA_DOUBLECALC == GPUCA_DOUBLECALC_BINARY64 +typedef GPUdoubleBinary64 GPUdoubleCalc; +#else + #error "Invalid setting for GPUCA_DOUBLECALC" +#endif + +} // namespace o2::gpu + +#endif // GPUCOMMONDOUBLE_H diff --git a/GPU/Common/GPUCommonDoubleBinary64.h b/GPU/Common/GPUCommonDoubleBinary64.h new file mode 100644 index 0000000000000..07ef68cd834de --- /dev/null +++ b/GPU/Common/GPUCommonDoubleBinary64.h @@ -0,0 +1,390 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUCommonDoubleBinary64.h +/// \brief IEEE-754 binary64 in software, for a device that has none + +#ifndef GPUCOMMONDOUBLEBINARY64_H +#define GPUCOMMONDOUBLEBINARY64_H + +#include "GPUCommonDef.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif + +// IEEE-754 binary64 in software, for a device that has no double at all. Round to +// nearest even only, with subnormals, infinities and NaNs; NaN propagation follows +// the ARM64 order, so an Apple host is a bit-exact reference down to the payload. +// There is no fused multiply-add, no square root and no other rounding mode. +// +// This is not meant for production: it costs of the order of a hundred times plain +// float on an M-series GPU, against roughly two for GPUdoubleCalcImpl. It is here so that a +// Metal build can be made to reproduce the CPU result exactly, which turns a +// disagreement between the two into a search over code rather than over numerics. + +namespace o2::gpu +{ + +namespace binary64_detail +{ + +#ifdef __METAL__ +typedef ulong u64; +typedef long i64; +typedef uint u32; +#else +typedef uint64_t u64; +typedef int64_t i64; +typedef uint32_t u32; +#endif + +#define GPUCA_B64_ALWAYS inline __attribute__((always_inline)) +#ifdef __METAL__ +// The arithmetic has to stay out of line: inlined into a kernel it drops the +// occupancy (maxTotalThreadsPerThreadgroup 832 -> 384) and runs two to five +// times slower than the call. +#define GPUCA_B64_OP __attribute__((noinline)) +#else +#define GPUCA_B64_OP inline +#endif + +#ifdef __METAL__ +GPUCA_B64_ALWAYS int32_t clz64(u64 x) { return (int32_t)metal::clz(x); } // clz(0) == 64 in MSL +GPUCA_B64_ALWAYS int32_t clz32(u32 x) { return (int32_t)metal::clz(x); } +GPUCA_B64_ALWAYS u64 mulhiu(u64 a, u64 b) { return metal::mulhi(a, b); } +GPUCA_B64_ALWAYS i64 mulhis(i64 a, i64 b) { return metal::mulhi(a, b); } +GPUCA_B64_ALWAYS u32 asu32(float f) { return as_type(f); } +GPUCA_B64_ALWAYS float asf32(u32 u) { return as_type(u); } +#else +GPUCA_B64_ALWAYS int32_t clz64(u64 x) { return x ? __builtin_clzll(x) : 64; } +GPUCA_B64_ALWAYS int32_t clz32(u32 x) { return x ? __builtin_clz(x) : 32; } +GPUCA_B64_ALWAYS u64 mulhiu(u64 a, u64 b) { return (u64)(((unsigned __int128)a * b) >> 64); } +GPUCA_B64_ALWAYS i64 mulhis(i64 a, i64 b) { return (i64)(((__int128)a * b) >> 64); } +GPUCA_B64_ALWAYS u32 asu32(float f) { return __builtin_bit_cast(uint32_t, f); } +GPUCA_B64_ALWAYS float asf32(u32 u) { return __builtin_bit_cast(float, u); } +#endif + +#define GPUCA_B64_SIGN 0x8000000000000000ULL +#define GPUCA_B64_FRAC 0x000fffffffffffffULL +#define GPUCA_B64_IMPL 0x0010000000000000ULL +#define GPUCA_B64_QUIET 0x0008000000000000ULL +#define GPUCA_B64_INF 0x7ff0000000000000ULL +#define GPUCA_B64_DNAN 0x7ff8000000000000ULL // ARM default NaN + +// right shift keeping a sticky bit; any count >= 0, no shift count ever reaches 64 +GPUCA_B64_ALWAYS u64 shrJam(u64 m, int32_t s) +{ + const int32_t sc = s > 63 ? 63 : s; + const u64 r = m >> sc; + const u64 lost = (m << (63 - sc)) << 1; + const bool big = s > 63; + const u64 rr = big ? 0ULL : r; + const u64 st = big ? m : lost; + return rr | (st != 0 ? 1ULL : 0ULL); +} + +// m: leading bit at 62 for a normal result, 10 guard bits below the 53-bit significand, +// value = m * 2^(e - 1085) with e the biased exponent. Packing (e-1)<<52 + m lets a +// rounding carry ripple into the exponent. +GPUCA_B64_ALWAYS u64 roundPack(u64 sign, int32_t e, u64 m) +{ + if (e >= 0x7ff) { + return sign | GPUCA_B64_INF; + } + if (e <= 0) { + m = shrJam(m, 1 - e); + e = 1; + } + const u64 r = m & 0x3ffULL; + m >>= 10; + if (r > 0x200ULL || (r == 0x200ULL && (m & 1ULL))) { + ++m; + } + return sign | (((u64)(e - 1) << 52) + m); +} + +// an sNaN operand wins over a qNaN one, among equals the first wins, the result is quietened +GPUCA_B64_ALWAYS u64 propNaN(u64 a, u64 b, bool an, bool bn) +{ + const bool as = an && !(a & GPUCA_B64_QUIET), bs = bn && !(b & GPUCA_B64_QUIET); + if (as) { + return a | GPUCA_B64_QUIET; + } + if (bs) { + return b | GPUCA_B64_QUIET; + } + return an ? a : b; +} + +GPUCA_B64_OP u64 addsub(u64 a, u64 b0, bool neg) +{ + const u64 b = neg ? (b0 ^ GPUCA_B64_SIGN) : b0; + const bool sw = (a & ~GPUCA_B64_SIGN) < (b & ~GPUCA_B64_SIGN); + const u64 x = sw ? b : a, y = sw ? a : b; // |x| >= |y| + int32_t ex = (int32_t)((x >> 52) & 0x7ff), ey = (int32_t)((y >> 52) & 0x7ff); + u64 mx = x & GPUCA_B64_FRAC, my = y & GPUCA_B64_FRAC; + if (ex == 0x7ff) { // y can only be inf/NaN if x is too + const bool an = ((a >> 52) & 0x7ff) == 0x7ff && (a & GPUCA_B64_FRAC) != 0, bn = ((b0 >> 52) & 0x7ff) == 0x7ff && (b0 & GPUCA_B64_FRAC) != 0; + if (an || bn) { + return propNaN(a, b0, an, bn); + } + if (ey == 0x7ff) { + return ((x ^ y) & GPUCA_B64_SIGN) ? GPUCA_B64_DNAN : x; + } + return x; + } + const u64 sx = x & GPUCA_B64_SIGN; + const bool sub = ((x ^ y) >> 63) != 0; + mx = (ex ? (mx | GPUCA_B64_IMPL) : mx) << 10; + ex = ex ? ex : 1; + my = (ey ? (my | GPUCA_B64_IMPL) : my) << 10; + ey = ey ? ey : 1; + my = shrJam(my, ex - ey); + const u64 s = sub ? mx - my : mx + my; + const bool carry = (s >> 63) != 0; + const int32_t lz = clz64(s) - 1; // -1 on carry, 63 on zero + const u64 sN = carry ? ((s >> 1) | (s & 1ULL)) : (s << (lz & 63)); + const int32_t eN = carry ? ex + 1 : ex - lz; + const bool zero = s == 0; + return roundPack((zero && sub) ? 0ULL : sx, zero ? -1 : eN, sN); +} + +GPUCA_B64_OP u64 mul(u64 a, u64 b) +{ + const u64 sign = (a ^ b) & GPUCA_B64_SIGN; + int32_t ea = (int32_t)((a >> 52) & 0x7ff), eb = (int32_t)((b >> 52) & 0x7ff); + u64 ma = a & GPUCA_B64_FRAC, mb = b & GPUCA_B64_FRAC; + if (ea == 0x7ff || eb == 0x7ff) { + const bool an = ea == 0x7ff && ma != 0, bn = eb == 0x7ff && mb != 0; + if (an || bn) { + return propNaN(a, b, an, bn); + } + if ((ea == 0 && ma == 0) || (eb == 0 && mb == 0)) { + return GPUCA_B64_DNAN; // inf * 0 + } + return sign | GPUCA_B64_INF; + } + if (ea == 0) { + if (ma == 0) { + return sign; + } + const int32_t lz = clz64(ma) - 11; + ma <<= lz; + ea = 1 - lz; + } else { + ma |= GPUCA_B64_IMPL; + } + if (eb == 0) { + if (mb == 0) { + return sign; + } + const int32_t lz = clz64(mb) - 11; + mb <<= lz; + eb = 1 - lz; + } else { + mb |= GPUCA_B64_IMPL; + } + const u64 lo = ma * mb, hi = mulhiu(ma, mb); // 106-bit product in [2^104, 2^106) + const bool top = (hi & (1ULL << 41)) != 0; + const u64 m1 = (hi << 21) | (lo >> 43), m0 = (hi << 22) | (lo >> 42); + const u64 st = top ? (lo & ((1ULL << 43) - 1)) : (lo & ((1ULL << 42) - 1)); + const u64 m = (top ? m1 : m0) | (st != 0 ? 1ULL : 0ULL); + return roundPack(sign, ea + eb - 1023 + (top ? 1 : 0), m); +} + +GPUCA_B64_OP u64 div(u64 a, u64 b) +{ + const u64 sign = (a ^ b) & GPUCA_B64_SIGN; + int32_t ea = (int32_t)((a >> 52) & 0x7ff), eb = (int32_t)((b >> 52) & 0x7ff); + u64 ma = a & GPUCA_B64_FRAC, mb = b & GPUCA_B64_FRAC; + if (ea == 0x7ff || eb == 0x7ff) { + const bool an = ea == 0x7ff && ma != 0, bn = eb == 0x7ff && mb != 0; + if (an || bn) { + return propNaN(a, b, an, bn); + } + if (ea == 0x7ff && eb == 0x7ff) { + return GPUCA_B64_DNAN; // inf / inf + } + return ea == 0x7ff ? (sign | GPUCA_B64_INF) : sign; // inf / x, x / inf + } + if (eb == 0 && mb == 0) { + return (ea == 0 && ma == 0) ? GPUCA_B64_DNAN : (sign | GPUCA_B64_INF); // x / 0 + } + if (ea == 0) { + if (ma == 0) { + return sign; + } + const int32_t lz = clz64(ma) - 11; + ma <<= lz; + ea = 1 - lz; + } else { + ma |= GPUCA_B64_IMPL; + } + if (eb == 0) { + const int32_t lz = clz64(mb) - 11; + mb <<= lz; + eb = 1 - lz; + } else { + mb |= GPUCA_B64_IMPL; + } + const bool lt = ma < mb; + const int32_t e = ea - eb + 1023 - (lt ? 1 : 0); + const u64 A2 = lt ? (ma << 1) : ma; // A2 / mb in [1, 2) + // reciprocal R ~ 2^114 / mb in (2^61, 2^62], seeded from a float division on the top 24 bits + const u32 rb = asu32(1.0f / (float)(u32)(mb >> 29)); + u64 R = (u64)((rb & 0x7fffffu) | 0x800000u) << ((int32_t)((rb >> 23) & 0xff) - 127 + 62); + const u64 Bn = mb << 11; + for (int32_t it = 0; it < 2; ++it) { + const i64 E = (i64)(1ULL << 61) - (i64)mulhiu(Bn, R); + R = (u64)((i64)R + mulhis((i64)R, E << 3)); + } + R = R > (1ULL << 62) ? (1ULL << 62) : R; + // Q ~ A2 * 2^62 / mb, then the exact remainder, which fits in a signed 64-bit + // word because Q is within a few units + u64 Q = mulhiu(A2 << 10, (R << 2) - 1); + i64 rem = (i64)((A2 << 62) - Q * mb); + const i64 adj = mulhis(rem, (i64)R) >> 50; + Q = (u64)((i64)Q + adj); + rem -= adj * (i64)mb; + // after adj the remainder is within one divisor of [0, mb): one predicated step each way + const bool ng = rem < 0; + Q = ng ? Q - 1 : Q; + rem = ng ? rem + (i64)mb : rem; + const bool bg = rem >= (i64)mb; + Q = bg ? Q + 1 : Q; + rem = bg ? rem - (i64)mb : rem; + return roundPack(sign, e, Q | (rem != 0 ? 1ULL : 0ULL)); +} + +GPUCA_B64_OP u64 fromFloat(float f) +{ + const u32 u = asu32(f); + const u64 sign = (u64)(u & 0x80000000u) << 32; + int32_t e = (int32_t)((u >> 23) & 0xff); + u32 m = u & 0x7fffffu; + if (e == 0xff) { + return sign | GPUCA_B64_INF | ((u64)m << 29) | (m ? GPUCA_B64_QUIET : 0ULL); + } + if (e == 0) { + if (m == 0) { + return sign; + } + const int32_t lz = clz32(m) - 8; + m <<= lz; + e = 1 - lz; + } + return sign | ((u64)(e - 127 + 1023) << 52) | ((u64)(m & 0x7fffffu) << 29); +} + +// binary64 -> binary32, round to nearest even, subnormals, inf, NaN (payload kept, quietened) +GPUCA_B64_OP float toFloat(u64 d) +{ + const u32 sign = (u32)(d >> 32) & 0x80000000u; + const int32_t be = (int32_t)((d >> 52) & 0x7ff); + const u32 man = (u32)((d & GPUCA_B64_FRAC) >> 29); + const u32 drop = (u32)d & 0x1fffffffu; + u32 bits; + if (be == 0x7ff) { + bits = sign | 0x7f800000u | man | ((d & GPUCA_B64_FRAC) ? 0x400000u : 0u); + } else if (be == 0) { + bits = sign; + } else { + const int32_t e = be - 1023 + 127; + if (e >= 0xff) { + bits = sign | 0x7f800000u; + } else if (e > 0) { + bits = sign | ((u32)e << 23) | man; + if ((drop & 0x10000000u) && ((drop & 0x0fffffffu) || (man & 1u))) { + bits += 1u; + } + } else if (e > -24) { + const u32 full = man | 0x800000u; + const u32 sh = (u32)(1 - e); + const u32 lost = full & ((1u << sh) - 1u); + const u32 halfb = 1u << (sh - 1); + u32 sub = full >> sh; + if (lost > halfb || (lost == halfb && ((sub & 1u) || drop))) { + sub += 1u; + } + bits = sign | sub; + } else { + bits = sign; + } + } + return asf32(bits); +} + +} // namespace binary64_detail + +class GPUdoubleBinary64 +{ + public: + GPUdDefault() GPUdoubleBinary64() = default; + GPUdi() GPUdoubleBinary64(float v) : mBits(binary64_detail::fromFloat(v)) {} + GPUdi() operator float() const { return binary64_detail::toFloat(mBits); } + + GPUdi() static GPUdoubleBinary64 fromBits(binary64_detail::u64 b) + { + GPUdoubleBinary64 r; + r.mBits = b; + return r; + } + GPUdi() binary64_detail::u64 bits() const { return mBits; } + + GPUdi() GPUdoubleBinary64 operator-() const { return fromBits(mBits ^ GPUCA_B64_SIGN); } + GPUdi() GPUdoubleBinary64 operator+(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::addsub(mBits, b.mBits, false)); } + GPUdi() GPUdoubleBinary64 operator-(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::addsub(mBits, b.mBits, true)); } + GPUdi() GPUdoubleBinary64 operator*(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::mul(mBits, b.mBits)); } + GPUdi() GPUdoubleBinary64 operator/(GPUdoubleBinary64 b) const { return fromBits(binary64_detail::div(mBits, b.mBits)); } + + GPUdi() GPUdoubleBinary64 operator+(float b) const { return *this + GPUdoubleBinary64(b); } + GPUdi() GPUdoubleBinary64 operator-(float b) const { return *this - GPUdoubleBinary64(b); } + GPUdi() GPUdoubleBinary64 operator*(float b) const { return *this * GPUdoubleBinary64(b); } + GPUdi() GPUdoubleBinary64 operator/(float b) const { return *this / GPUdoubleBinary64(b); } +#ifndef __METAL__ + GPUdi() GPUdoubleBinary64 operator+(double b) const { return *this + (float)b; } + GPUdi() GPUdoubleBinary64 operator-(double b) const { return *this - (float)b; } + GPUdi() GPUdoubleBinary64 operator*(double b) const { return *this * (float)b; } + GPUdi() GPUdoubleBinary64 operator/(double b) const { return *this / (float)b; } +#endif + + GPUdi() GPUdoubleBinary64& operator+=(GPUdoubleBinary64 b) { return *this = *this + b; } + GPUdi() GPUdoubleBinary64& operator-=(GPUdoubleBinary64 b) { return *this = *this - b; } + GPUdi() GPUdoubleBinary64& operator*=(GPUdoubleBinary64 b) { return *this = *this * b; } + GPUdi() GPUdoubleBinary64& operator/=(GPUdoubleBinary64 b) { return *this = *this / b; } + + private: + binary64_detail::u64 mBits; +}; + +GPUdi() GPUdoubleBinary64 operator+(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) + b; } +GPUdi() GPUdoubleBinary64 operator-(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) - b; } +GPUdi() GPUdoubleBinary64 operator*(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) * b; } +GPUdi() GPUdoubleBinary64 operator/(float a, GPUdoubleBinary64 b) { return GPUdoubleBinary64(a) / b; } +#ifndef __METAL__ +GPUdi() GPUdoubleBinary64 operator+(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) + b; } +GPUdi() GPUdoubleBinary64 operator-(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) - b; } +GPUdi() GPUdoubleBinary64 operator*(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) * b; } +GPUdi() GPUdoubleBinary64 operator/(double a, GPUdoubleBinary64 b) { return GPUdoubleBinary64((float)a) / b; } +#endif + +// rounds once, as `someFloat += someDouble` does on the host +#ifdef __METAL__ +GPUdi() thread float& operator+=(thread float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +GPUdi() device float& operator+=(device float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +GPUdi() threadgroup float& operator+=(threadgroup float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +#else +GPUdi() float& operator+=(float& a, GPUdoubleBinary64 b) { return a = (float)(GPUdoubleBinary64(a) + b); } +#endif + +} // namespace o2::gpu + +#endif // GPUCOMMONDOUBLEBINARY64_H diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 7a78a5881dcfa..5d8dd6e99bbe8 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -87,7 +87,9 @@ class GPUCommonMath GPUd() constexpr static float Sin(float x); GPUd() constexpr static float Cos(float x); GPUhdni() static void SinCos(float x, float& s, float& c); +#ifndef __METAL__ // MSL has no double; every other backend keeps the twin GPUhdni() static void SinCosd(double x, double& s, double& c); +#endif GPUd() constexpr static float Tan(float x); GPUd() constexpr static float Pow(float x, float y); GPUd() constexpr static float Log(float x); @@ -105,7 +107,11 @@ class GPUCommonMath GPUd() constexpr static bool Finite(float x); GPUd() constexpr static bool IsNaN(float x); #ifndef __FAST_MATH__ +#ifdef __METAL__ // MSL has no nan(uint) + GPUd() constexpr static float QuietNaN() { return __builtin_nanf(""); } +#else GPUd() constexpr static float QuietNaN() { return GPUCA_CHOICE(std::numeric_limits::quiet_NaN(), __builtin_nanf(""), nan(0u)); } +#endif #endif GPUd() constexpr static uint32_t Clz(uint32_t val); GPUd() constexpr static uint32_t Ctz(uint32_t val); @@ -245,7 +251,11 @@ GPUdi() float2 GPUCommonMath::MakeFloat2(float x, float y) } GPUdi() constexpr float GPUCommonMath::Modf(float x, float y) { return GPUCA_CHOICE(fmodf(x, y), fmodf(x, y), fmod(x, y)); } +#ifdef __METAL__ // MSL has no remainder(); this is its definition +GPUhdi() float GPUCommonMath::Remainderf(float x, float y) { return x - y * rint(x / y); } +#else GPUhdi() float GPUCommonMath::Remainderf(float x, float y) { return GPUCA_CHOICE(std::remainderf(x, y), remainderf(x, y), remainder(x, y)); } +#endif GPUdi() uint32_t GPUCommonMath::Float2UIntReint(const float& x) { @@ -302,12 +312,20 @@ GPUhdi() void GPUCommonMath::SinCos(float x, float& s, float& c) __sincosf(x, &s, &c); #elif !defined(GPUCA_GPUCODE_DEVICE) && (defined(__GNU_SOURCE__) || defined(_GNU_SOURCE) || defined(GPUCA_GPUCODE)) sincosf(x, &s, &c); +#else +#ifdef __METAL__ // MSL's sincos returns sin and takes cos by thread reference, + // so it cannot write straight through a generic one + float metalCos; + s = sincos(x, metalCos); + c = metalCos; #else GPUCA_CHOICE((void)((s = sinf(x)) + (c = cosf(x))), sincosf(x, &s, &c), s = sincos(x, &c)); +#endif #endif ) // clang-format on } +#ifndef __METAL__ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c) { #if !defined(GPUCA_GPUCODE_DEVICE) && defined(__APPLE__) @@ -318,6 +336,7 @@ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c) GPUCA_CHOICE((void)((s = sin(x)) + (c = cos(x))), sincos(x, &s, &c), s = sincos(x, &c)); #endif } +#endif GPUdi() constexpr uint32_t GPUCommonMath::Clz(uint32_t x) { @@ -444,11 +463,13 @@ GPUhdi() constexpr float GPUCommonMath::Abs(float x) return GPUCA_CHOICE(fabsf(x), fabsf(x), fabs(x)); } +#ifndef __METAL__ template <> GPUhdi() constexpr double GPUCommonMath::Abs(double x) { return GPUCA_CHOICE(fabs(x), fabs(x), fabs(x)); } +#endif template <> GPUhdi() constexpr int32_t GPUCommonMath::Abs(int32_t x) @@ -456,6 +477,17 @@ GPUhdi() constexpr int32_t GPUCommonMath::Abs(int32_t x) return GPUCA_CHOICE(abs(x), abs(x), abs(x)); } +#ifdef __METAL__ +// The counters these operate on are plain integers in the transferred structures, +// as they are for CUDA and HIP. MSL's atomic operations want metal::atomic, which +// has the same size and alignment; the overloads keep the address space, which a +// generic pointer would not carry into atomic_*_explicit. +template +GPUdi() threadgroup metal::atomic* GPUCommonMathMetalAtomic(threadgroup T* p) { return reinterpret_cast*>(p); } +template +GPUdi() device metal::atomic* GPUCommonMathMetalAtomic(T* p) { return (device metal::atomic*)p; } +#endif + template GPUdi() uint32_t GPUCommonMath::AtomicExchInternal(S* addr, T val) { @@ -466,7 +498,7 @@ GPUdi() uint32_t GPUCommonMath::AtomicExchInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) return ::atomicExch(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - return atomic_exchange_explicit(addr, val, memory_order_relaxed); + return atomic_exchange_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #elif defined(WITH_OPENMP) uint32_t old; __atomic_exchange(addr, &val, &old, __ATOMIC_SEQ_CST); @@ -486,7 +518,7 @@ GPUdi() bool GPUCommonMath::AtomicCASInternal(S* addr, T cmp, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) return ::atomicCAS(addr, cmp, val) == cmp; #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - return atomic_compare_exchange_weak_explicit(addr, &cmp, val, memory_order_relaxed, memory_order_relaxed); + return atomic_compare_exchange_weak_explicit(GPUCommonMathMetalAtomic(addr), &cmp, val, memory_order_relaxed, memory_order_relaxed); #elif defined(WITH_OPENMP) return __atomic_compare_exchange(addr, &cmp, &val, true, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); #else @@ -504,7 +536,7 @@ GPUdi() uint32_t GPUCommonMath::AtomicAddInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) return ::atomicAdd(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - return atomic_fetch_add_explicit(addr, val, memory_order_relaxed); + return atomic_fetch_add_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #elif defined(WITH_OPENMP) return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST) - val; #else @@ -522,7 +554,7 @@ GPUdi() void GPUCommonMath::AtomicMaxInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) ::atomicMax(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - atomic_fetch_max_explicit(addr, val, memory_order_relaxed); + atomic_fetch_max_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #else S current; while ((current = *(volatile S*)addr) < val && !AtomicCASInternal(addr, current, val)) { @@ -540,7 +572,7 @@ GPUdi() void GPUCommonMath::AtomicMinInternal(S* addr, T val) #elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) ::atomicMin(addr, val); #elif defined(GPUCA_GPUCODE) && defined(__METAL__) - atomic_fetch_min_explicit(addr, val, memory_order_relaxed); + atomic_fetch_min_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); #else S current; while ((current = *(volatile S*)addr) > val && !AtomicCASInternal(addr, current, val)) { diff --git a/GPU/Common/GPUCommonTypeTraits.h b/GPU/Common/GPUCommonTypeTraits.h index 3f83e151b0f33..6d72f164771bc 100644 --- a/GPU/Common/GPUCommonTypeTraits.h +++ b/GPU/Common/GPUCommonTypeTraits.h @@ -114,7 +114,13 @@ struct is_pointer_t { }; template struct is_pointer { +#ifdef __METAL__ + // A bare T* partial specialization does not match a pointer type deduced from + // an argument, which carries its address space; metal::is_pointer does. + enum { value = metal::is_pointer::value }; +#else enum { value = is_pointer_t::type>::value }; +#endif }; template diff --git a/GPU/Common/test/testGPUsortCUDA.cu b/GPU/Common/test/testGPUsortCUDA.cu index b19235f9e8c6b..95464dcc96c0d 100644 --- a/GPU/Common/test/testGPUsortCUDA.cu +++ b/GPU/Common/test/testGPUsortCUDA.cu @@ -96,12 +96,12 @@ __global__ void sortInThreadWithOperator(float* data, size_t dataLength) __global__ void sortInBlock(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength); } __global__ void sortInBlockWithOperator(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength, [](float a, float b) { return a < b; }); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength, [](float a, float b) { return a < b; }); } /////////////////////////////////////////////////////////////// diff --git a/GPU/GPUTracking/Base/GPUGeneralKernels.cxx b/GPU/GPUTracking/Base/GPUGeneralKernels.cxx index e1a3ce69dd8df..d27b778701ea1 100644 --- a/GPU/GPUTracking/Base/GPUGeneralKernels.cxx +++ b/GPU/GPUTracking/Base/GPUGeneralKernels.cxx @@ -19,12 +19,12 @@ using namespace o2::gpu; template <> GPUdii() void GPUMemClean16::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() processors, GPUglobalref() void* ptr, uint64_t size) { - const uint64_t stride = get_global_size(0); + const uint64_t stride = (nBlocks * nThreads); int4 i0; i0.x = i0.y = i0.z = i0.w = 0; int4* ptra = (int4*)ptr; uint64_t len = (size + sizeof(int4) - 1) / sizeof(int4); - for (uint64_t i = get_global_id(0); i < len; i += stride) { + for (uint64_t i = (iBlock * nThreads + iThread); i < len; i += stride) { ptra[i] = i0; } } @@ -32,8 +32,8 @@ GPUdii() void GPUMemClean16::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_ template <> GPUdii() void GPUitoa::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() processors, GPUglobalref() int32_t* ptr, uint64_t size) { - const uint64_t stride = get_global_size(0); - for (uint64_t i = get_global_id(0); i < size; i += stride) { + const uint64_t stride = (nBlocks * nThreads); + for (uint64_t i = (iBlock * nThreads + iThread); i < size; i += stride) { ptr[i] = i; } } diff --git a/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h b/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h index cc1c62bed507d..0887cadd7338d 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h +++ b/GPU/GPUTracking/Base/GPUReconstructionKernelMacros.h @@ -63,8 +63,17 @@ #define GPUCA_ATTRRES(...) GPUCA_M_EXPAND(GPUCA_M_CAT(GPUCA_ATTRRES_, GPUCA_M_FIRST(__VA_ARGS__)))(__VA_ARGS__) // GPU Kernel entry point +// MSL requires every kernel parameter to carry an attribute, and supplies the +// grid dimensions the same way, so the backend gets to shape both ends of the +// parameter list. +#ifndef GPUCA_KRNL_SECTOR_ARG +#define GPUCA_KRNL_SECTOR_ARG int32_t _iSector_internal +#endif +#ifndef GPUCA_KRNL_GRID_ARGS +#define GPUCA_KRNL_GRID_ARGS +#endif #define GPUCA_KRNLGPU_DEF(x_class, x_attributes, x_arguments, ...) \ - GPUg() void GPUCA_ATTRRES(GPUCA_M_STRIP(x_attributes)) GPUCA_M_CAT(krnl_, GPUCA_M_KRNL_NAME(x_class))(GPUCA_CONSMEM_PTR int32_t _iSector_internal GPUCA_M_STRIP(x_arguments)) + GPUg() void GPUCA_ATTRRES(GPUCA_M_STRIP(x_attributes)) GPUCA_M_CAT(krnl_, GPUCA_M_KRNL_NAME(x_class))(GPUCA_CONSMEM_PTR GPUCA_KRNL_SECTOR_ARG GPUCA_M_STRIP(x_arguments) GPUCA_KRNL_GRID_ARGS) #ifdef GPUCA_KRNL_DEFONLY #define GPUCA_KRNLGPU(...) GPUCA_KRNLGPU_DEF(__VA_ARGS__); diff --git a/GPU/GPUTracking/Base/GPUReconstructionThreading.h b/GPU/GPUTracking/Base/GPUReconstructionThreading.h index 374c7545e65da..f03ac3c06102a 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionThreading.h +++ b/GPU/GPUTracking/Base/GPUReconstructionThreading.h @@ -35,25 +35,25 @@ struct GPUReconstructionThreading { #endif -#define GPUCA_TBB_KERNEL_LOOP_HOST(rec, vartype, varname, iEnd, code) \ - for (vartype varname = get_global_id(0); varname < iEnd; varname += get_global_size(0)) { \ - code \ +#define GPUCA_TBB_KERNEL_LOOP_HOST(rec, nBlocks, nThreads, iBlock, iThread, vartype, varname, iEnd, code) \ + for (vartype varname = (iBlock) * (nThreads) + (iThread); varname < iEnd; varname += (nBlocks) * (nThreads)) { \ + code \ } #ifdef GPUCA_GPUCODE #define GPUCA_TBB_KERNEL_LOOP GPUCA_TBB_KERNEL_LOOP_HOST #else -#define GPUCA_TBB_KERNEL_LOOP(rec, vartype, varname, iEnd, code) \ - if (!rec.GetProcessingSettings().inKernelParallel) { \ - rec.mThreading->activeThreads->execute([&] { \ - tbb::parallel_for(tbb::blocked_range(get_global_id(0), iEnd, get_global_size(0)), [&](const tbb::blocked_range& _r_internal) { \ - for (vartype varname = _r_internal.begin(); varname < _r_internal.end(); varname += get_global_size(0)) { \ - code \ - } \ - }); \ - }); \ - } else { \ - GPUCA_TBB_KERNEL_LOOP_HOST(rec, vartype, varname, iEnd, code) \ +#define GPUCA_TBB_KERNEL_LOOP(rec, nBlocks, nThreads, iBlock, iThread, vartype, varname, iEnd, code) \ + if (!rec.GetProcessingSettings().inKernelParallel) { \ + rec.mThreading->activeThreads->execute([&] { \ + tbb::parallel_for(tbb::blocked_range((iBlock) * (nThreads) + (iThread), iEnd, (nBlocks) * (nThreads)), [&](const tbb::blocked_range& _r_internal) { \ + for (vartype varname = _r_internal.begin(); varname < _r_internal.end(); varname += (nBlocks) * (nThreads)) { \ + code \ + } \ + }); \ + }); \ + } else { \ + GPUCA_TBB_KERNEL_LOOP_HOST(rec, nBlocks, nThreads, iBlock, iThread, vartype, varname, iEnd, code) \ } #endif diff --git a/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip b/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip index ed13124ef65df..5758faaebebda 100644 --- a/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip +++ b/GPU/GPUTracking/Base/hip/test/testGPUsortHIP.hip @@ -104,12 +104,12 @@ __global__ void sortInThreadWithOperator(float* data, size_t dataLength) __global__ void sortInBlock(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength); } __global__ void sortInBlockWithOperator(float* data, size_t dataLength) { - o2::gpu::CAAlgo::sortInBlock(data, data + dataLength, [](float a, float b) { return a < b; }); + o2::gpu::CAAlgo::sortInBlock(blockDim.x, threadIdx.x, data, data + dataLength, [](float a, float b) { return a < b; }); } /////////////////////////////////////////////////////////////// diff --git a/GPU/GPUTracking/Base/metal/CMakeLists.txt b/GPU/GPUTracking/Base/metal/CMakeLists.txt new file mode 100644 index 0000000000000..9c9b7e4fd2b15 --- /dev/null +++ b/GPU/GPUTracking/Base/metal/CMakeLists.txt @@ -0,0 +1,110 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +set(MODULE GPUTrackingMETAL) +enable_language(ASM) + +message(STATUS "Building GPUTracking with Metal support") + +# convenience variables +if(ALIGPU_BUILD_TYPE STREQUAL "Standalone") + set(GPUDIR ${CMAKE_SOURCE_DIR}/../) +else() + set(GPUDIR ${CMAKE_SOURCE_DIR}/GPU/GPUTracking) +endif() +set(METAL_SRC ${GPUDIR}/Base/metal/GPUReconstructionMETAL.metal) +set(METAL_BIN ${CMAKE_CURRENT_BINARY_DIR}/GPUReconstructionMetalCode) + +# MSL 4.1 is the first version with a generic address space: earlier versions +# reject an unannotated pointer or `this` outright, which GPUCommonDefAPI.h +# relies on for GPUgeneric() and GPUdDefault(). +set(METAL_FLAGS -std=metal4.1 ${GPUCA_METAL_DENORMALS_FLAGS}) +if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_NO_FAST_MATH}) + set(METAL_FLAGS ${METAL_FLAGS} ${GPUCA_METAL_NO_FAST_MATH_FLAGS}) +endif() +set(METAL_DEFINES "-D$,$-D>" + "-I$,EXCLUDE,^/usr/include/?>,$-I>" + -I${CMAKE_SOURCE_DIR}/Detectors/TRD/base/src + -I${CMAKE_SOURCE_DIR}/Detectors/Base/src + -I${CMAKE_SOURCE_DIR}/DataFormats/Reconstruction/src +) + +set(SRCS GPUReconstructionMetal.mm GPUReconstructionMetalKernels.mm) +set(HDRS GPUReconstructionMetal.h GPUReconstructionMetalIncludesHost.h) + +if(ALIGPU_BUILD_TYPE STREQUAL "O2") + o2_add_library(${MODULE} + SOURCES ${SRCS} + PUBLIC_LINK_LIBRARIES O2::GPUTracking + TARGETVARNAME targetName) + + target_link_libraries(${targetName} PUBLIC ${METAL_FRAMEWORKS}) + + target_compile_definitions(${targetName} PRIVATE $) + # the compile_defitions are not propagated automatically on purpose (they are + # declared PRIVATE) so we are not leaking them outside of the GPU** + # directories +endif() + +if(ALIGPU_BUILD_TYPE STREQUAL "Standalone") + add_library(${MODULE} SHARED ${SRCS}) + target_link_libraries(${MODULE} GPUTracking) + install(TARGETS ${MODULE}) + set(targetName ${MODULE}) +endif() + +if(METAL_ENABLED) # BUILD Metal source code for runtime compilation target + + # executes clang to preprocess + add_custom_command( + OUTPUT ${METAL_BIN}.metal + COMMAND xcrun -sdk macosx metal + -Wno-unused-command-line-argument + ${METAL_FLAGS} + ${METAL_DEFINES} + -MD -MT ${METAL_BIN}.src -MF ${METAL_BIN}.src.d + -E -P ${METAL_SRC} > ${METAL_BIN}.metal + DEPENDS ${METAL_SRC} + DEPFILE ${METAL_BIN}.src.d + COMMAND_EXPAND_LISTS + COMMENT "Preparing Metal source file for run time compilation ${METAL_BIN}.metal") + + # Create the ir + add_custom_command( + OUTPUT ${METAL_BIN}.ir + COMMAND xcrun -sdk macosx metal + -Wno-unused-command-line-argument + -Wno-c++17-extensions + -ferror-limit=10000 + ${METAL_FLAGS} + ${METAL_DEFINES} + ${METAL_BIN}.metal + -o ${METAL_BIN}.ir + DEPENDS ${METAL_BIN}.metal + COMMAND_EXPAND_LISTS + COMMENT "Preparing Metal intermediate representation for run time compilation ${METAL_BIN}.ir") + + add_custom_target(metal_preprocessed_code ALL DEPENDS ${METAL_BIN}.metal COMMENT "Needed to inject dependency on its creation") + add_custom_target(metal_intermediate_representation ALL DEPENDS ${METAL_BIN}.ir COMMENT "Needed to inject dependency on its creation") + + # Pack the compiled library into __DATA,__gpu_resource during final link. This + # way we do not need to create an intermediate object. Compiling the source at + # run time is not an option: the driver's compiler service dies on it. + target_link_options(${targetName} + PRIVATE + "-Wl,-sectcreate,__DATA,__gpu_resource,${METAL_BIN}.ir") + add_dependencies(${targetName} metal_preprocessed_code) + add_dependencies(${targetName} metal_intermediate_representation) +endif() + +install(FILES ${HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/GPU) + +target_compile_definitions(${targetName} PRIVATE GPUCA_METAL_BUILD_FLAGS=$ ) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal new file mode 100644 index 0000000000000..004220e7e268d --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMETAL.metal @@ -0,0 +1,78 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUReconstructionMETAL.metal + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" +// clang-format off + +// --- Backend selection ------------------------------------------------------- +#define GPUCA_GPUTYPE_METAL 1 + +// --- Metal stdlib ------------------------------------------------------------ +#include +// MSL rejects derived classes outside of this pragma, and the kernels are +// class-based throughout. metal_stdlib itself uses it in 46 paired places. +#pragma METAL internals : enable +using namespace metal; + +// --- OpenCL compatibility shims --------------------------------------------- + +// Address space aliases (match OpenCL vernacular used by the project); constant +// is spelled the same in MSL +#define global device +#define local threadgroup + +#ifndef M_PI +#define M_PI 3.1415926535f +#endif + +// Disable assertions inside GPU code (same as OpenCL variant) +#ifdef assert +# undef assert +#endif +#define assert(param) + +// --- Project headers --------------------------------------------------------- +#include "GPUCommonDef.h" +#include "GPUCommonTypeTraits.h" +#include "GPUCommonArray.h" + +#include "GPUConstantMem.h" +#include "GPUReconstructionIncludesDeviceAll.h" + +// --- Kernel list expansion --------------------------------------------------- +#define GPUCA_KRNL(...) GPUCA_KRNLGPU(__VA_ARGS__) + +// --- Constant memory + global heap plumbing --------------------------------- +// The heap and the constant memory arrive as buffer(0) and buffer(1). The latter +// is untyped because a buffer of GPUConstantMem, which has base classes, is not +// a valid kernel argument type. +#define GPUCA_CONSMEM_PTR \ + device char* gpu_mem [[buffer(0)]], \ + device char* pConstantRaw [[buffer(1)]], +#define GPUCA_CONSMEM (*(device GPUConstantMem*)pConstantRaw) + +// Every kernel parameter needs an attribute, so the sector index arrives as a +// buffer rather than by value, and the grid dimensions come in at the end, where +// GPUCommonDefAPI.h's get_group_id() and friends pick them up. +#define GPUCA_KRNL_SECTOR_ARG constant int32_t& _iSector_internal [[buffer(2)]] +#define GPUCA_KRNL_GRID_ARGS \ + , uint _metalTgIg [[threadgroup_position_in_grid]] \ + , uint _metalTiTg [[thread_position_in_threadgroup]] \ + , uint _metalTPerTg [[threads_per_threadgroup]] \ + , uint _metalTgPerG [[threadgroups_per_grid]] + +#include "GPUReconstructionKernelList.h" + +// clang-format on +#pragma clang diagnostic pop diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h new file mode 100644 index 0000000000000..66cf5ab7bf121 --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h @@ -0,0 +1,67 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef GPURECONSTRUCTIONMETAL_H +#define GPURECONSTRUCTIONMETAL_H + +#include "GPUReconstructionDeviceBase.h" + +extern "C" o2::gpu::GPUReconstruction* GPUReconstruction_Create_METAL(const o2::gpu::GPUSettingsDeviceBackend& cfg); + +namespace o2::gpu +{ +struct GPUReconstructionMetalInternals; + +class GPUReconstructionMetal : public GPUReconstructionProcessing::KernelInterface +{ + public: + GPUReconstructionMetal(const GPUSettingsDeviceBackend& cfg); + ~GPUReconstructionMetal() override; + + template + void runKernelBackend(const krnlSetupTime& _xyz, const Args&... args); + + protected: + int32_t InitDevice_Runtime() override; + int32_t ExitDevice_Runtime() override; + + virtual int32_t GPUChkErrInternal(const int64_t error, const char* file, int32_t line) const override; + + void SynchronizeGPU() override; + int32_t GPUDebug(const char* state = "UNKNOWN", int32_t stream = -1, bool force = false) override; + void SynchronizeStream(int32_t stream) override; + void SynchronizeEvents(deviceEvent* evList, int32_t nEvents = 1) override; + void StreamWaitForEvents(int32_t stream, deviceEvent* evList, int32_t nEvents = 1) override; + bool IsEventDone(deviceEvent* evList, int32_t nEvents = 1) override; + + size_t WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream = -1, deviceEvent* ev = nullptr) override; + size_t GPUMemCpy(void* dst, const void* src, size_t size, int32_t stream, int32_t toGPU, deviceEvent* ev = nullptr, deviceEvent* evList = nullptr, int32_t nEvents = 1) override; + void ReleaseEvent(deviceEvent ev) override; + void RecordMarker(deviceEvent* ev, int32_t stream) override; + + template + int32_t AddKernel(); + + GPUReconstructionMetalInternals* mInternals; + float mOclVersion; + + template + S& getKernelObject(); + + int32_t GetMetalPrograms(); + + private: + int32_t AddKernels(); +}; + +} // namespace o2::gpu + +#endif // GPURECONSTRUCTIONMETAL_H diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm new file mode 100644 index 0000000000000..8448a9d86184b --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -0,0 +1,414 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "GPUReconstructionMetal.h" +#include "GPUConstantMem.h" +#include "GPUDefParametersLoad.inc" +#include "GPUReconstructionMetalIncludesHost.h" + +#include +#include + +#include +#include +#include // _mh_execute_header + +#define GPUErrorReturn(...) \ + { \ + GPUError(__VA_ARGS__); \ + return (1); \ + } + +#include "utils/qGetLdBinarySymbols.h" +QGET_LD_BINARY_SYMBOLS(GPUReconstructionMetalCode_src); + +GPUReconstruction* GPUReconstruction_Create_METAL(const GPUSettingsDeviceBackend& cfg) { return new GPUReconstructionMetal(cfg); } + +GPUReconstructionMetal::GPUReconstructionMetal(const GPUSettingsDeviceBackend& cfg) : GPUReconstructionProcessing::KernelInterface(cfg, sizeof(GPUReconstructionDeviceBase)) +{ + if (mMaster == nullptr) { + mInternals = new GPUReconstructionMetalInternals; + *mParDevice = o2::gpu::internal::GPUDefParametersLoad(); + } + mDeviceBackendSettings->deviceType = DeviceType::METAL; +} + +GPUReconstructionMetal::~GPUReconstructionMetal() +{ + Exit(); // Make sure we destroy everything (in particular the ITS tracker) before we exit + if (mMaster == nullptr) { + delete mInternals; + } +} + +int32_t GPUReconstructionMetal::InitDevice_Runtime() +{ + // Propagate processing settings to PoCL runtime. + // Won't affect other OpenCL runtimes. + if (int nThreads = mProcessingSettings->nHostThreads; nThreads > 0) { + auto nThreadsStr = std::to_string(nThreads); + setenv("PMETAL_CPU_MAX_CU_COUNT", nThreadsStr.c_str(), 1); + } + + if (mMaster == nullptr) { + mInternals->device = MTLCreateSystemDefaultDevice(); + + int64_t deviceGlobalMem, deviceLocalMem; + MTLSize deviceMaxWorkGroup = mInternals->device.maxThreadsPerThreadgroup; + + std::string device_name = [mInternals->device.name UTF8String]; + // On Apple Silicon, treat recommended working set as an upper bound + deviceGlobalMem = mInternals->device.recommendedMaxWorkingSetSize; + deviceLocalMem = mInternals->device.maxThreadgroupMemoryLength; + if (GetProcessingSettings().debugLevel >= 2) { + GPUInfo("Using Metal device %s with properties:", device_name.c_str()); + GPUInfo("\tUnified Memory Architecture = %ld ", mInternals->device.hasUnifiedMemory); + GPUInfo("\tRecommended Max Working Set = %ld bytes", deviceGlobalMem); + GPUInfo("\tMax thread group memory = %ld bytes", deviceLocalMem); + GPUInfo("\tmaxWorkGroup = (%ld, %ld, %ld)", deviceMaxWorkGroup.width, deviceMaxWorkGroup.height, deviceMaxWorkGroup.depth); + GPUInfo(" "); + } + + mDeviceName = device_name.c_str(); + // Basically a random number for now. + mMaxBackendThreads = 1000; + + if (GetMetalPrograms()) { + return 1; + } + + if (GetProcessingSettings().debugLevel >= 2) { + GPUInfo("Metal program and kernels loaded successfully"); + } + + // We only support internal GPUs, so the ownership of the memory is always shared + mInternals->mem_gpu = [mInternals->device newBufferWithLength:mDeviceMemorySize options:MTLResourceStorageModeShared]; + if (mInternals->mem_gpu == nil) { + GPUErrorReturn("Metal Memory Allocation Error"); + } + + // We only support internal GPUs, so the ownership of the memory is always shared. + // FIXME: until I understand how to enable the gGPUConstantMemBufferSize constexpr + int32_t tmpGPUContantMemBufferSize = 100000; // gGPUConstantMemBufferSize + mInternals->mem_constant = [mInternals->device newBufferWithLength:tmpGPUContantMemBufferSize options:MTLResourceStorageModeShared]; + if (mInternals->mem_constant) { + GPUErrorReturn("Metal Constant Memory Allocation Error"); + } + + for (int32_t i = 0; i < mNStreams; i++) { + mInternals->commandQueues[i] = [mInternals->device newCommandQueue]; + if (mInternals->commandQueues[i] == nil) { + GPUErrorReturn("Error creating Metal command queue"); + } + mInternals->commandBuffers[i] = [mInternals->commandQueues[i] commandBuffer]; + if (mInternals->commandBuffers[i] == nil) { + GPUErrorReturn("Error creating Metal command buffer"); + } + } + + mInternals->mem_host = [mInternals->device newBufferWithLength:mHostMemorySize options:MTLResourceStorageModeShared]; + if (mInternals->mem_host == nil) { + GPUErrorReturn("Error allocating pinned host memory"); + } + + mHostMemoryBase = mInternals->mem_host.contents; + mHostMemorySize = mInternals->mem_host.allocatedSize; + mDeviceMemoryBase = mInternals->mem_gpu.contents; + mDeviceMemorySize = mInternals->mem_gpu.allocatedSize; + mDeviceConstantMem = (GPUConstantMem*)mInternals->mem_constant.contents; + + if (GetProcessingSettings().debugLevel >= 1) { + GPUInfo("Memory ptrs: GPU (%ld bytes): %p - Host (%ld bytes): %p", (int64_t)mDeviceMemorySize, mDeviceMemoryBase, (int64_t)mHostMemorySize, mHostMemoryBase); + memset(mHostMemoryBase, 0xDD, mHostMemorySize); + } + + GPUInfo("Metal Initialisation successfull"); + } else { + auto* master = dynamic_cast(mMaster); + mWarpSize = master->mWarpSize; + mMaxBackendThreads = master->mMaxBackendThreads; + mDeviceName = master->mDeviceName; + mDeviceConstantMem = master->mDeviceConstantMem; + mInternals = master->mInternals; + } + + for (uint32_t i = 0; i < mEvents.size(); i++) { + auto* events = (id*)mEvents[i].data(); + new (events) id[ mEvents[i].size() ]; + } + + return (0); +} + +int32_t GPUReconstructionMetal::ExitDevice_Runtime() +{ + // Uninitialize OPENCL + SynchronizeGPU(); + + if (mMaster == nullptr) { + if (mDeviceMemoryBase) { + [mInternals->mem_gpu release]; + [mInternals->mem_constant release]; + for (uint32_t i = 0; i < mInternals->functions.size(); i++) { + [mInternals->functions[i] release]; + } + mInternals->functions.clear(); + } + if (mHostMemoryBase) { + for (int32_t i = 0; i < mNStreams; i++) { + [mInternals->commandQueues[i] release]; + [mInternals->commandBuffers[i] release]; + } + [mInternals->mem_host release]; + } + + [mInternals->library release]; + [mInternals->device release]; + GPUInfo("Metal disposed correctly"); + } + mDeviceMemoryBase = nullptr; + mHostMemoryBase = nullptr; + + return (0); +} + +size_t GPUReconstructionMetal::GPUMemCpy(void* dst, const void* src, size_t sizeBytes, int32_t stream, int32_t toGPU, deviceEvent* ev, deviceEvent* evList, int32_t nEvents) +{ + if (evList == nullptr) { + nEvents = 0; + } + if (GetProcessingSettings().debugLevel >= 3) { + stream = -1; + } + + if (stream == -1) { + SynchronizeGPU(); + } + + auto realStream = stream == -1 ? 0 : stream; + id cb = mInternals->commandBuffers[realStream]; + id blit = [cb blitCommandEncoder]; + id sourceBuffer = nil; + id destBuffer = nil; + ptrdiff_t sourceOffset = 0; + ptrdiff_t destOffset = 0; + + // Sigh. + if (src > mHostMemoryBase && src < ((char*)mHostMemoryBase + mHostMemorySize)) { + sourceBuffer = mInternals->mem_host; + sourceOffset = (char*)src - (char*)mHostMemoryBase; + } else if (src > mDeviceMemoryBase && src < ((char*)mDeviceMemoryBase + mDeviceMemorySize)) { + sourceBuffer = mInternals->mem_gpu; + sourceOffset = (char*)src - (char*)mDeviceMemoryBase; + } else { + GPUErrorReturn("Unknown buffer at %x", src); + } + + if (dst > mHostMemoryBase && dst < ((char*)mHostMemoryBase + mHostMemorySize)) { + destBuffer = mInternals->mem_host; + destOffset = (char*)src - (char*)mHostMemoryBase; + } else if (dst > mDeviceMemoryBase && dst < ((char*)mDeviceMemoryBase + mDeviceMemorySize)) { + destBuffer = mInternals->mem_gpu; + destOffset = (char*)dst - (char*)mDeviceMemoryBase; + } else { + GPUErrorReturn("Unknown buffer at %x", src); + } + + [blit copyFromBuffer:sourceBuffer + sourceOffset:sourceOffset + toBuffer:destBuffer + destinationOffset:destOffset + size:sizeBytes]; + + [blit endEncoding]; + [cb commit]; + + if (GetProcessingSettings().serializeGPU & 2) { + GPUDebug(("GPUMemCpy " + std::to_string(toGPU)).c_str(), stream, true); + } + return sizeBytes; +} + +size_t GPUReconstructionMetal::WriteToConstantMemory(size_t offset, const void* src, size_t size, int32_t stream, deviceEvent* ev) +{ + if (stream == -1) { + SynchronizeGPU(); + } + + auto realStream = stream == -1 ? 0 : stream; + id cb = mInternals->commandBuffers[realStream]; + id blit = [cb blitCommandEncoder]; + id sourceBuffer = nil; + ptrdiff_t sourceOffset = 0; + if (src > mHostMemoryBase && src < ((char*)mHostMemoryBase + mHostMemorySize)) { + sourceBuffer = mInternals->mem_host; + sourceOffset = (char*)src - (char*)mHostMemoryBase; + } else if (src > mDeviceMemoryBase && src < ((char*)mDeviceMemoryBase + mDeviceMemorySize)) { + sourceBuffer = mInternals->mem_gpu; + sourceOffset = (char*)src - (char*)mDeviceMemoryBase; + } else { + GPUErrorReturn("Unknown buffer at %x", src); + } + [blit copyFromBuffer:sourceBuffer + sourceOffset:sourceOffset + toBuffer:mInternals->mem_constant + destinationOffset:offset + size:size]; + + [blit endEncoding]; + [cb commit]; + + if (GetProcessingSettings().serializeGPU & 2) { + GPUDebug("WriteToConstantMemory", stream, true); + } + return size; +} + +void GPUReconstructionMetal::ReleaseEvent(deviceEvent ev) +{ + // FIXME: is this supposed to reset the event for it to be repurposed + // or to decrease the ref count? + auto mtlEvent = (__bridge id)(ev.get()); + [mtlEvent setSignaledValue:0]; +} + +void GPUReconstructionMetal::RecordMarker(deviceEvent* ev, int32_t stream) +{ + id cb = mInternals->commandBuffers[stream]; + // Does not change the retain count, so it's important we manage + // the lifetime of the events outside here. + auto mtlEvent = (__bridge id)(ev->get()); + [cb encodeSignalEvent:mtlEvent value:1]; + [cb commit]; +} + +void GPUReconstructionMetal::SynchronizeGPU() +{ + for (int32_t i = 0; i < mNStreams; i++) { + [mInternals->commandBuffers[i] waitUntilCompleted]; + } +} + +void GPUReconstructionMetal::SynchronizeStream(int32_t stream) +{ + [mInternals->commandBuffers[stream] waitUntilCompleted]; +} + +void GPUReconstructionMetal::SynchronizeEvents(deviceEvent* evList, int32_t nEvents) +{ + // I wait for everything to complete for now... + for (int32_t si = 0; si < mNStreams; si++) { + id cb = mInternals->commandBuffers[si]; + [cb waitUntilCompleted]; + } +} + +void GPUReconstructionMetal::StreamWaitForEvents(int32_t stream, deviceEvent* evList, int32_t nEvents) +{ + // Encode commands to wait for all the events + id cb = mInternals->commandBuffers[stream]; + for (int32_t ei = 0; ei < nEvents; ei++) { + auto mtlEvent = (__bridge id)(evList[ei].get()); + [cb encodeWaitForEvent:mtlEvent value:1]; + } + [cb commit]; + [cb waitUntilCompleted]; +} + +bool GPUReconstructionMetal::IsEventDone(deviceEvent* evList, int32_t nEvents) +{ + for (int32_t i = 0; i < nEvents; i++) { + auto mtlEvent = (__bridge id)(evList[i].get()); + if (mtlEvent.signaledValue == 0) { + return false; + } + } + return true; +} + +int32_t GPUReconstructionMetal::GPUDebug(const char* state, int32_t stream, bool force) +{ + // Wait for Metal-Kernel to finish and check for Metal errors afterwards, in case of debugmode + if (!force && GetProcessingSettings().debugLevel <= 0) { + return (0); + } + for (int32_t si = 0; si < mNStreams; si++) { + [mInternals->commandBuffers[si] waitUntilCompleted]; + } + if (GetProcessingSettings().debugLevel >= 3) { + GPUInfo("GPU Sync Done"); + } + return (0); +} + +int32_t GPUReconstructionMetal::GPUChkErrInternal(const int64_t error, const char* file, int32_t line) const +{ + // Not sure how metal returns errors. + if (error != 0) { + GPUError("Metal Error: %ld / %s (%s:%d)", error, "Unknown", file, line); + } + return error != 0; +} + +// Return pointer+size for (__DATA|__DATA_CONST, "__gpu_resource") from the image +// that matches `image_name_substr` (e.g. "libO2GPUReconstruction.dylib"). +static const uint8_t* find_gpu_resource_in_image(const char* image_name_substr, + unsigned long* out_size) +{ + uint32_t count = _dyld_image_count(); + for (uint32_t i = 0; i < count; ++i) { + const char* name = _dyld_get_image_name(i); + if (!name || !strstr(name, image_name_substr)) { + continue; + } + + const struct mach_header* mh = _dyld_get_image_header(i); + + const auto* mh64 = (const struct mach_header_64*)mh; + const auto* p = (const uint8_t*) + getsectiondata(mh64, "__DATA", "__gpu_resource", out_size); + if (!p) { + p = (const uint8_t*) + getsectiondata(mh64, "__DATA_CONST", "__gpu_resource", out_size); + } + if (p) { + return p; + } + } + return nullptr; +} + +int32_t GPUReconstructionMetal::GetMetalPrograms() +{ + GPUInfo("Loading Metal library (Platform version %s)", [mInternals->device.architecture.name cStringUsingEncoding:NSUTF8StringEncoding]); + + unsigned long sz = 0; + const uint8_t* p = find_gpu_resource_in_image("libO2GPUTrackingMETAL.dylib", &sz); + if (p == nullptr || sz == 0) { + GPUError("Metal library not found in the __gpu_resource section"); + return 1; + } + + // the section is part of the mapped image, so it outlives the dispatch_data_t + // and does not have to be copied + dispatch_data_t blob = dispatch_data_create(p, sz, nullptr, ^{}); + + NSError* error = nil; + mInternals->library = [mInternals->device newLibraryWithData:blob error:&error]; + + if (mInternals->library == nil) { + NSLog(@"%@", error); + GPUError("Error loading the Metal library"); + return 1; + } + + return AddKernels(); +} diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h new file mode 100644 index 0000000000000..e8c62623cef5a --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h @@ -0,0 +1,61 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef GPURECONSTRUCTIONOMETALINCLUDESHOST_H +#define GPURECONSTRUCTIONOMETALINCLUDESHOST_H + +#define GPUCA_GPUTYPE_METAL + +#import +#ifndef __METAL_VERSION__ +// __METAL_VERSION__ is only defined in device code. +#define __METAL_HOST__ +#endif + +#import + +#include +#include +#include +#include "GPULogging.h" + +#include "GPUReconstructionMetal.h" +#include "GPUReconstructionIncludes.h" +#include "GPUCommonHelpers.h" + +using namespace o2::gpu; + +#include +#include +#include +#include + +namespace o2::gpu +{ + +struct GPUReconstructionMetalInternals { + id device; + + std::array, constants::GPU_MAX_STREAMS> commandQueues; // ~ cl_command_queue[] + std::array, constants::GPU_MAX_STREAMS> commandBuffers; + + std::vector> functions; // ~ cl_kernel (symbols) + std::vector> pipelines; // compiled kernels + + id mem_gpu; // ~ cl_mem (device/global) + id mem_constant; // ~ cl_mem (constant-like) + id mem_host; // ~ cl_mem (host-visible) + + id library; // ~ cl_program +}; +} // namespace o2::gpu + +#endif // GPURECONSTRUCTIONOMETALINCLUDESHOST_H diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm new file mode 100644 index 0000000000000..54adb3076b60d --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm @@ -0,0 +1,82 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include "GPUReconstructionKernelIncludes.h" +#include "GPUReconstructionMetalIncludesHost.h" + +#include "GPUReconstructionMetalKernelsSpecialize.inc" +#include "GPUReconstructionProcessingKernels.inc" + +template void GPUReconstructionProcessing::KernelInterface::runKernelVirtual(const int num, const void* args); + +template +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, const Args&... args) +{ + id function = mInternals->functions[GetKernelNum()]; + auto& kExec = _xyz.x; + auto& runRange = _xyz.y; + // auto& events = _xyz.z; + // auto& t = _xyz.t; + + NSError* error = nil; + auto pso = [mInternals->device newComputePipelineStateWithFunction:function error:&error]; + id computeEncoder = [mInternals->commandBuffers[kExec.stream] computeCommandEncoder]; + + // Map buffers and states + [computeEncoder setComputePipelineState:pso]; + [computeEncoder setBuffer:mInternals->mem_gpu offset:0 atIndex:0]; + [computeEncoder setBuffer:mInternals->mem_constant offset:0 atIndex:1]; + [computeEncoder setBuffer:mInternals->mem_host offset:0 atIndex:2]; + + MTLSize gridSize = MTLSizeMake(runRange.index, 1, 1); + + NSUInteger threadGroupSize = pso.maxTotalThreadsPerThreadgroup; + if (threadGroupSize > runRange.index) { + threadGroupSize = runRange.index; + } + + MTLSize threadgroupSize = MTLSizeMake(threadGroupSize, 1, 1); + [computeEncoder dispatchThreads:gridSize + threadsPerThreadgroup:threadgroupSize]; +} + +template +int32_t GPUReconstructionMetal::AddKernel() +{ + NSString* kname = [[NSString alloc] initWithFormat:@"krnl_%s", GetKernelName()]; + + id krnl = [mInternals->library newFunctionWithName:kname]; + if (krnl == nil) { + GPUError("Error creating Metal Kernel: %s", [kname cStringUsingEncoding:NSUTF8StringEncoding]); + return 1; + } + + mInternals->functions.emplace_back(krnl); + return 0; +} + +template +S& GPUReconstructionMetal::getKernelObject() +{ + return mInternals->functions[GetKernelNum()]; +} + +int32_t GPUReconstructionMetal::AddKernels() +{ +#define GPUCA_KRNL(x_class, ...) \ + if (AddKernel()) { \ + return 1; \ + } +#include "GPUReconstructionKernelList.h" +#undef GPUCA_KRNL + return 0; +} diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc new file mode 100644 index 0000000000000..1ee192cff822f --- /dev/null +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -0,0 +1,27 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUReconstructionMetalKernelsSpecialize.inc + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size) +{ + const uint64_t offset = static_cast(ptr) - static_cast(mDeviceMemoryBase); + const uint64_t length = (size + 15ull) & ~15ull; + + id cb = mInternals->commandBuffers[_xyz.x.stream]; + id blit = [cb blitCommandEncoder]; + + [blit fillBuffer:mInternals->mem_gpu range:NSMakeRange(offset, length) value:0]; + [blit endEncoding]; + + [cb commit]; +} diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt index ca58d91212084..cc44b2003a01d 100644 --- a/GPU/GPUTracking/CMakeLists.txt +++ b/GPU/GPUTracking/CMakeLists.txt @@ -463,41 +463,39 @@ if (onnxruntime_FOUND) endif() # Add CMake recipes for GPU Tracking librararies -if(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED) - if(CMAKE_SYSTEM_NAME MATCHES Darwin) - message(WARNING "GPU Tracking disabled on MacOS") - else() - make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch) - set(GPU_CONST_PARAM_FILES) - foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES}) - set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par) - add_custom_command( - OUTPUT ${PARAMFILE} - COMMAND bash -c - "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'" - | root -l -b > /dev/null - VERBATIM - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch - MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C - DEPENDS ${GPU_DEFAULT_PARAMS_HEADER} - ${GPU_DEFAULT_PARAMS_HEADER_DEVICE} - ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h - ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc - COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}") - LIST(APPEND GPU_CONST_PARAM_FILES ${PARAMFILE}) - endforeach() - add_custom_target(${MODULE}_GPU_CONST_PARAM_ARCHS ALL DEPENDS ${GPU_CONST_PARAM_FILES}) - install(FILES ${GPU_CONST_PARAM_FILES} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/arch_param) - - if(CUDA_ENABLED) - add_subdirectory(Base/cuda) - endif() - if(OPENCL_ENABLED) - add_subdirectory(Base/opencl) - endif() - if(HIP_ENABLED) - add_subdirectory(Base/hip) - endif() +if(METAL_ENABLED) + add_subdirectory(Base/metal) +elseif(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED) + make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch) + set(GPU_CONST_PARAM_FILES) + foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES}) + set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par) + add_custom_command( + OUTPUT ${PARAMFILE} + COMMAND bash -c + "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'" + | root -l -b > /dev/null + VERBATIM + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch + MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C + DEPENDS ${GPU_DEFAULT_PARAMS_HEADER} + ${GPU_DEFAULT_PARAMS_HEADER_DEVICE} + ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h + ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc + COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}") + LIST(APPEND GPU_CONST_PARAM_FILES ${PARAMFILE}) + endforeach() + add_custom_target(${MODULE}_GPU_CONST_PARAM_ARCHS ALL DEPENDS ${GPU_CONST_PARAM_FILES}) + install(FILES ${GPU_CONST_PARAM_FILES} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/arch_param) + + if(CUDA_ENABLED) + add_subdirectory(Base/cuda) + endif() + if(OPENCL_ENABLED) + add_subdirectory(Base/opencl) + endif() + if(HIP_ENABLED) + add_subdirectory(Base/hip) endif() endif() diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index b499ea10e679b..7e74d209cbd68 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -33,7 +33,7 @@ GPUdii() void GPUTPCCompressionKernels::Thread(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); #else // GPUCA_DETERMINISTIC_MODE if (param.rec.tpc.compressionSortOrder == GPUSettings::SortZPadTime) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } else if (param.rec.tpc.compressionSortOrder == GPUSettings::SortZTimePad) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } else if (param.rec.tpc.compressionSortOrder == GPUSettings::SortPad) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } else if (param.rec.tpc.compressionSortOrder == GPUSettings::SortTime) { - CAAlgo::sortInBlock(sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); + CAAlgo::sortInBlock(nThreads, iThread, sortBuffer, sortBuffer + count, GPUTPCCompressionKernels_Compare(clusters->clusters[iSector][iRow])); } #endif // GPUCA_DETERMINISTIC_MODE GPUbarrier(); } - for (uint32_t j = get_local_id(0); j < count; j += get_local_size(0)) { + for (uint32_t j = iThread; j < count; j += nThreads) { int32_t outidx = idOffsetOut + totalCount + j; const ClusterNative& GPUrestrict() orgCl = clusters -> clusters[iSector][iRow][sortBuffer[j]]; diff --git a/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx index 0d2140c32e4a9..f245e3b53d2bd 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCDecompressionKernels.cxx @@ -31,7 +31,7 @@ GPUdii() void GPUTPCDecompressionKernels::ThreadnClusters[sector][row]; k++) { @@ -125,7 +125,7 @@ GPUdii() void GPUTPCDecompressionUtilKernels::ThreadclusterOffset[sector][row]; diff --git a/GPU/GPUTracking/Definitions/GPUDef.h b/GPU/GPUTracking/Definitions/GPUDef.h index 692e0c5ebe231..ece7cc57f7096 100644 --- a/GPU/GPUTracking/Definitions/GPUDef.h +++ b/GPU/GPUTracking/Definitions/GPUDef.h @@ -21,17 +21,27 @@ #include "GPUDefParametersWrapper.h" #include "GPUCommonRtypes.h" -// Macros for masking ptrs in OpenCL kernel calls as uint64_t (The API only allows us to pass buffer objects) +// Macros for kernel arguments. OpenCL can only pass buffer objects, so pointers +// are masked as uint64_t and cast back inside the kernel. MSL needs an explicit +// buffer index on every parameter, but can bind a pointer directly. The index is +// emitted per argument by o2_gpu_add_kernel; 0, 1 and 2 are taken by gpu_mem, +// the constant memory and the sector index. #ifdef __OPENCL__ - #define GPUPtr1(a, b) uint64_t b - #ifdef __OPENCL__ - #define GPUPtr2(a, b) ((__generic a) (a) b) - #else - #define GPUPtr2(a, b) ((__global a) (a) b) - #endif + #define GPUPtr1(idx, a, b) uint64_t b + #define GPUPtr2(a, b) ((__generic a) (a) b) + #define GPUArg1(idx, a, b) a b +#elif defined(__METAL__) + // As for OpenCL, pointers travel as a 64-bit address: a pointer to a derived + // class is not a valid kernel argument type in MSL either. + #define GPUPtr1(idx, a, b) constant uint64_t& b [[buffer(idx)]] + // through device and then to generic: the kernel's own buffers are device + // memory, but the Thread() entry points take the pointer unannotated + #define GPUPtr2(a, b) ((a)((device a)(b))) + #define GPUArg1(idx, a, b) constant a& b [[buffer(idx)]] #else - #define GPUPtr1(a, b) a b + #define GPUPtr1(idx, a, b) a b #define GPUPtr2(a, b) b + #define GPUArg1(idx, a, b) a b #endif #define GPUCA_EVDUMP_FILE "event" @@ -39,19 +49,19 @@ #ifdef GPUCA_GPUCODE #define GPUCA_MAKE_SHARED_REF(vartype, varname, varglobal, varshared) const GPUsharedref() vartype& __restrict__ varname = varshared; #define GPUCA_SHARED_STORAGE(storage) storage - #define GPUCA_SHARED_CACHE(target, src, size) \ + #define GPUCA_SHARED_CACHE(nThreads, iThread, target, src, size) \ static_assert((size) % sizeof(int32_t) == 0, "Invalid shared cache size"); \ - for (uint32_t i_shared_cache = get_local_id(0); i_shared_cache < (size) / sizeof(int32_t); i_shared_cache += get_local_size(0)) { \ + for (uint32_t i_shared_cache = (iThread); i_shared_cache < (size) / sizeof(int32_t); i_shared_cache += (nThreads)) { \ reinterpret_cast(target)[i_shared_cache] = reinterpret_cast(src)[i_shared_cache]; \ } - #define GPUCA_SHARED_CACHE_REF(target, src, size, reftype, ref) \ - GPUCA_SHARED_CACHE(target, src, size) \ + #define GPUCA_SHARED_CACHE_REF(nThreads, iThread, target, src, size, reftype, ref) \ + GPUCA_SHARED_CACHE(nThreads, iThread, target, src, size) \ GPUsharedref() const reftype* __restrict__ ref = (target) #else #define GPUCA_MAKE_SHARED_REF(vartype, varname, varglobal, varshared) const GPUglobalref() vartype & __restrict__ varname = varglobal; #define GPUCA_SHARED_STORAGE(storage) - #define GPUCA_SHARED_CACHE(target, src, size) - #define GPUCA_SHARED_CACHE_REF(target, src, size, reftype, ref) GPUglobalref() const reftype* __restrict__ ref = src + #define GPUCA_SHARED_CACHE(nThreads, iThread, target, src, size) + #define GPUCA_SHARED_CACHE_REF(nThreads, iThread, target, src, size, reftype, ref) GPUglobalref() const reftype* __restrict__ ref = src #endif #endif //GPUTPCDEF_H diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 3ee65bc02d983..134068af8af6b 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -221,7 +221,7 @@ AddSubConfig(GPUSettingsRecDynamic, dyn) AddHelp("help", 'h') EndConfig() -#ifndef __OPENCL__ +#if !defined(__OPENCL__) && !defined(__METAL__) // these use std::string / std::vector, which device code does not have // Parameters that might affect the RTC code (if these change, the cache cannot be used) BeginSubConfig(GPUSettingsProcessingRTC, rtc, configStandalone.proc, "RTC", 0, "Processing settings", proc_rtc) AddOption(cacheOutput, bool, false, "", 0, "Cache RTC compilation results") @@ -428,7 +428,7 @@ AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) AddSubConfig(GPUSettingsProcessingScaling, scaling) AddHelp("help", 'h') EndConfig() -#endif // __OPENCL__ +#endif // !__OPENCL__ && !__METAL__ #ifndef GPUCA_GPUCODE_DEVICE // Light settings concerning the event display (can be changed without rebuilding vertices) diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 12c87afceb86f..f846641cefc9e 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -1,117 +1,117 @@ -Architecture,default,default_cpu,MI100,VEGA,TAHITI,TESLA,FERMI,PASCAL,KEPLER,AMPERE,TURING,HOPPER,ADA,OPENCL,RDNA,MI210,BLACKWELL,MI300 -,,,,,,,,,,,,,,,,,, -CORE:,,,,,,,,,,,,,,,,,, -WARP_SIZE,0,,64,64,32,32,32,32,32,32,32,32,32,32,32,64,32,64 -THREAD_COUNT_DEFAULT,256,,256,256,,,,,,512,512,,512,256,512,512,512, -,,,,,,,,,,,,,,,,,, -LB:,,,,,,,,,,,,,,,,,, -GPUTPCCreateTrackingData,256,,"[256, 7]","[192, 2]",,,,,,"[224, 7]",256,"[128, 14]",416,,"[64, 21]",,384,"[320, 2]" -GPUTPCTrackletConstructor,256,,"[768, 8]","[512, 10]","[256, 2]","[256, 1]","[256, 2]","[1024, 2]","[512, 4]",1024,"[256, 2]",1024,"[1024, 1]",,"[768, 2]",,768,512 -GPUTPCTrackletSelector,256,,"[384, 5]","[192, 10]","[256, 3]","[256, 1]","[256, 3]","[512, 4]","[256, 3]","[288, 3]","[192, 3]","[544, 1]","[32, 2]",,"[384, 3]",,992,"[256, 6]" -GPUTPCNeighboursFinder,256,,"[192, 8]","[960, 8]",256,256,256,512,256,864,"[640, 1]","[512, 2]","[736, 1]",,"[480, 3]",,992,"[704, 1]" -GPUTPCNeighboursCleaner,256,,"[128, 5]","[384, 9]",256,256,256,256,256,544,512,"[192, 9]","[512, 1]",,"[384, 5]",,672,"[640, 1]" -GPUTPCExtrapolationTracking,256,,"[256, 7]","[256, 2]",,,,,,"[352, 4]","[192, 2]","[896, 1]","[352, 1]",,"[1024, 1]",,896,1024 -GPUTRDTrackerKernels_gpuVersion,512,,,,,,,,,512,,512,512,,512,,,512 -GPUTPCCreateOccupancyMap_fill,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCCreateOccupancyMap_fold,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTRDTrackerKernels_o2Version,512,,,,,,,,,512,,512,512,,512,,,512 -GPUTPCCompressionKernels_step0attached,256,,"[128, 1]","[64, 2]",,,,,,"[160, 2]",128,"[448, 1]",352,,"[1024, 1]",,"[96, 3]","[128, 4]" -GPUTPCCompressionKernels_step1unattached,256,,"[512, 2]","[512, 2]",,,,,,"[288, 4]","[512, 2]","[256, 4]","[512, 2]",,"[512, 3]",,"[512, 2]","[512, 3]" -GPUTPCDecompressionKernels_step0attached,256,,"[128, 2]","[128, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,"[128, 1]",,"[32, 1]","[128, 1]" -GPUTPCDecompressionKernels_step1unattached,256,,"[64, 2]","[64, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,"[64, 1]",,"[32, 1]","[64, 1]" -GPUTPCDecompressionUtilKernels_sortPerSectorRow,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCDecompressionUtilKernels_countFilteredClusters,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCDecompressionUtilKernels_storeFilteredClusters,256,,,,,,,,,256,,256,256,,256,,,256 -GPUTPCCFDecodeZS,"[128, 4]",,"[64, 4]","[64, 1]",,,,,,"[32, 10]","[64, 8]","[32, 10]","[32, 10]",,"[64, 1]",,"[64, 10]","[64, 1]" -GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,,,,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,64,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""" -GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,"[64, 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]" -GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,"[928, 1]",,"[1024, 1]","[320, 2]" -COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,, -GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]" -GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,"[128, 16]",,"[224, 3]","[256, 7]" -GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,"[32, 20]",,"[32, 10]","[64, 4]" -GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerUnpackGlobal,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step0,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step1,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step2,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step3,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerResolve_step4,256,,512,256,,,,,,"[256, 4]","[256, 4]","[256, 4]","[256, 4]",,256,,"[256, 4]",256 -GPUTPCGMMergerClearLinks,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerMergeWithinPrepare,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerMergeSectorsPrepare,256,,256,256,,,,,,"[256, 2]","[256, 2]","[256, 2]","[256, 2]",,256,,"[256, 2]",256 -GPUTPCGMMergerMergeBorders_step0,256,,512,256,,,,,,192,192,192,192,,256,,192,256 -GPUTPCGMMergerMergeBorders_step2,256,,512,256,,,,,,"[64, 2]",256,"[64, 2]","[64, 2]",,256,,"[64, 2]",256 -GPUTPCGMMergerMergeCE,256,,512,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerLinkExtrapolatedTracks,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerCollect,256,,"[768, 1]","[1024, 1]",,,,,,"[864, 1]","[128, 2]","[896, 1]",128,,1024,,"[288, 1]","[384, 4]" -GPUTPCGMMergerSortTracksPrepare,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerPrepareForFit_step0,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerPrepareForFit_step1,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,256,,256,,256,256 -GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMO2Output_prepare,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCGMO2Output_output,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTPCStartHitsFinder,256,,"[1024, 2]","[1024, 7]",256,256,256,256,256,"[224, 1]",512,"[416, 4]",928,,"[320, 5]",,608,"[448, 3]" -GPUTPCStartHitsSorter,256,,"[1024, 5]","[512, 7]",256,256,256,256,256,"[320, 2]","[512, 1]","[864, 1]","[96, 2]",,"[192, 5]",,608,"[448, 1]" -GPUTPCCFCheckPadBaseline,576,,"[576, 2]","[576, 2]",,,,,,"[576, 3]",,"[576, 1]","[576, 1]",,"[576, 2]",,"[576, 2]",576 -GPUTPCCFHIPTailConnector,256,,256,256,,,,,,"[224, 2]",,"[320, 5]","[704, 1]",,"[128, 7]",,,"[448, 4]" -GPUTPCCFHIPClusterizer,256,,256,256,,,,,,"[288, 5]",,"[480, 3]","[448, 3]",,352,,,"[512, 3]" -GPUTPCCFChargeMapFiller_fillIndexMap,512,,512,512,,,,,,448,,448,448,,512,,448,512 -GPUTPCCFChargeMapFiller_fillFromDigits,512,,512,512,,,,,,448,,448,448,,512,,448,512 -GPUTPCCFChargeMapFiller_findFragmentStart,512,,512,512,,,,,,448,,448,448,,512,,448,512 -GPUTPCCFPeakFinder,512,,"[512, 9]","[512, 4]",,,,,,416,,992,"[672, 1]",,"[384, 2]",,"[128, 5]","[192, 10]" -GPUTPCCFNoiseSuppression,512,,512,512,,,,,,608,,896,480,,160,,,448 -GPUTPCCFDeconvolution,512,,"[512, 5]","[512, 5]",,,,,,"[480, 4]",,224,512,,480,,384,"[448, 3]" -GPUTPCCFClusterizer,512,,"[448, 3]","[512, 2]",,,,,,"[608, 3]",,736,"[192, 3]",,576,,"[160, 5]","[832, 2]" -GPUTPCNNClusterizerKernels,512,,,,,,,,,,,,,,,,, -GPUTrackingRefitKernel_mode0asGPU,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUTrackingRefitKernel_mode1asTrackParCov,256,,,,,,,,,256,,256,256,,256,,256,256 -GPUMemClean16,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,"[256, 1]",,,"[256, 1]" -GPUitoa,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,"[256, 1]",,,"[256, 1]" -GPUTPCCFNoiseSuppression_noiseSuppression,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,448, -GPUTPCCFNoiseSuppression_updatePeaks,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,448, -GPUTPCNNClusterizerKernels_runCfClusterizer,"""GPUCA_LB_GPUTPCCFClusterizer""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_fillInputNNCPU,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_fillInputNNGPU,1024,,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_determineClass1Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_determineClass2Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_publishClass1Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_publishClass2Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCNNClusterizerKernels_publishDeconvolutionFlags,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanStart,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanUp,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanTop,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_scanDown,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCFStreamCompaction_compactDigits,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_unbuffered,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_buffered32,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_buffered64,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_buffered128,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCCompressionGatherKernels_multiBlock,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,, -GPUTPCGMMergerFinalize_0,256,,256,,,,,,,256,256,,256,,,,256, -GPUTPCGMMergerFinalize_1,256,,256,,,,,,,256,256,,256,,,,256, -GPUTPCGMMergerFinalize_2,256,,256,,,,,,,256,256,,256,,,,256, -GPUTPCConvertKernel,,,,,,,,,,,,,,,256,,,256 -,,,,,,,,,,,,,,,,,, -PAR:,,,,,,,,,,,,,,,,,, -AMD_EUS_PER_CU,0,0,4,4,,,,,,,,,,,4,,0,4 -SORT_STARTHITS,1,0,,,,,,,,1,,1,1,,1,,1,1 -NEIGHBOURS_FINDER_MAX_NNEIGHUP,6,0,10,4,,,,,,4,4,4,4,,5,,2,5 -NEIGHBOURS_FINDER_UNROLL_GLOBAL,4,0,4,2,,,,,,2,,8,8,,4,,2,2 -NEIGHBOURS_FINDER_UNROLL_SHARED,1,0,0,0,,,,,,1,,1,0,,1,,1,1 -TRACKLET_SELECTOR_HITS_REG_SIZE,12,0,9,27,,,,,,20,20,20,20,,20,,2,20 -ALTERNATE_BORDER_SORT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1 -SORT_BEFORE_FIT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1 -NO_ATOMIC_PRECHECK,0,0,1,1,,,,,,1,1,1,1,,1,,1,1 -DEDX_STORAGE_TYPE,"""half""","""float""",,,,,,,,,,,,,,,, -MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,, -COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,4,,4,4 -COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,3,,3,3 -CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,1024,,,448 -MERGER_SPLIT_LOOP_INTERPOLATION,,,,,,,,,,,,,,,1,,,1 +Architecture,default,default_cpu,MI100,VEGA,TAHITI,TESLA,FERMI,PASCAL,KEPLER,AMPERE,TURING,HOPPER,ADA,OPENCL,METAL,RDNA,MI210,BLACKWELL,MI300 +,,,,,,,,,,,,,,,,,,, +CORE:,,,,,,,,,,,,,,,,,,, +WARP_SIZE,0,,64,64,32,32,32,32,32,32,32,32,32,32,32,32,64,32,64 +THREAD_COUNT_DEFAULT,256,,256,256,,,,,,512,512,,512,256,256,512,512,512, +,,,,,,,,,,,,,,,,,,, +LB:,,,,,,,,,,,,,,,,,,, +GPUTPCCreateTrackingData,256,,"[256, 7]","[192, 2]",,,,,,"[224, 7]",256,"[128, 14]",416,,,"[64, 21]",,384,"[320, 2]" +GPUTPCTrackletConstructor,256,,"[768, 8]","[512, 10]","[256, 2]","[256, 1]","[256, 2]","[1024, 2]","[512, 4]",1024,"[256, 2]",1024,"[1024, 1]",,,"[768, 2]",,768,512 +GPUTPCTrackletSelector,256,,"[384, 5]","[192, 10]","[256, 3]","[256, 1]","[256, 3]","[512, 4]","[256, 3]","[288, 3]","[192, 3]","[544, 1]","[32, 2]",,,"[384, 3]",,992,"[256, 6]" +GPUTPCNeighboursFinder,256,,"[192, 8]","[960, 8]",256,256,256,512,256,864,"[640, 1]","[512, 2]","[736, 1]",,,"[480, 3]",,992,"[704, 1]" +GPUTPCNeighboursCleaner,256,,"[128, 5]","[384, 9]",256,256,256,256,256,544,512,"[192, 9]","[512, 1]",,,"[384, 5]",,672,"[640, 1]" +GPUTPCExtrapolationTracking,256,,"[256, 7]","[256, 2]",,,,,,"[352, 4]","[192, 2]","[896, 1]","[352, 1]",,,"[1024, 1]",,896,1024 +GPUTRDTrackerKernels_gpuVersion,512,,,,,,,,,512,,512,512,,,512,,,512 +GPUTPCCreateOccupancyMap_fill,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCCreateOccupancyMap_fold,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTRDTrackerKernels_o2Version,512,,,,,,,,,512,,512,512,,,512,,,512 +GPUTPCCompressionKernels_step0attached,256,,"[128, 1]","[64, 2]",,,,,,"[160, 2]",128,"[448, 1]",352,,,"[1024, 1]",,"[96, 3]","[128, 4]" +GPUTPCCompressionKernels_step1unattached,256,,"[512, 2]","[512, 2]",,,,,,"[288, 4]","[512, 2]","[256, 4]","[512, 2]",,,"[512, 3]",,"[512, 2]","[512, 3]" +GPUTPCDecompressionKernels_step0attached,256,,"[128, 2]","[128, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,,"[128, 1]",,"[32, 1]","[128, 1]" +GPUTPCDecompressionKernels_step1unattached,256,,"[64, 2]","[64, 2]",,,,,,"[32, 1]","[32, 1]","[32, 1]","[32, 1]",,,"[64, 1]",,"[32, 1]","[64, 1]" +GPUTPCDecompressionUtilKernels_sortPerSectorRow,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCDecompressionUtilKernels_countFilteredClusters,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCDecompressionUtilKernels_storeFilteredClusters,256,,,,,,,,,256,,256,256,,,256,,,256 +GPUTPCCFDecodeZS,"[128, 4]",,"[64, 4]","[64, 1]",,,,,,"[32, 10]","[64, 8]","[32, 10]","[32, 10]",,,"[64, 1]",,"[64, 10]","[64, 1]" +GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,,,,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""",,,64,,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_SIZE""" +GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,,"[64, 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]" +GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,,"[928, 1]",,"[1024, 1]","[320, 2]" +COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,,, +GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,,"[32, 24]",,"[64, 8]","[64, 6]" +GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,,"[128, 16]",,"[224, 3]","[256, 7]" +GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,,"[32, 20]",,"[32, 10]","[64, 4]" +GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerUnpackGlobal,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step0,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step1,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step2,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step3,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerResolve_step4,256,,512,256,,,,,,"[256, 4]","[256, 4]","[256, 4]","[256, 4]",,,256,,"[256, 4]",256 +GPUTPCGMMergerClearLinks,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerMergeWithinPrepare,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerMergeSectorsPrepare,256,,256,256,,,,,,"[256, 2]","[256, 2]","[256, 2]","[256, 2]",,,256,,"[256, 2]",256 +GPUTPCGMMergerMergeBorders_step0,256,,512,256,,,,,,192,192,192,192,,,256,,192,256 +GPUTPCGMMergerMergeBorders_step2,256,,512,256,,,,,,"[64, 2]",256,"[64, 2]","[64, 2]",,,256,,"[64, 2]",256 +GPUTPCGMMergerMergeCE,256,,512,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerLinkExtrapolatedTracks,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerCollect,256,,"[768, 1]","[1024, 1]",,,,,,"[864, 1]","[128, 2]","[896, 1]",128,,,1024,,"[288, 1]","[384, 4]" +GPUTPCGMMergerSortTracksPrepare,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerPrepareForFit_step0,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerPrepareForFit_step1,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,256,,,256,,256,256 +GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMO2Output_prepare,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCGMO2Output_output,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTPCStartHitsFinder,256,,"[1024, 2]","[1024, 7]",256,256,256,256,256,"[224, 1]",512,"[416, 4]",928,,,"[320, 5]",,608,"[448, 3]" +GPUTPCStartHitsSorter,256,,"[1024, 5]","[512, 7]",256,256,256,256,256,"[320, 2]","[512, 1]","[864, 1]","[96, 2]",,,"[192, 5]",,608,"[448, 1]" +GPUTPCCFCheckPadBaseline,576,,"[576, 2]","[576, 2]",,,,,,"[576, 3]",,"[576, 1]","[576, 1]",,,"[576, 2]",,"[576, 2]",576 +GPUTPCCFHIPTailConnector,256,,256,256,,,,,,"[224, 2]",,"[320, 5]","[704, 1]",,,"[128, 7]",,,"[448, 4]" +GPUTPCCFHIPClusterizer,256,,256,256,,,,,,"[288, 5]",,"[480, 3]","[448, 3]",,,352,,,"[512, 3]" +GPUTPCCFChargeMapFiller_fillIndexMap,512,,512,512,,,,,,448,,448,448,,,512,,448,512 +GPUTPCCFChargeMapFiller_fillFromDigits,512,,512,512,,,,,,448,,448,448,,,512,,448,512 +GPUTPCCFChargeMapFiller_findFragmentStart,512,,512,512,,,,,,448,,448,448,,,512,,448,512 +GPUTPCCFPeakFinder,512,,"[512, 9]","[512, 4]",,,,,,416,,992,"[672, 1]",,,"[384, 2]",,"[128, 5]","[192, 10]" +GPUTPCCFNoiseSuppression,512,,512,512,,,,,,608,,896,480,,,160,,,448 +GPUTPCCFDeconvolution,512,,"[512, 5]","[512, 5]",,,,,,"[480, 4]",,224,512,,,480,,384,"[448, 3]" +GPUTPCCFClusterizer,512,,"[448, 3]","[512, 2]",,,,,,"[608, 3]",,736,"[192, 3]",,,576,,"[160, 5]","[832, 2]" +GPUTPCNNClusterizerKernels,512,,,,,,,,,,,,,,,,,, +GPUTrackingRefitKernel_mode0asGPU,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUTrackingRefitKernel_mode1asTrackParCov,256,,,,,,,,,256,,256,256,,,256,,256,256 +GPUMemClean16,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,,"[256, 1]",,,"[256, 1]" +GPUitoa,"[""GPUCA_THREAD_COUNT_DEFAULT"", 1]",,,,,,,,,"[512, 1]",,"[512, 1]","[512, 1]",,,"[256, 1]",,,"[256, 1]" +GPUTPCCFNoiseSuppression_noiseSuppression,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,,448, +GPUTPCCFNoiseSuppression_updatePeaks,"""GPUCA_LB_GPUTPCCFNoiseSuppression""",,,,,,,,,,,,,,,,,448, +GPUTPCNNClusterizerKernels_runCfClusterizer,"""GPUCA_LB_GPUTPCCFClusterizer""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_fillInputNNCPU,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_fillInputNNGPU,1024,,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_determineClass1Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_determineClass2Labels,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_publishClass1Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_publishClass2Regression,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCNNClusterizerKernels_publishDeconvolutionFlags,"""GPUCA_LB_GPUTPCNNClusterizerKernels""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanStart,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanUp,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanTop,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_scanDown,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCFStreamCompaction_compactDigits,"""GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_unbuffered,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_buffered32,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_buffered64,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_buffered128,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCCompressionGatherKernels_multiBlock,"""GPUCA_LB_COMPRESSION_GATHER""",,,,,,,,,,,,,,,,,, +GPUTPCGMMergerFinalize_0,256,,256,,,,,,,256,256,,256,,,,,256, +GPUTPCGMMergerFinalize_1,256,,256,,,,,,,256,256,,256,,,,,256, +GPUTPCGMMergerFinalize_2,256,,256,,,,,,,256,256,,256,,,,,256, +GPUTPCConvertKernel,,,,,,,,,,,,,,,,256,,,256 +,,,,,,,,,,,,,,,,,,, +PAR:,,,,,,,,,,,,,,,,,,, +AMD_EUS_PER_CU,0,0,4,4,,,,,,,,,,,,4,,0,4 +SORT_STARTHITS,1,0,,,,,,,,1,,1,1,,,1,,1,1 +NEIGHBOURS_FINDER_MAX_NNEIGHUP,6,0,10,4,,,,,,4,4,4,4,,,5,,2,5 +NEIGHBOURS_FINDER_UNROLL_GLOBAL,4,0,4,2,,,,,,2,,8,8,,,4,,2,2 +NEIGHBOURS_FINDER_UNROLL_SHARED,1,0,0,0,,,,,,1,,1,0,,,1,,1,1 +TRACKLET_SELECTOR_HITS_REG_SIZE,12,0,9,27,,,,,,20,20,20,20,,,20,,2,20 +ALTERNATE_BORDER_SORT,1,0,1,1,,,,,,1,1,1,1,,,1,,1,1 +SORT_BEFORE_FIT,1,0,1,1,,,,,,1,1,1,1,,,1,,1,1 +NO_ATOMIC_PRECHECK,0,0,1,1,,,,,,1,1,1,1,,,1,,1,1 +DEDX_STORAGE_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, +MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, +COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,,4,,4,4 +COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,,3,,3,3 +CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,,1024,,,448 +MERGER_SPLIT_LOOP_INTERPOLATION,,,,,,,,,,,,,,,,1,,,1 diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 759aaf818028e..868400401d7a1 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -300,11 +300,11 @@ class GPUChainTracking : public GPUChain int32_t RunTPCTrackingSectors_internal(); int32_t RunTPCClusterizer_prepare(bool restorePointers, const GPUTPCExtraADC& extraADCs); #ifndef GPUCA_RUN2 - std::pair RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& fragment, int32_t lane, const GPUTPCExtraADC& extraADCs); + std::pair RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& frag, int32_t lane, const GPUTPCExtraADC& extraADCs); void RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int32_t stage, bool doGPU, int32_t lane); - std::pair TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& fragment); - std::pair TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& fragment); - void TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& fragment); + std::pair TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& frag); + std::pair TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& frag); + void TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& frag); void TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs); void TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs); #endif diff --git a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx index 8c6534d74b31d..1557b2fab0934 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx @@ -70,7 +70,7 @@ using namespace o2::tpc::constants; using namespace o2::dataformats; #ifndef GPUCA_RUN2 -std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& fragment) +std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdate(uint32_t iSector, const CfFragment& frag) { bool doGPU = mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCClusterFinding; GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; @@ -78,7 +78,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdat uint32_t digits = 0; uint32_t pages = 0; for (uint16_t j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) { - clusterer.mMinMaxCN[j] = mCFContext->fragmentData[fragment.index].minMaxCN[iSector][j]; + clusterer.mMinMaxCN[j] = mCFContext->fragmentData[frag.index].minMaxCN[iSector][j]; if (doGPU) { uint16_t posInEndpoint = 0; uint16_t pagesEndpoint = 0; @@ -86,7 +86,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdat const uint32_t pageFirst = (k == clusterer.mMinMaxCN[j].zsPtrFirst) ? clusterer.mMinMaxCN[j].zsPageFirst : 0; const uint32_t pageLast = (k + 1 == clusterer.mMinMaxCN[j].zsPtrLast) ? clusterer.mMinMaxCN[j].zsPageLast : mIOPtrs.tpcZS->sector[iSector].nZSPtr[j][k]; for (uint32_t l = pageFirst; l < pageLast; l++) { - uint16_t pageDigits = mCFContext->fragmentData[fragment.index].pageDigits[iSector][j][posInEndpoint++]; + uint16_t pageDigits = mCFContext->fragmentData[frag.index].pageDigits[iSector][j][posInEndpoint++]; if (pageDigits) { *(o++) = GPUTPCClusterFinder::ZSOffset{digits, j, pagesEndpoint}; digits += pageDigits; @@ -94,35 +94,35 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCountUpdat pagesEndpoint++; } } - if (pagesEndpoint != mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()) { + if (pagesEndpoint != mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()) { if (GetProcessingSettings().ignoreNonFatalGPUErrors) { - GPUError("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()); + GPUError("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()); return {0, 0}; } else { - GPUFatal("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[fragment.index].pageDigits[iSector][j].size()); + GPUFatal("TPC raw page count mismatch in TPCClusterizerDecodeZSCountUpdate: expected %d / buffered %lu", pagesEndpoint, mCFContext->fragmentData[frag.index].pageDigits[iSector][j].size()); } } } else { clusterer.mPzsOffsets[j] = GPUTPCClusterFinder::ZSOffset{digits, j, 0}; - digits += mCFContext->fragmentData[fragment.index].nDigits[iSector][j]; - pages += mCFContext->fragmentData[fragment.index].nPages[iSector][j]; + digits += mCFContext->fragmentData[frag.index].nDigits[iSector][j]; + pages += mCFContext->fragmentData[frag.index].nPages[iSector][j]; } } if (doGPU) { pages = o - processors()->tpcClusterer[iSector].mPzsOffsets; } if (GetProcessingSettings().clusterizerZSSanityCheck && mCFContext->zsVersion >= ZSVersion::ZSVersionDenseLinkBased) { - TPCClusterizerEnsureZSOffsets(iSector, fragment); + TPCClusterizerEnsureZSOffsets(iSector, frag); } return {digits, pages}; } -void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& fragment) +void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfFragment& frag) { GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; uint32_t nAdcs = 0; for (uint16_t endpoint = 0; endpoint < GPUTrackingInOutZS::NENDPOINTS; endpoint++) { - const auto& data = mCFContext->fragmentData[fragment.index]; + const auto& data = mCFContext->fragmentData[frag.index]; uint32_t pagesEndpoint = 0; const uint32_t nAdcsExpected = data.nDigits[iSector][endpoint]; const uint32_t nPagesExpected = data.nPages[iSector][endpoint]; @@ -144,15 +144,15 @@ void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfF } if (pagesEndpoint != nPagesExpected) { - GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector, endpoint, fragment.index, pagesEndpoint, nPagesExpected); + GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC raw page count mismatch: expected %d / buffered %u", iSector, endpoint, frag.index, pagesEndpoint, nPagesExpected); } if (nAdcDecoded != nAdcsExpected) { - GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector, endpoint, fragment.index, nAdcsExpected, nAdcDecoded); + GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC count mismatch: expected %u, buffered %u", iSector, endpoint, frag.index, nAdcsExpected, nAdcDecoded); } if (nAdcs != clusterer.mPzsOffsets[endpoint].offset) { - GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC offset mismatch: expected %u, buffered %u", iSector, endpoint, fragment.index, nAdcs, clusterer.mPzsOffsets[endpoint].offset); + GPUFatal("Sector %d, Endpoint %d, Fragment %d: TPC ADC offset mismatch: expected %u, buffered %u", iSector, endpoint, frag.index, nAdcs, clusterer.mPzsOffsets[endpoint].offset); } nAdcs += nAdcsExpected; @@ -162,10 +162,10 @@ void GPUChainTracking::TPCClusterizerEnsureZSOffsets(uint32_t iSector, const CfF void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs) { const int32_t iSector = clusterer.mISector; - const auto& fragment = clusterer.mPmemory->fragment; + const auto& frag = clusterer.mPmemory->frag; const auto& digits = extraADCs.digitsBySector[iSector]; - if (fragment.index != 0) { + if (frag.index != 0) { return; } @@ -188,11 +188,11 @@ void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clust SynchronizeStream(lane); for (const auto& d : digits) { - if (!fragment.contains(d.getTimeStamp())) { + if (!frag.contains(d.getTimeStamp())) { continue; } - CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - fragment.start)}; + CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - frag.start)}; chargeMapHost[pos] = PackedCharge(d.getChargeFloat()); extraPositions.push_back(pos); @@ -208,10 +208,10 @@ void GPUChainTracking::TPCClusterizerTransferExtraADC(GPUTPCClusterFinder& clust void GPUChainTracking::TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int lane, const GPUTPCExtraADC& extraADCs) { const int32_t iSector = clusterer.mISector; - const auto& fragment = clusterer.mPmemory->fragment; + const auto& frag = clusterer.mPmemory->frag; const auto& digits = extraADCs.digitsBySector[iSector]; - if (fragment.index != 0) { + if (frag.index != 0) { return; } @@ -233,11 +233,11 @@ void GPUChainTracking::TPCClusterizerCheckExtraADCZeros(GPUTPCClusterFinder& clu size_t nNonZeroADCs = 0; for (const auto& d : digits) { - if (!fragment.contains(d.getTimeStamp())) { + if (!frag.contains(d.getTimeStamp())) { continue; } - CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - fragment.start)}; + CfChargePos pos{(tpccf::Row)d.getRow(), (tpccf::Pad)d.getPad(), (tpccf::TPCFragmentTime)(d.getTimeStamp() - frag.start)}; auto adc = chargeMapHost[pos].unpack(); @@ -326,7 +326,7 @@ GPUTPCExtraADC GenerateSaturatedSignals(size_t seed = 42) } // namespace -std::pair GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& fragment) +std::pair GPUChainTracking::TPCClusterizerDecodeZSCount(uint32_t iSector, const CfFragment& frag) { mRec->getGeneralStepTimer(GeneralStep::Prepare).Start(); uint32_t nDigits = 0; @@ -349,7 +349,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCount(uint std::vector> fragments; fragments.reserve(mCFContext->nFragments); - fragments.emplace_back(std::pair{fragment, {0, 0, 0, 0, 0, -1}}); + fragments.emplace_back(std::pair{frag, {0, 0, 0, 0, 0, -1}}); for (uint32_t i = 1; i < mCFContext->nFragments; i++) { fragments.emplace_back(std::pair{fragments.back().first.next(), {0, 0, 0, 0, 0, -1}}); } @@ -601,14 +601,14 @@ void GPUChainTracking::RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clust } } -std::pair GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& fragment, int32_t lane, const GPUTPCExtraADC& extraADCs) +std::pair GPUChainTracking::RunTPCClusterizer_transferZS(int32_t iSector, const CfFragment& frag, int32_t lane, const GPUTPCExtraADC& extraADCs) { bool doGPU = GetRecoStepsGPU() & RecoStep::TPCClusterFinding; if (mCFContext->abandonTimeframe) { return {0, 0}; } - auto retVal = TPCClusterizerDecodeZSCountUpdate(iSector, fragment); - if (fragment.index == 0) { + auto retVal = TPCClusterizerDecodeZSCountUpdate(iSector, frag); + if (frag.index == 0) { retVal.first += extraADCs.digitsBySector[iSector].size(); } if (doGPU) { @@ -994,12 +994,12 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) std::vector laneHasData(GetProcessingSettings().nTPCClustererLanes, false); static_assert(NSECTORS <= constants::GPU_MAX_STREAMS, "Stream events must be able to hold all sectors"); const int32_t maxLane = std::min(GetProcessingSettings().nTPCClustererLanes, NSECTORS - iSectorBase); - for (CfFragment fragment = mCFContext->fragmentFirst; !fragment.isEnd(); fragment = fragment.next()) { + for (CfFragment frag = mCFContext->fragmentFirst; !frag.isEnd(); frag = frag.next()) { if (GetProcessingSettings().debugLevel >= 3) { - GPUInfo("Processing time bins [%d, %d) for sectors %d to %d", fragment.start, fragment.last(), iSectorBase, iSectorBase + GetProcessingSettings().nTPCClustererLanes - 1); + GPUInfo("Processing time bins [%d, %d) for sectors %d to %d", frag.start, frag.last(), iSectorBase, iSectorBase + GetProcessingSettings().nTPCClustererLanes - 1); } mRec->runParallelOuterLoop(doGPU, maxLane, [&](uint32_t lane) { - if (doGPU && fragment.index != 0) { + if (doGPU && frag.index != 0) { SynchronizeStream(lane); // Don't overwrite charge map from previous iteration until cluster computation is finished } @@ -1007,7 +1007,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer; clusterer.mPmemory->counters.nPeaks = clusterer.mPmemory->counters.nClusters = 0; - clusterer.mPmemory->fragment = fragment; + clusterer.mPmemory->frag = frag; if (mIOPtrs.tpcPackedDigits) { bool setDigitsOnGPU = doGPU && not mIOPtrs.tpcZS; @@ -1037,7 +1037,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) using PeakMapType = decltype(*clustererShadow.mPpeakMap); runKernel({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPchargeMap, TPCMapMemoryLayout::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(ChargeMapType)); runKernel({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpeakMap, TPCMapMemoryLayout::items(GetProcessingSettings().overrideClusterizerFragmentLen) * sizeof(PeakMapType)); - if (fragment.index == 0) { + if (frag.index == 0) { runKernel({GetGridAutoStep(lane, RecoStep::TPCClusterFinding)}, clustererShadow.mPpadIsNoisy, TPC_CLUSTERER_STRIDED_PAD_COUNT * sizeof(*clustererShadow.mPpadIsNoisy)); } DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererZeroedCharges, clusterer, &GPUTPCClusterFinder::DumpChargeMap, *mDebugFile, "Zeroed Charges"); @@ -1060,7 +1060,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) } if (propagateMCLabels) { - if (fragment.index == 0) { + if (frag.index == 0) { // Must be only called on the first fragment as some buffers are used across the whole timeframe clusterer.AllocMCBuffers(); } @@ -1112,7 +1112,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) SynchronizeStream(lane); } if (mIOPtrs.tpcZS) { - CfFragment f = fragment.next(); + CfFragment f = frag.next(); int32_t nextSector = iSector; if (f.isEnd()) { nextSector += GetProcessingSettings().nTPCClustererLanes; @@ -1138,7 +1138,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) } bool checkForNoisyPads = (rec()->GetParam().rec.tpc.maxTimeBinAboveThresholdIn1000Bin > 0) || (rec()->GetParam().rec.tpc.maxConsecTimeBinAboveThreshold > 0); - checkForNoisyPads &= (rec()->GetParam().rec.tpc.noisyPadsQuickCheck ? fragment.index == 0 : true); + checkForNoisyPads &= (rec()->GetParam().rec.tpc.noisyPadsQuickCheck ? frag.index == 0 : true); checkForNoisyPads &= !GetProcessingSettings().disableTPCNoisyPadFilter; // TODO Move hipTailFilter flag to ProcessingSettings? // TODO Add some warning when re enabling pad filter with this flag, so it's not just silently enabled when disabling was requested @@ -1152,7 +1152,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) const int32_t nBlocks = GPUTPCGeometry::NROWS; runKernel({GetGridBlk(nBlocks, lane), {iSector}}); - getKernelTimer(RecoStep::TPCClusterFinding, iSector, TPC_REAL_PADS_IN_SECTOR * fragment.lengthWithoutOverlap() * sizeof(PackedCharge), false); + getKernelTimer(RecoStep::TPCClusterFinding, iSector, TPC_REAL_PADS_IN_SECTOR * frag.lengthWithoutOverlap() * sizeof(PackedCharge), false); } DoDebugAndDump(RecoStep::TPCClusterFinding, GPUChainTrackingDebugFlags::TPCClustererDigits, clusterer, &GPUTPCClusterFinder::DumpDigits, *mDebugFile); @@ -1197,7 +1197,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) GPUTPCClusterFinder& clusterer = processors()->tpcClusterer[iSector]; GPUTPCClusterFinder& clustererShadow = doGPU ? processorsShadow()->tpcClusterer[iSector] : clusterer; - const bool resetClusterCounters = fragment.index == 0; + const bool resetClusterCounters = frag.index == 0; // The reset must also run for an empty first fragment since later fragments can contain data. if (clusterer.mPmemory->counters.nPositions == 0 && !resetClusterCounters) { return; @@ -1403,7 +1403,7 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) } if (GetProcessingSettings().debugLevel >= 3) { - GPUInfo("Sector %02d Fragment %02d Lane %d: Found clusters: digits %u peaks %u clusters %u", iSector, fragment.index, lane, (int32_t)clusterer.mPmemory->counters.nPositions, (int32_t)clusterer.mPmemory->counters.nPeaks, (int32_t)clusterer.mPmemory->counters.nClusters); + GPUInfo("Sector %02d Fragment %02d Lane %d: Found clusters: digits %u peaks %u clusters %u", iSector, frag.index, lane, (int32_t)clusterer.mPmemory->counters.nPositions, (int32_t)clusterer.mPmemory->counters.nPeaks, (int32_t)clusterer.mPmemory->counters.nClusters); } TransferMemoryResourcesToHost(RecoStep::TPCClusterFinding, &clusterer, lane); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index e89081aa40350..4598e61993201 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1201,6 +1201,7 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in // PrintMergeGraph(track1, std::cout); // PrintMergeGraph(track2, std::cout); + bool nextTrack = false; while (track2->PrevSegmentNeighbour() >= 0) { track2 = &mSectorTrackInfos[track2->PrevSegmentNeighbour()]; } @@ -1211,26 +1212,41 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in while (track1->PrevSegmentNeighbour() >= 0) { track1 = &mSectorTrackInfos[track1->PrevSegmentNeighbour()]; if (track1 == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + continue; + } GPUCommonAlgorithm::swap(track1, track1Base); for (int32_t k = 0; k < 2; k++) { GPUTPCGMSectorTrack* tmp = track1Base; while (tmp->Neighbour(k) >= 0) { tmp = &mSectorTrackInfos[tmp->Neighbour(k)]; if (tmp == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + break; + } + } + if (nextTrack) { + continue; } while (track1->NextSegmentNeighbour() >= 0) { track1 = &mSectorTrackInfos[track1->NextSegmentNeighbour()]; if (track1 == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + continue; + } } else { while (track1->PrevSegmentNeighbour() >= 0) { track1 = &mSectorTrackInfos[track1->PrevSegmentNeighbour()]; @@ -1244,9 +1260,16 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in while (tmp->Neighbour(k) >= 0) { tmp = &mSectorTrackInfos[tmp->Neighbour(k)]; if (tmp == track2) { - goto NextTrack; + nextTrack = true; + break; } } + if (nextTrack) { + break; + } + } + if (nextTrack) { + continue; } float z1min, z1max, z2min, z2max; @@ -1318,7 +1341,6 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSectors(GPUResolveSharedMemory& smem, in } // GPUInfo("Result"); // PrintMergeGraph(track1, std::cout); - NextTrack:; } } } @@ -1480,8 +1502,8 @@ struct GPUTPCGMMerger_CompareClusterIds { GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { - static constexpr int32_t kMaxParts = 16; - static constexpr int32_t kMaxClusters = constants::MERGER_MAX_TRACK_CLUSTERS; + constexpr int32_t kMaxParts = 16; + constexpr int32_t kMaxClusters = constants::MERGER_MAX_TRACK_CLUSTERS; GPUTPCGMSectorTrack* trackParts[kMaxParts]; @@ -1934,7 +1956,7 @@ GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t GPUd() void GPUTPCGMMerger::MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { const float lowPtThresh = Param().rec.tpc.rejectQPtB5 * 1.1f; // Might need to merge tracks above the threshold with parts below the rejection threshold - for (uint32_t i = get_global_id(0); i < mMemory->nMergedTracks; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < mMemory->nMergedTracks; i += (nBlocks * nThreads)) { const auto& trk = mMergedTracks[i]; const auto& p = trk.GetParam(); const float qptabs = CAMath::Abs(p.GetQPt()); @@ -2003,7 +2025,7 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, } #endif - for (uint32_t i = get_global_id(0); i < mMemory->nLooperMatchCandidates; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < mMemory->nLooperMatchCandidates; i += (nBlocks * nThreads)) { for (uint32_t j = i + 1; j < mMemory->nLooperMatchCandidates; j++) { // int32_t bs = 0; assert(CAMath::Abs(candidates[i].refz) <= CAMath::Abs(candidates[j].refz)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 2a111b8ce89af..ea0620d3ea4c1 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -22,7 +22,7 @@ template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) { const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), nBlocks, nThreads, iBlock, iThread, int32_t, ii, iEnd, { const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : mode ? merger.TrackOrderProcess()[ii] : ii; GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, mode == -1); }); @@ -31,7 +31,7 @@ GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThread template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), uint32_t, i, merger.Memory()->nLoopData, { + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), nBlocks, nThreads, iBlock, iThread, uint32_t, i, merger.Memory()->nLoopData, { GPUTPCGMTrackParam::PropagateLooper(&merger, i); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 5d00451516aa8..d703f1b890443 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -81,7 +81,7 @@ class GPUTPCGMMergerResolve : public GPUTPCGMMergerGeneral }; template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, Args... args); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& clusterer, Args... args); }; class GPUTPCGMMergerClearLinks : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index d7c8eb9c44aab..39690ae078599 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -58,7 +58,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlock GPUTPCGMMerger::tmpSort* GPUrestrict() trackSort = merger.TrackSortO2(); uint2* GPUrestrict() tmpData = merger.ClusRefTmp(); - for (uint32_t i = get_global_id(0); i < nTracks; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < nTracks; i += (nBlocks * nThreads)) { if (!tracks[i].OK()) { continue; } @@ -120,7 +120,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks uint2* GPUrestrict() tmpData = merger.ClusRefTmp(); float const SNPThresh = 0.999990f; - for (int32_t iTmp = get_global_id(0); iTmp < nTracks; iTmp += get_global_size(0)) { + for (int32_t iTmp = (iBlock * nThreads + iThread); iTmp < nTracks; iTmp += (nBlocks * nThreads)) { TrackTPC oTrack; const int32_t i = trackSort[iTmp].x; const auto& track = tracks[i]; @@ -288,7 +288,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks, in auto labelAssigner = GPUTPCTrkLbl(clusters->clustersMCTruth, 0.1f); uint32_t* clusRefs = merger.OutputClusRefsTPCO2(); - for (uint32_t i = get_global_id(0); i < merger.NOutputTracksTPCO2(); i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < merger.NOutputTracksTPCO2(); i += (nBlocks * nThreads)) { labelAssigner.reset(); const auto& trk = merger.OutputTracksTPCO2()[i]; for (int32_t j = 0; j < trk.getNClusters(); j++) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 23842d8a1f859..e418a8c7fecfa 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -49,7 +49,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) { - static constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); const GPUParam& GPUrestrict() param = merger->Param(); @@ -366,8 +366,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) { - static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; @@ -575,7 +575,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) { - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; if (Merger->Param().rec.tpc.disableRefitAttachment & 2) { return dodEdx; } @@ -678,7 +678,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float toX = mX; bool inFlyDirection = (Merger->MergedTracks()[iTrack].Leg() & 1) ^ up; - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = Merger->Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; @@ -744,7 +744,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) { - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; float Y = mP[2] > 0 ? -mX : mX; diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 4ef0f29eabdff..9e86e4627a2fb 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -398,8 +398,8 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov trk.NormalizeAlpha(alpha); prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { - static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; + constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float kSectAngle = 2 * M_PI / 18.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX)) { if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(kSectAngle / 2.f)) { diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx index f99544f239bb7..bc67075b4f820 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefitKernel.cxx @@ -22,7 +22,7 @@ template GPUdii() void GPUTrackingRefitKernel::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() processors) { auto& refit = processors.trackingRefit; - for (uint32_t i = get_global_id(0); i < processors.ioPtrs.nMergedTracks; i += get_global_size(0)) { + for (uint32_t i = (iBlock * nThreads + iThread); i < processors.ioPtrs.nMergedTracks; i += (nBlocks * nThreads)) { if (refit.mPTracks[i].OK()) { GPUTPCGMMergedTrack trk = refit.mPTracks[i]; int32_t retval; diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx index 784b60baec3d6..6280d6e7afaf2 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCExtrapolationTracking.cxx @@ -160,7 +160,7 @@ GPUd() void GPUTPCExtrapolationTracking::PerformExtrapolationTracking(int32_t nB template <> GPUdii() void GPUTPCExtrapolationTracking::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() tracker) { - GPUCA_SHARED_CACHE(&smem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); + GPUCA_SHARED_CACHE(nThreads, iThread, &smem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); GPUbarrier(); if (tracker.NHitsTotal() == 0) { @@ -202,7 +202,7 @@ GPUd() void GPUTPCExtrapolationTracking::ExtrapolationTrackingSectorLeftRight(ui template <> GPUdii() void GPUTPCExtrapolationTrackingCopyNumbers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() tracker, int32_t n) { - for (int32_t i = get_global_id(0); i < n; i += get_global_size(0)) { + for (int32_t i = (iBlock * nThreads + iThread); i < n; i += (nBlocks * nThreads)) { GPUconstantref() GPUTPCTracker& GPUrestrict() trk = (&tracker)[i]; trk.CommonMemory()->nLocalTracks = trk.CommonMemory()->nTracks; trk.CommonMemory()->nLocalTrackHits = trk.CommonMemory()->nTrackHits; diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx index 5151427377b05..eb3038602283e 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCNeighboursFinder.cxx @@ -73,11 +73,11 @@ GPUdii() void GPUTPCNeighboursFinder::Thread<0>(int32_t /*nBlocks*/, int32_t nTh return; } - static constexpr uint32_t UNROLL_GLOBAL = GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL > 1 ? GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL : 1; + constexpr uint32_t UNROLL_GLOBAL = GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL > 1 ? GPUCA_PAR_NEIGHBOURS_FINDER_UNROLL_GLOBAL : 1; static_assert(constants::NEIGHBOURS_MAX_N % UNROLL_GLOBAL == 0); - static constexpr uint32_t MAX_SHARED = GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP; - static constexpr uint32_t MAX_GLOBAL = (MAX_SHARED < constants::NEIGHBOURS_MAX_N) ? (((constants::NEIGHBOURS_MAX_N - MAX_SHARED - 1) / UNROLL_GLOBAL + 1) * UNROLL_GLOBAL) : 0; - static constexpr uint32_t MAX_TOTAL = MAX_SHARED + MAX_GLOBAL; + constexpr uint32_t MAX_SHARED = GPUCA_PAR_NEIGHBOURS_FINDER_MAX_NNEIGHUP; + constexpr uint32_t MAX_GLOBAL = (MAX_SHARED < constants::NEIGHBOURS_MAX_N) ? (((constants::NEIGHBOURS_MAX_N - MAX_SHARED - 1) / UNROLL_GLOBAL + 1) * UNROLL_GLOBAL) : 0; + constexpr uint32_t MAX_TOTAL = MAX_SHARED + MAX_GLOBAL; const float chi2Cut = 3.f * 3.f * 4 * (s.mUpDx * s.mUpDx + s.mDnDx * s.mDnDx); // float chi2Cut = 3.f*3.f*(s.mUpDx*s.mUpDx + s.mDnDx*s.mDnDx ); //SG diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx index 6ce031882caec..28a1cff148df6 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx @@ -304,10 +304,16 @@ GPUd() bool GPUTPCTrackParam::TransportToXWithMaterial(float x, GPUTPCTrackLinea { //* Transport the track parameters to X=x taking into account material budget +#ifdef __METAL__ // MSL rejects variables declared static at function scope + constexpr float kRho = 1.025e-3f; // [g/cm^3] + constexpr float kRadLen = 28811.7f; //[cm] + constexpr float kRadLenInv = 1.f / kRadLen; +#else static constexpr float kRho = 1.025e-3f; // [g/cm^3] static constexpr float kRadLen = 28811.7f; //[cm] static constexpr float kRadLenInv = 1.f / kRadLen; +#endif float dl; if (!TransportToX(x, t0, Bz, maxSinPhi, &dl)) { diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 33a3264a87ab3..ba31c30840631 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -479,14 +479,14 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::DoTracklet(GPUconstantref() GPUTPCT template <> GPUdii() void GPUTPCTrackletConstructor::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& sMem, processorType& GPUrestrict() tracker) { - if (get_local_id(0) == 0) { + if (iThread == 0) { sMem.mNStartHits = *tracker.NStartHits(); } - GPUCA_SHARED_CACHE(&sMem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); + GPUCA_SHARED_CACHE(nThreads, iThread, &sMem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); GPUbarrier(); GPUTPCThreadMemory rMem; - for (rMem.mISH = get_global_id(0); rMem.mISH < sMem.mNStartHits; rMem.mISH += get_global_size(0)) { + for (rMem.mISH = (iBlock * nThreads + iThread); rMem.mISH < sMem.mNStartHits; rMem.mISH += (nBlocks * nThreads)) { rMem.mGo = 1; DoTracklet(tracker, sMem, rMem); } diff --git a/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h b/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h index 3f1265e6d0634..64aa0f6fcfe6d 100644 --- a/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h +++ b/GPU/GPUTracking/TPCClusterFinder/CfChargePos.h @@ -34,6 +34,14 @@ struct CfChargePos { : gpad(tpcGlobalPadIdx(row, pad)), timePadded(t + GPUCF_PADDING_TIME) { } +#ifdef __METAL__ + // INVALID_CHARGE_POS below lives in the constant address space, which a + // generic `this` does not reach in MSL. + constexpr GPUhdi() CfChargePos(tpccf::Row row, tpccf::Pad pad, tpccf::TPCFragmentTime t) constant + : gpad(tpcGlobalPadIdx(row, pad)), timePadded(t + GPUCF_PADDING_TIME) + { + } +#endif GPUdi() CfChargePos(const tpccf::GlobalPad& p, const tpccf::TPCFragmentTime& t) : gpad(p), timePadded(t) {} diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx index 752c85634f928..95f78bd9890ee 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChargeMapFiller.cxx @@ -24,22 +24,22 @@ template <> GPUdii() void GPUTPCCFChargeMapFiller::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D indexMap(clusterer.mPindexMap); - fillIndexMapImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer.mPmemory->fragment, clusterer.mPdigits, indexMap, clusterer.mPmemory->counters.nDigitsInFragment); + fillIndexMapImpl(nBlocks, nThreads, iBlock, iThread, clusterer.mPmemory->frag, clusterer.mPdigits, indexMap, clusterer.mPmemory->counters.nDigitsInFragment); } GPUd() void GPUTPCCFChargeMapFiller::fillIndexMapImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, - const CfFragment& fragment, + const CfFragment& frag, const tpc::Digit* digits, CfArray2D& indexMap, size_t maxDigit) { - size_t idx = get_global_id(0); + size_t idx = (iBlock * nThreads + iThread); if (idx >= maxDigit) { return; } - CPU_ONLY(idx += fragment.digitsStart); + CPU_ONLY(idx += frag.digitsStart); CPU_ONLY(tpc::Digit digit = digits[idx]); - CPU_ONLY(CfChargePos pos(digit.getRow(), digit.getPad(), fragment.toLocal(digit.getTimeStamp()))); + CPU_ONLY(CfChargePos pos(digit.getRow(), digit.getPad(), frag.toLocal(digit.getTimeStamp()))); CPU_ONLY(indexMap.safeWrite(pos, idx)); } @@ -47,21 +47,21 @@ template <> GPUdii() void GPUTPCCFChargeMapFiller::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); - fillFromDigitsImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer, clusterer.mPmemory->fragment, clusterer.mPmemory->counters.nPositions, clusterer.mPdigits, clusterer.mPpositions, chargeMap); + fillFromDigitsImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->frag, clusterer.mPmemory->counters.nPositions, clusterer.mPdigits, clusterer.mPpositions, chargeMap); } -GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, const CfFragment& fragment, size_t digitNum, +GPUd() void GPUTPCCFChargeMapFiller::fillFromDigitsImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, const CfFragment& frag, size_t digitNum, const tpc::Digit* digits, CfChargePos* positions, CfArray2D& chargeMap) { - size_t idx = get_global_id(0); + size_t idx = (iBlock * nThreads + iThread); if (idx >= digitNum) { return; } - tpc::Digit digit = digits[fragment.digitsStart + idx]; + tpc::Digit digit = digits[frag.digitsStart + idx]; - CfChargePos pos(digit.getRow(), digit.getPad(), fragment.toLocal(digit.getTimeStamp())); + CfChargePos pos(digit.getRow(), digit.getPad(), frag.toLocal(digit.getTimeStamp())); positions[idx] = pos; float q = digit.getChargeFloat(); q *= clusterer.GetConstantMem()->calibObjects.tpcPadGain->getGainCorrection(clusterer.mISector, digit.getRow(), digit.getPad()); @@ -77,10 +77,10 @@ GPUdii() void GPUTPCCFChargeMapFiller::Threadcounters.nDigits; const tpc::Digit* digits = clusterer.mPdigits; - size_t st = findTransition(clusterer.mPmemory->fragment.first(), digits, nDigits, 0); - size_t end = findTransition(clusterer.mPmemory->fragment.last(), digits, nDigits, st); + size_t st = findTransition(clusterer.mPmemory->frag.first(), digits, nDigits, 0); + size_t end = findTransition(clusterer.mPmemory->frag.last(), digits, nDigits, st); - clusterer.mPmemory->fragment.digitsStart = st; + clusterer.mPmemory->frag.digitsStart = st; size_t elems = end - st; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx index 7470f86877490..c1a459d138dbd 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx @@ -224,7 +224,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPU(int32_t nBlocks, int32_t return; } - const CfFragment& fragment = clusterer.mPmemory->fragment; + const CfFragment& frag = clusterer.mPmemory->frag; const bool hipFilterOn = clusterer.Param().rec.tpc.hipTailFilter; const Charge hipTailThreshold = clusterer.Param().rec.tpc.hipTailFilterThreshold; const Charge hipTailFilterAlpha = clusterer.Param().rec.tpc.hipTailFilterAlpha; @@ -253,7 +253,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPU(int32_t nBlocks, int32_t // saturated signal in overlap region can create tails in the next fragment // even when cleared in current fragment as they're decoded twice const TPCFragmentTime firstTB = 0; - const TPCFragmentTime lastTB = fragment.length; + const TPCFragmentTime lastTB = frag.length; for (uint16_t t = firstTB; t < lastTB; t += NumOfCachedTBs) { @@ -384,7 +384,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t const int32_t nPads = geo.NPads(row); const int32_t nVecPads = (nPads + PadsPerCacheline - 1) / PadsPerCacheline; - const CfFragment& fragment = clusterer.mPmemory->fragment; + const CfFragment& frag = clusterer.mPmemory->frag; const bool hipFilterOn = clusterer.Param().rec.tpc.hipTailFilter; const Charge hipTailThreshold = clusterer.Param().rec.tpc.hipTailFilterThreshold; const Charge hipTailFilterAlpha = clusterer.Param().rec.tpc.hipTailFilterAlpha; @@ -409,7 +409,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t std::vector activeHIPTailEndV(nVecPads, -1); std::vector tailFilterChargeV(nVecPads, Charge8{Vc::Zero}); - for (int16_t t = 0; t < fragment.length; t += NumOfCachedTBs) { + for (int16_t t = 0; t < frag.length; t += NumOfCachedTBs) { bool hasAnyTrigger = false; @@ -432,7 +432,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t for (tpccf::TPCFragmentTime localtime = 0; localtime < NumOfCachedTBs; localtime++) { const uint16_t* packedChargeStart = reinterpret_cast(&chargeMap[basePos.delta({0, localtime})]); - const UShort8 packedCharges = t + localtime < fragment.length + const UShort8 packedCharges = t + localtime < frag.length ? UShort8{packedChargeStart, Vc::Aligned} : UShort8{Vc::Zero}; const auto isCharge = packedCharges != 0; @@ -585,7 +585,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t auto activeHIPTailEnd = activeHIPTailEndV[iVecPad]; const auto shouldCloseTail = activeHIPTailStart > -1; - activeHIPTailEnd(shouldCloseTail && activeHIPTailEnd < 0) = fragment.length; + activeHIPTailEnd(shouldCloseTail && activeHIPTailEnd < 0) = frag.length; if (hipFilterOn && shouldCloseTail.isNotEmpty()) { for (int16_t p = 0; p < PadsPerCacheline; p++) { @@ -639,8 +639,8 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t GPUd() void GPUTPCCFCheckPadBaseline::updatePadBaseline(int32_t pad, const GPUTPCClusterFinder& clusterer, int32_t totalCharges, int32_t consecCharges, Charge maxCharge) { - const CfFragment& fragment = clusterer.mPmemory->fragment; - const int32_t totalChargesBaseline = clusterer.Param().rec.tpc.maxTimeBinAboveThresholdIn1000Bin * fragment.lengthWithoutOverlap() / 1000; + const CfFragment& frag = clusterer.mPmemory->frag; + const int32_t totalChargesBaseline = clusterer.Param().rec.tpc.maxTimeBinAboveThresholdIn1000Bin * frag.lengthWithoutOverlap() / 1000; const int32_t consecChargesBaseline = clusterer.Param().rec.tpc.maxConsecTimeBinAboveThreshold; const uint16_t saturationThreshold = clusterer.Param().rec.tpc.noisyPadSaturationThreshold; const bool isNoisy = (!saturationThreshold || maxCharge < saturationThreshold) && ((totalChargesBaseline > 0 && totalCharges >= totalChargesBaseline) || (consecChargesBaseline > 0 && consecCharges >= consecChargesBaseline)); @@ -668,7 +668,7 @@ GPUd() void GPUTPCCFHIPTailConnector::Thread<0>(int32_t nBlocks, int32_t nThread #ifdef GPUCA_DETERMINISTIC_MODE // Races in tail comparisons and atomic swap can lead to slightly different clusters. // So need a sequential fallback for deterministic mode - GPUCommonAlgorithm::sortInBlock(tails + 1, tails + nTails + 1, [](auto&& t1, auto&& t2) { + GPUCommonAlgorithm::sortInBlock(nThreads, iThread, tails + 1, tails + nTails + 1, [](auto&& t1, auto&& t2) { if (t1.pad != t2.pad) { return t1.pad < t2.pad; } else if (t1.tailStart != t2.tailStart) { @@ -728,7 +728,7 @@ GPUd() void GPUTPCCFHIPClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, nTails = CAMath::Min(nTails, (uint32_t)MaxHIPTailsPerRow - 1); const auto* tails = GetHIPTails(clusterer, row); - const auto& fragment = clusterer.mPmemory->fragment; + const auto& frag = clusterer.mPmemory->frag; auto* clusterPosInRow = clusterer.mPhipClusterPosInRow ? clusterer.mPhipClusterPosInRow + row * MaxHIPTailsPerRow @@ -776,7 +776,7 @@ GPUd() void GPUTPCCFHIPClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, cn.qMax = qMax; cn.setSaturatedQtot(qTot); cn.setSaturatedTailLength(tailEnd - tailStart); - float clusterTime = fragment.start + timeMean - clusterer.Param().rec.tpc.clustersShiftTimebinsClusterizer; + float clusterTime = frag.start + timeMean - clusterer.Param().rec.tpc.clustersShiftTimebinsClusterizer; cn.setTimeFlags(clusterTime, 0); cn.setPad(padMean); cn.setSigmaPad(padSigma); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx index c9a8c093153a2..37db02d1b8559 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.cxx @@ -35,5 +35,5 @@ GPUdii() void GPUTPCCFClusterizer::Thread<0>(int32_t nBlocks, int32_t nThreads, tpc::ClusterNative* clusterOut = onlyMC ? nullptr : clusterer.mPclusterByRow; - GPUTPCCFClusterizer::computeClustersImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, true); + GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->frag, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, true); } diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h index ce673c778e42d..ab40e1c3bc2c3 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.h @@ -59,7 +59,7 @@ class GPUTPCCFClusterizer : public GPUKernelTemplate static GPUd() void computeClustersImpl(int32_t, int32_t, int32_t, int32_t, processorType&, const CfFragment&, GPUSharedMemory&, const CfArray2D&, const CfChargePos*, const GPUSettingsRec&, MCLabelAccumulator*, uint32_t, uint32_t, uint32_t*, tpc::ClusterNative*, uint32_t*, int8_t); - static GPUd() void buildCluster(const GPUSettingsRec&, const CfArray2D&, CfChargePos, CfChargePos*, PackedCharge*, uint8_t*, ClusterAccumulator*, MCLabelAccumulator*); + static GPUd() void buildCluster(const GPUSettingsRec&, uint16_t, const CfArray2D&, CfChargePos, CfChargePos*, PackedCharge*, uint8_t*, ClusterAccumulator*, MCLabelAccumulator*); static GPUd() uint32_t sortIntoBuckets(processorType&, const tpc::ClusterNative&, uint32_t, uint32_t, uint32_t*, tpc::ClusterNative*); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc index ca396f8aab83e..f21f9b47480af 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFClusterizer.inc @@ -17,7 +17,7 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, processorType& clusterer, - const CfFragment& fragment, + const CfFragment& frag, GPUSharedMemory& smem, const CfArray2D& chargeMap, const CfChargePos* filteredPeakPositions, @@ -30,7 +30,7 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t uint32_t* clusterPosInRow, int8_t isAccepted) { - uint32_t idx = get_global_id(0); + uint32_t idx = (iBlock * nThreads + iThread); // For certain configurations dummy work items are added, so the total // number of work items is dividable by 64. @@ -43,6 +43,7 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t buildCluster( calib, + iThread, chargeMap, pos, smem.posBcast, @@ -54,14 +55,14 @@ GPUdii() void GPUTPCCFClusterizer::computeClustersImpl(int32_t nBlocks, int32_t if (idx >= clusternum) { return; } - if (fragment.isOverlap(pos.time())) { + if (frag.isOverlap(pos.time())) { if (clusterPosInRow) { clusterPosInRow[idx] = maxClusterPerRow; } return; } tpc::ClusterNative myCluster; - pc.finalize(pos, charge, fragment.start); + pc.finalize(pos, charge, frag.start); bool rejectCluster = !pc.toNative(pos, charge, myCluster, clusterer.Param(), chargeMap); if (!isAccepted) { rejectCluster = true; @@ -145,6 +146,7 @@ GPUdii() void GPUTPCCFClusterizer::updateClusterOuter( GPUdii() void GPUTPCCFClusterizer::buildCluster( const GPUSettingsRec& calib, + uint16_t ll, const CfArray2D& chargeMap, CfChargePos pos, CfChargePos* posBcast, @@ -153,8 +155,6 @@ GPUdii() void GPUTPCCFClusterizer::buildCluster( ClusterAccumulator* myCluster, MCLabelAccumulator* labelAcc) { - uint16_t ll = get_local_id(0); - posBcast[ll] = pos; GPUbarrier(); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index a1c4a3dc4aadd..0f2d30362b4f0 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -37,12 +37,12 @@ using namespace o2::tpc::constants; // =========================================================================== template <> -GPUdii() void GPUTPCCFDecodeZS::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, int32_t firstHBF, int32_t tpcTimeBinCut) +GPUdii() void GPUTPCCFDecodeZS::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& clusterer, int32_t firstHBF, int32_t tpcTimeBinCut) { GPUTPCCFDecodeZS::decode(clusterer, smem, nBlocks, nThreads, iBlock, iThread, firstHBF, tpcTimeBinCut); } -GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut) +GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUsharedref() GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut) { const uint32_t sector = clusterer.mISector; #ifdef GPUCA_GPUCODE @@ -80,7 +80,7 @@ GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUShared for (uint32_t j = minJ; j < maxJ; j++) { #endif const uint32_t* pageSrc = (const uint32_t*)(((const uint8_t*)zs.zsPtr[endpoint][i]) + j * TPCZSHDR::TPC_ZS_PAGE_SIZE); - GPUCA_SHARED_CACHE_REF(&s.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); + GPUCA_SHARED_CACHE_REF(nThreads, iThread, &s.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); GPUbarrier(); const uint8_t* page = (const uint8_t*)pageCache; const o2::header::RAWDataHeader* rdh = (const o2::header::RAWDataHeader*)page; @@ -172,13 +172,13 @@ GPUdii() void GPUTPCCFDecodeZS::decode(GPUTPCClusterFinder& clusterer, GPUShared seqLen = rowData[(nSeq + 1) * 2] - rowData[nSeq * 2]; pad = rowData[nSeq++ * 2 + 1]; } - const CfFragment& fragment = clusterer.mPmemory->fragment; + const CfFragment& frag = clusterer.mPmemory->frag; TPCTime globalTime = timeBin + l; - bool discardTimeBin = not fragment.contains(globalTime); + bool discardTimeBin = not frag.contains(globalTime); discardTimeBin |= (tpcTimeBinCut > 0 && globalTime > tpcTimeBinCut); Row row = rowOffset + m; - CfChargePos pos(row, Pad(pad), discardTimeBin ? INVALID_TIME_BIN : fragment.toLocal(globalTime)); + CfChargePos pos(row, Pad(pad), discardTimeBin ? INVALID_TIME_BIN : frag.toLocal(globalTime)); positions[nDigitsTmp++] = pos; if (!discardTimeBin) { @@ -219,7 +219,7 @@ GPUdii() void GPUTPCCFDecodeZSLink::Thread<0>(int32_t nBlocks, int32_t nThreads, GPUd() size_t GPUTPCCFDecodeZSLink::DecodePage(GPUSharedMemory& smem, DecodeCtx& ctx) { - const CfFragment& fragment = ctx.clusterer.mPmemory->fragment; + const CfFragment& frag = ctx.clusterer.mPmemory->frag; const auto* rdHdr = ConsumeHeader(ctx.page); @@ -246,7 +246,7 @@ GPUd() size_t GPUTPCCFDecodeZSLink::DecodePage(GPUSharedMemory& smem, DecodeCtx& nDecoded += nAdc; - bool discardTimeBin = not fragment.contains(timeBin); + bool discardTimeBin = not frag.contains(timeBin); discardTimeBin |= (ctx.tpcTimeBinCut > 0 && timeBin > ctx.tpcTimeBinCut); if (discardTimeBin) { @@ -331,9 +331,9 @@ GPUd() void GPUTPCCFDecodeZSLink::DecodeTB( } o2::tpc::PadPos padAndRow = GetPadAndRowFromFEC(ctx.clusterer, cru, rawFECChannel, fecInPartition); - const CfFragment& fragment = ctx.clusterer.mPmemory->fragment; + const CfFragment& frag = ctx.clusterer.mPmemory->frag; float charge = ADCToFloat(adc, DECODE_MASK, DECODE_BITS_FACTOR); - WriteCharge(ctx.clusterer, charge, padAndRow, fragment.toLocal(timeBin), ctx.pageDigitOffset + myOffset); + WriteCharge(ctx.clusterer, charge, padAndRow, frag.toLocal(timeBin), ctx.pageDigitOffset + myOffset); } // for (uint8_t i = iThread; blockOffset < nAdc; i += NThreads) } @@ -393,7 +393,7 @@ GPUd() void GPUTPCCFDecodeZSLinkBase::Decode(int32_t nBlocks, int32_t nThreads, #endif const uint32_t* pageSrc = (const uint32_t*)(((const uint8_t*)zs.zsPtr[endpoint][i]) + j * TPCZSHDR::TPC_ZS_PAGE_SIZE); // Cache zs page in shared memory. Curiously this actually degrades performance... - // GPUCA_SHARED_CACHE_REF(&smem.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); + // GPUCA_SHARED_CACHE_REF(nThreads, iThread, &smem.ZSPage[0], pageSrc, TPCZSHDR::TPC_ZS_PAGE_SIZE, uint32_t, pageCache); // GPUbarrier(); // const uint8_t* page = (const uint8_t*)pageCache; const uint8_t* page = (const uint8_t*)pageSrc; @@ -651,7 +651,7 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( constexpr int32_t NTHREADS = GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCCFDecodeZSDenseLink); static_assert(NTHREADS == GPUCA_WARP_SIZE, "Decoding TB Headers in parallel assumes block size is a single warp."); - const CfFragment& fragment = ctx.clusterer.mPmemory->fragment; + const CfFragment& frag = ctx.clusterer.mPmemory->frag; // Read timebin block header uint16_t tbbHdr = ConsumeByte(ctx.page); @@ -721,7 +721,7 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( const uint8_t* adcData = ConsumeBytes(ctx.page, (nSamplesInTB * DECODE_BITS + 7) / 8); MAYBE_PAGE_OVERFLOW(ctx.page); - bool discardTimeBin = not fragment.contains(timeBin); + bool discardTimeBin = not frag.contains(timeBin); discardTimeBin |= (ctx.tpcTimeBinCut > 0 && timeBin > ctx.tpcTimeBinCut); if (discardTimeBin) { @@ -754,7 +754,7 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( o2::tpc::PadPos padAndRow = GetPadAndRowFromFEC(ctx.clusterer, cru, rawFECChannelLink, smem.linkIds[iLink]); float charge = ADCToFloat(adc, DECODE_MASK, DECODE_BITS_FACTOR); - WriteCharge(ctx.clusterer, charge, padAndRow, fragment.toLocal(timeBin), ctx.pageDigitOffset + sample); + WriteCharge(ctx.clusterer, charge, padAndRow, frag.toLocal(timeBin), ctx.pageDigitOffset + sample); } // for (uint16_t sample = iThread; sample < nSamplesInTB; sample += NTHREADS) diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h index 21d4ec0a28958..eed5e0d6f30ac 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h @@ -45,7 +45,7 @@ class GPUTPCCFDecodeZS : public GPUKernelTemplate decodeZS, }; - static GPUd() void decode(GPUTPCClusterFinder& clusterer, GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut); + static GPUd() void decode(GPUTPCClusterFinder& clusterer, GPUsharedref() GPUSharedMemory& s, int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t firstHBF, int32_t tpcTimeBinCut); typedef GPUTPCClusterFinder processorType; GPUhdi() static processorType* Processor(GPUConstantMem& processors) @@ -59,7 +59,7 @@ class GPUTPCCFDecodeZS : public GPUKernelTemplate } template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, Args... args); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& clusterer, Args... args); }; class GPUTPCCFDecodeZSLinkBase : public GPUKernelTemplate diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx index d6b8703a9b35d..b234094648eee 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx @@ -26,7 +26,7 @@ GPUdii() void GPUTPCCFDeconvolution::Thread<0>(int32_t nBlocks, int32_t nThreads { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CfArray2D isPeakMap(clusterer.mPpeakMap); - GPUTPCCFDeconvolution::deconvolutionImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), smem, isPeakMap, chargeMap, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, overwriteCharge); + GPUTPCCFDeconvolution::deconvolutionImpl(nBlocks, nThreads, iBlock, iThread, smem, isPeakMap, chargeMap, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, overwriteCharge); } GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, @@ -36,7 +36,7 @@ GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t const uint32_t digitnum, uint8_t overwriteCharge) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); bool iamDummy = (idx >= digitnum); idx = iamDummy ? digitnum - 1 : idx; @@ -47,7 +47,7 @@ GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t int8_t peakCount = (iamPeak) ? 1 : 0; - uint16_t ll = get_local_id(0); + uint16_t ll = iThread; uint16_t partId = ll; uint16_t in3x3 = 0; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx index 8b4f28f517782..a9e4fdbbca064 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFMCLabelFlattener.cxx @@ -46,7 +46,7 @@ template <> GPUd() void GPUTPCCFMCLabelFlattener::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory&, processorType& clusterer) { #if !defined(GPUCA_GPUCODE) - const Row row = get_global_id(0); + const Row row = (iBlock * nThreads + iThread); const size_t clusterInRow = clusterer.mPclusterInRow[row]; auto& labels = clusterer.mPlabelsByRow[row].data; @@ -66,7 +66,7 @@ template <> GPUd() void GPUTPCCFMCLabelFlattener::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory&, processorType& clusterer, GPUTPCLinearLabels* out) { #if !defined(GPUCA_GPUCODE) - uint32_t row = get_global_id(0); + uint32_t row = (iBlock * nThreads + iThread); uint32_t headerOffset = clusterer.mPlabelsHeaderGlobalOffset; uint32_t dataOffset = clusterer.mPlabelsDataGlobalOffset; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx index 4dfa50d9439e4..336cfb6d801a9 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.cxx @@ -26,14 +26,14 @@ GPUdii() void GPUTPCCFNoiseSuppression::Thread chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CfArray2D isPeakMap(clusterer.mPpeakMap); - noiseSuppressionImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), smem, clusterer.Param().rec, chargeMap, isPeakMap, clusterer.mPpeakPositions, clusterer.mPmemory->counters.nPeaks, clusterer.mPisPeak); + noiseSuppressionImpl(nBlocks, nThreads, iBlock, iThread, smem, clusterer.Param().rec, chargeMap, isPeakMap, clusterer.mPpeakPositions, clusterer.mPmemory->counters.nPeaks, clusterer.mPisPeak); } template <> GPUdii() void GPUTPCCFNoiseSuppression::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { CfArray2D isPeakMap(clusterer.mPpeakMap); - updatePeaksImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer.mPpeakPositions, clusterer.mPisPeak, clusterer.mPmemory->counters.nPeaks, isPeakMap); + updatePeaksImpl(nBlocks, nThreads, iBlock, iThread, clusterer.mPpeakPositions, clusterer.mPisPeak, clusterer.mPmemory->counters.nPeaks, isPeakMap); } GPUdii() void GPUTPCCFNoiseSuppression::noiseSuppressionImpl(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, @@ -44,7 +44,7 @@ GPUdii() void GPUTPCCFNoiseSuppression::noiseSuppressionImpl(int32_t nBlocks, in const uint32_t peaknum, uint8_t* isPeakPredicate) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); CfChargePos pos = peakPositions[CAMath::Min(idx, (SizeT)(peaknum - 1))]; Charge charge = chargeMap[pos].unpack(); @@ -54,6 +54,7 @@ GPUdii() void GPUTPCCFNoiseSuppression::noiseSuppressionImpl(int32_t nBlocks, in chargeMap, peakMap, calibration, + iThread, charge, pos, smem.posBcast, @@ -80,7 +81,7 @@ GPUd() void GPUTPCCFNoiseSuppression::updatePeaksImpl(int32_t nBlocks, int32_t n const uint32_t peakNum, CfArray2D& peakMap) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); if (idx >= peakNum) { return; @@ -167,6 +168,7 @@ GPUd() void GPUTPCCFNoiseSuppression::findMinimaAndPeaks( const CfArray2D& chargeMap, const CfArray2D& peakMap, const GPUSettingsRec& calibration, + uint16_t ll, float q, const CfChargePos& pos, CfChargePos* posBcast, @@ -175,8 +177,6 @@ GPUd() void GPUTPCCFNoiseSuppression::findMinimaAndPeaks( uint64_t* bigger, uint64_t* peaks) { - uint16_t ll = get_local_id(0); - posBcast[ll] = pos; GPUbarrier(); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h index bdee75dc87732..8c251f310a1a8 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFNoiseSuppression.h @@ -69,7 +69,7 @@ class GPUTPCCFNoiseSuppression : public GPUKernelTemplate static GPUdi() bool keepPeak(uint64_t, uint64_t); - static GPUd() void findMinimaAndPeaks(const CfArray2D&, const CfArray2D&, const GPUSettingsRec&, float, const CfChargePos&, CfChargePos*, PackedCharge*, uint64_t*, uint64_t*, uint64_t*); + static GPUd() void findMinimaAndPeaks(const CfArray2D&, const CfArray2D&, const GPUSettingsRec&, uint16_t, float, const CfChargePos&, CfChargePos*, PackedCharge*, uint64_t*, uint64_t*, uint64_t*); }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx index 7c93435f8bef8..ff71af8838b83 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx @@ -27,11 +27,12 @@ GPUdii() void GPUTPCCFPeakFinder::Thread<0>(int32_t nBlocks, int32_t nThreads, i { CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CfArray2D isPeakMap(clusterer.mPpeakMap); - findPeaksImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), smem, chargeMap, clusterer.mPpadIsNoisy, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, clusterer.Param().rec, *clusterer.GetConstantMem()->calibObjects.tpcPadGain, clusterer.mPisPeak, isPeakMap); + findPeaksImpl(nBlocks, nThreads, iBlock, iThread, smem, chargeMap, clusterer.mPpadIsNoisy, clusterer.mPpositions, clusterer.mPmemory->counters.nPositions, clusterer.Param().rec, *clusterer.GetConstantMem()->calibObjects.tpcPadGain, clusterer.mPisPeak, isPeakMap); } GPUdii() bool GPUTPCCFPeakFinder::isPeak( GPUSharedMemory& smem, + uint16_t ll, Charge q, const CfChargePos& pos, uint16_t N, @@ -40,8 +41,6 @@ GPUdii() bool GPUTPCCFPeakFinder::isPeak( CfChargePos* posBcast, PackedCharge* buf) { - uint16_t ll = get_local_id(0); - bool belowThreshold = (uint32_t)q <= calib.tpc.cfQMaxCutoff; uint16_t lookForPeaks; @@ -100,7 +99,7 @@ GPUd() void GPUTPCCFPeakFinder::findPeaksImpl(int32_t nBlocks, int32_t nThreads, uint8_t* isPeakPredicate, CfArray2D& peakMap) { - SizeT idx = get_global_id(0); + SizeT idx = (iBlock * nThreads + iThread); // For certain configurations dummy work items are added, so the total // number of work items is dividable by 64. @@ -111,7 +110,7 @@ GPUd() void GPUTPCCFPeakFinder::findPeaksImpl(int32_t nBlocks, int32_t nThreads, bool hasLostBaseline = pos.valid() ? padHasLostBaseline[pos.gpad] : true; charge = hasLostBaseline ? 0.f : charge; - uint8_t peak = isPeak(smem, charge, pos, SCRATCH_PAD_SEARCH_N, chargeMap, calib, smem.posBcast, smem.buf); + uint8_t peak = isPeak(smem, iThread, charge, pos, SCRATCH_PAD_SEARCH_N, chargeMap, calib, smem.posBcast, smem.buf); // Exit early if dummy. See comment above. bool iamDummy = (idx >= digitnum); diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h index 0d61378d3e6f2..8cf32c1589e61 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.h @@ -53,7 +53,7 @@ class GPUTPCCFPeakFinder : public GPUKernelTemplate private: static GPUd() void findPeaksImpl(int32_t, int32_t, int32_t, int32_t, GPUSharedMemory&, const CfArray2D&, const uint8_t*, const CfChargePos*, tpccf::SizeT, const GPUSettingsRec&, const TPCPadGainCalib&, uint8_t*, CfArray2D&); - static GPUd() bool isPeak(GPUSharedMemory&, tpccf::Charge, const CfChargePos&, uint16_t, const CfArray2D&, const GPUSettingsRec&, CfChargePos*, PackedCharge*); + static GPUd() bool isPeak(GPUSharedMemory&, uint16_t, tpccf::Charge, const CfChargePos&, uint16_t, const CfArray2D&, const GPUSettingsRec&, CfChargePos*, PackedCharge*); }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx index 0f2fd235dc0d0..4622638674d9a 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx @@ -30,7 +30,7 @@ GPUdii() void GPUTPCCFStreamCompaction::Thread auto* scanOffset = clusterer.GetScanBuffer(iBuf - 1); auto* scanOffsetNext = clusterer.GetScanBuffer(iBuf); - int32_t iThreadGlobal = get_global_id(0); + int32_t iThreadGlobal = (iBlock * nThreads + iThread); int32_t offsetInBlock = work_group_scan_inclusive_add((iThreadGlobal < nElems) ? scanOffset[iThreadGlobal] : 0); if (iThreadGlobal < nElems) { @@ -70,7 +70,7 @@ template <> GPUdii() void GPUTPCCFStreamCompaction::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer, int32_t iBuf, int32_t nElems) { #ifdef GPUCA_GPUCODE - int32_t iThreadGlobal = get_global_id(0); + int32_t iThreadGlobal = (iBlock * nThreads + iThread); int32_t* scanOffset = clusterer.GetScanBuffer(iBuf - 1); bool inBounds = (iThreadGlobal < nElems); @@ -87,7 +87,7 @@ template <> GPUdii() void GPUTPCCFStreamCompaction::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& /*smem*/, processorType& clusterer, int32_t iBuf, uint32_t offset, int32_t nElems) { #ifdef GPUCA_GPUCODE - int32_t iThreadGlobal = get_global_id(0) + offset; + int32_t iThreadGlobal = (iBlock * nThreads + iThread) + offset; int32_t* scanOffsetPrev = clusterer.GetScanBuffer(iBuf - 1); const int32_t* scanOffset = clusterer.GetScanBuffer(iBuf); @@ -107,7 +107,7 @@ GPUdii() void GPUTPCCFStreamCompaction::Thread bufferSize) { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h index d169440a8d972..4a790750c773b 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinder.h @@ -63,7 +63,7 @@ class GPUTPCClusterFinder : public GPUProcessor uint32_t maxTimeBin = 0; uint32_t nPagesSubsector = 0; } counters; - CfFragment fragment; + CfFragment frag; }; struct ZSOffset { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx index 3b06db8efc1a3..d778dce37c4af 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCClusterFinderDump.cxx @@ -25,7 +25,7 @@ void GPUTPCClusterFinder::DumpDigits(std::ostream& out) { const auto nPositions = mPmemory->counters.nPositions; - out << "\nClusterer - Digits - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << ": " << nPositions << "\n"; + out << "\nClusterer - Digits - Sector " << mISector << " - Fragment " << mPmemory->frag.index << ": " << nPositions << "\n"; out << std::hex; for (size_t i = 0; i < mPmemory->counters.nPositions; i++) { @@ -37,7 +37,7 @@ void GPUTPCClusterFinder::DumpDigits(std::ostream& out) void GPUTPCClusterFinder::DumpChargeMap(std::ostream& out, std::string_view title) { - out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << "\n"; + out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->frag.index << "\n"; CfArray2D map(mPchargeMap); out << std::hex; @@ -70,7 +70,7 @@ void GPUTPCClusterFinder::DumpChargeMap(std::ostream& out, std::string_view titl void GPUTPCClusterFinder::DumpPeakMap(std::ostream& out, std::string_view title) { - out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << "\n"; + out << "\nClusterer - " << title << " - Sector " << mISector << " - Fragment " << mPmemory->frag.index << "\n"; CfArray2D map(mPpeakMap); @@ -106,7 +106,7 @@ void GPUTPCClusterFinder::DumpPeakMap(std::ostream& out, std::string_view title) void GPUTPCClusterFinder::DumpPeaks(std::ostream& out) { - out << "\nClusterer - Peaks - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << "\n"; + out << "\nClusterer - Peaks - Sector " << mISector << " - Fragment " << mPmemory->frag.index << "\n"; for (uint32_t i = 0; i < mPmemory->counters.nPositions; i++) { out << int32_t{mPisPeak[i]}; if ((i + 1) % 100 == 0) { @@ -119,7 +119,7 @@ void GPUTPCClusterFinder::DumpPeaksCompacted(std::ostream& out) { const auto nPeaks = mPmemory->counters.nPeaks; - out << "\nClusterer - Compacted Peaks - Sector " << mISector << " - Fragment " << mPmemory->fragment.index << ": " << nPeaks << "\n"; + out << "\nClusterer - Compacted Peaks - Sector " << mISector << " - Fragment " << mPmemory->frag.index << ": " << nPeaks << "\n"; for (size_t i = 0; i < nPeaks; i++) { const auto& pos = mPpeakPositions[i]; out << pos.time() << " " << int32_t{pos.pad()} << " " << int32_t{pos.row()} << "\n"; @@ -128,10 +128,10 @@ void GPUTPCClusterFinder::DumpPeaksCompacted(std::ostream& out) void GPUTPCClusterFinder::DumpSuppressedPeaks(std::ostream& out) { - const auto& fragment = mPmemory->fragment; + const auto& frag = mPmemory->frag; const auto nPeaks = mPmemory->counters.nPeaks; - out << "\nClusterer - NoiseSuppression - Sector " << mISector << " - Fragment " << fragment.index << mISector << "\n"; + out << "\nClusterer - NoiseSuppression - Sector " << mISector << " - Fragment " << frag.index << mISector << "\n"; for (uint32_t i = 0; i < nPeaks; i++) { out << int32_t{mPisPeak[i]}; if ((i + 1) % 100 == 0) { @@ -142,10 +142,10 @@ void GPUTPCClusterFinder::DumpSuppressedPeaks(std::ostream& out) void GPUTPCClusterFinder::DumpSuppressedPeaksCompacted(std::ostream& out) { - const auto& fragment = mPmemory->fragment; + const auto& frag = mPmemory->frag; const auto nPeaks = mPmemory->counters.nClusters; - out << "\nClusterer - Noise Suppression Peaks Compacted - Sector " << mISector << " - Fragment " << fragment.index << ": " << nPeaks << "\n"; + out << "\nClusterer - Noise Suppression Peaks Compacted - Sector " << mISector << " - Fragment " << frag.index << ": " << nPeaks << "\n"; for (size_t i = 0; i < nPeaks; i++) { const auto& peak = mPfilteredPeakPositions[i]; out << peak.time() << " " << int32_t{peak.pad()} << " " << int32_t{peak.row()} << "\n"; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx index 693ee4dd78e8d..e37ef5dbc454d 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx @@ -42,14 +42,14 @@ static_assert(GPUTPCNNClusterizerKernels::SCRATCH_PAD_WORK_GROUP_SIZE == GPUTPCC template <> GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); CPU_ONLY(MCLabelAccumulator labelAcc(clusterer)); tpc::ClusterNative* clusterOut = clusterer.mPclusterByRow; int8_t isAccepted = (clustererNN.mNnClusterizerUseClassification ? (clustererNN.mOutputDataClass[CAMath::Min(glo_idx, (uint32_t)clusterer.mPmemory->counters.nClusters - 1)] > 0) : 1); - GPUTPCCFClusterizer::computeClustersImpl(get_num_groups(0), get_local_size(0), get_group_id(0), get_local_id(0), clusterer, clusterer.mPmemory->fragment, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, isAccepted); + GPUTPCCFClusterizer::computeClustersImpl(nBlocks, nThreads, iBlock, iThread, clusterer, clusterer.mPmemory->frag, smem, chargeMap, clusterer.mPfilteredPeakPositions, clusterer.Param().rec, CPU_PTR(&labelAcc), clusterer.mPmemory->counters.nClusters, clusterer.mNMaxClusterPerRow, clusterer.mPclusterInRow, clusterOut, clusterer.mPclusterPosInRow, isAccepted); } template <> @@ -58,7 +58,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { return; } @@ -147,7 +147,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx + batchStart >= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -263,7 +263,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx + batchStart >= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -302,7 +302,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -327,6 +327,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -347,6 +348,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -355,7 +357,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).isOverlap(peak.time())) { + if ((clusterer.mPmemory->frag).isOverlap(peak.time())) { if (clusterer.mPclusterPosInRow) { clusterer.mPclusterPosInRow[full_glo_idx] = clusterer.mNMaxClusterPerRow; } @@ -379,7 +381,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, notSingleTime ? clustererNN.mOutputDataReg1_32[model_output_index + 3] : 0.f, clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -390,7 +392,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, notSingleTime ? clustererNN.mOutputDataReg1_16[model_output_index + 3].ToFloat() : 0.f, clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -498,7 +500,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint32_t batchStart) { - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { @@ -521,6 +523,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -541,6 +544,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadcollect(peak, central_charge)); GPUTPCCFClusterizer::buildCluster( clusterer.Param().rec, + iThread, chargeMap, peak, smem.posBcast, @@ -549,7 +553,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).isOverlap(peak.time())) { + if ((clusterer.mPmemory->frag).isOverlap(peak.time())) { if (clusterer.mPclusterPosInRow) { clusterer.mPclusterPosInRow[full_glo_idx] = clusterer.mNMaxClusterPerRow; } @@ -565,7 +569,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_32[model_output_index + 6], clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -576,7 +580,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_16[model_output_index + 6].ToFloat(), clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -619,7 +623,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_32[model_output_index + 7], clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -630,7 +634,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Threadfragment).start + publishTimePosition, + (clusterer.mPmemory->frag).start + publishTimePosition, clustererNN.mOutputDataReg2_16[model_output_index + 7].ToFloat(), clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); @@ -669,7 +673,7 @@ template <> GPUdii() void GPUTPCNNClusterizerKernels::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& processors, uint8_t sector, int8_t dtype, int8_t withMC, uint batchStart) { // Implements identical publishing logic as the heuristic clusterizer and deconvolution kernel - uint32_t glo_idx = get_global_id(0); + uint32_t glo_idx = (iBlock * nThreads + iThread); auto& clusterer = processors.tpcClusterer[sector]; auto& clustererNN = processors.tpcNNClusterer[sector]; if (glo_idx + batchStart >= clusterer.mPmemory->counters.nClusters || glo_idx >= (uint32_t)clustererNN.mNnClusterizerBatchedMode) { diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h b/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h index 0867582fffa14..c0a0f897ca6a7 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h +++ b/GPU/GPUTracking/TRDTracking/GPUTRDGeometry.h @@ -41,7 +41,7 @@ class GPUTRDpadPlane : private o2::trd::PadPlane GPUd() float GetColPos(int32_t col) const { return getColPos(col); } GPUd() float GetNrows() const { return getNrows(); } GPUd() float GetNcols() const { return getNcols(); } - GPUd() int32_t GetPadRowNumber(double z) const { return getPadRowNumber(z); } + GPUd() int32_t GetPadRowNumber(o2::gpu::GPUdoubleValue z) const { return getPadRowNumber(z); } }; class GPUTRDGeometry : private o2::trd::GeometryFlat diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx b/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx index dea4cdbca430e..6632be71cbb75 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx +++ b/GPU/GPUTracking/TRDTracking/GPUTRDTrackerKernels.cxx @@ -35,8 +35,8 @@ GPUdii() void GPUTRDTrackerKernels::Thread(int32_t nBlocks, int32_t nThreads, in } } #endif - GPUCA_TBB_KERNEL_LOOP(trdTracker->GetRec(), int32_t, i, trdTracker->NTracks(), { - trdTracker->DoTrackingThread(i, get_global_id(0)); + GPUCA_TBB_KERNEL_LOOP(trdTracker->GetRec(), nBlocks, nThreads, iBlock, iThread, int32_t, i, trdTracker->NTracks(), { + trdTracker->DoTrackingThread(i, (iBlock * nThreads + iThread)); }); } diff --git a/GPU/GPUTracking/cmake/kernel_helpers.cmake b/GPU/GPUTracking/cmake/kernel_helpers.cmake index cc50d28ecef9e..459165d86cf5d 100644 --- a/GPU/GPUTracking/cmake/kernel_helpers.cmake +++ b/GPU/GPUTracking/cmake/kernel_helpers.cmake @@ -55,11 +55,13 @@ function(o2_gpu_add_kernel kernel_name kernel_files) math(EXPR n "${n} - 1") foreach(i RANGE 3 ${n} 2) math(EXPR j "${i} + 1") + # buffer indices 0, 1 and 2 are gpu_mem, the constant memory and the sector + math(EXPR TMP_ARG_IDX "3 + (${i} - 3) / 2") if(${ARGV${i}} MATCHES "\\*$") - string(APPEND OPT1 ",GPUPtr1(${ARGV${i}},${ARGV${j}})") + string(APPEND OPT1 ",GPUPtr1(${TMP_ARG_IDX},${ARGV${i}},${ARGV${j}})") string(APPEND OPT2 ",GPUPtr2(${ARGV${i}},${ARGV${j}})") else() - string(APPEND OPT1 ",${ARGV${i}} ${ARGV${j}}") + string(APPEND OPT1 ",GPUArg1(${TMP_ARG_IDX},${ARGV${i}},${ARGV${j}})") string(APPEND OPT2 ",${ARGV${j}}") endif() string(APPEND OPT3 ",${ARGV${i}}") diff --git a/GPU/Utils/GPUCommonBitSet.h b/GPU/Utils/GPUCommonBitSet.h index 302334e01e29d..eebe00b2bcdcb 100644 --- a/GPU/Utils/GPUCommonBitSet.h +++ b/GPU/Utils/GPUCommonBitSet.h @@ -37,11 +37,21 @@ class bitset public: GPUdDefault() constexpr bitset() = default; +#ifndef __METAL__ + // MSL will not use a user-declared copy constructor to build an object in the + // constant address space, where GPUconstexpr() arrays of bitset live. Leaving + // it implicit is what makes those arrays constructible. GPUdDefault() constexpr bitset(const bitset&) = default; +#endif #ifdef __OPENCL__ GPUdDefault() constexpr bitset(const __constant bitset&) = default; #endif // __OPENCL__ GPUd() constexpr bitset(uint32_t vv) : v(vv) {}; +#ifdef __METAL__ + // Objects in the constant address space are built and read through their own + // overloads: a generic `this` does not reach constant memory in MSL. + GPUd() constexpr bitset(uint32_t vv) constant : v(vv) {}; +#endif static GPUglobalconstexpr() uint32_t full_set = ((1ul << N) - 1ul); GPUd() constexpr bool all() const { return (v & full_set) == full_set; } @@ -78,6 +88,13 @@ class bitset GPUd() constexpr bool operator!=(const bitset b) const { return v != b.v; } GPUd() constexpr bool operator[](uint32_t i) const { return (v >> i) & 1u; } +#ifdef __METAL__ + GPUd() constexpr bitset operator|(const bitset b) constant { return v | b.v; } + GPUd() constexpr bitset operator&(const bitset b) constant { return v & b.v; } + GPUd() constexpr bool operator[](uint32_t i) constant { return (v >> i) & 1u; } + GPUd() constexpr bool any() constant { return v & full_set; } + GPUd() constexpr uint32_t to_ulong() constant { return v; } +#endif GPUd() constexpr uint32_t to_ulong() const { return v; } diff --git a/GPU/Utils/NDPiecewisePolynomials.h b/GPU/Utils/NDPiecewisePolynomials.h index 8a629c64affbb..d72204a594873 100644 --- a/GPU/Utils/NDPiecewisePolynomials.h +++ b/GPU/Utils/NDPiecewisePolynomials.h @@ -296,12 +296,16 @@ class NDPiecewisePolynomials : public FlatObject /// \return returns step width of the inner grid /// \param dim dimension /// \param nAuxiliaryPoints number of Auxiliary points for given dimension +#ifndef __METAL__ // host-only accessors, and MSL has no double double getStepWidth(const uint32_t dim, const int32_t nAuxiliaryPoints) const { return 1 / (static_cast(mInvSpacing[dim]) * (nAuxiliaryPoints - 1)); } +#endif /// \return returns vertex position for given index and dimension /// \param ix index /// \param dim dimension +#ifndef __METAL__ double getVertexPosition(const uint32_t ix, const int32_t dim) const { return ix / static_cast(mInvSpacing[dim]) + mMin[dim]; } +#endif #if !defined(GPUCA_GPUCODE) /// \return returns the size of the parameters diff --git a/GPU/Utils/Spline1DSpec.h b/GPU/Utils/Spline1DSpec.h index b3f895cbde4ed..b6f1a1695f213 100644 --- a/GPU/Utils/Spline1DSpec.h +++ b/GPU/Utils/Spline1DSpec.h @@ -296,7 +296,11 @@ template GPUdi() void Spline1DContainerBase::setXrange(DataT xMin, DataT xMax) { mXmin = xMin; +#ifdef __METAL__ // MSL has no double + float l = ((float)xMax) - xMin; +#else double l = ((double)xMax) - xMin; +#endif if (l < 1.e-8) { l = 1.e-8; } diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index d2f426c448e12..801bcd17b691d 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -10,7 +10,7 @@ # or submit itself to any jurisdiction. # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources -# FindO2GPU.cmake Version 19 +# FindO2GPU.cmake Version 21 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual) set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908) @@ -20,7 +20,13 @@ set(HIP_AMDGPUTARGET_DEFAULT_MINIMAL gfx906) if(NOT DEFINED ENABLE_CUDA) set(ENABLE_CUDA "AUTO") endif() -if(NOT DEFINED ENABLE_OPENCL) +if(APPLE) + # macOS ships OpenCL 1.2 only, below the 2.x the OpenCL backend needs; Metal + # replaces it there. OFF rather than AUTO while the backend is unproven. + if(NOT DEFINED ENABLE_METAL) + set(ENABLE_METAL "OFF") + endif() +elseif(NOT DEFINED ENABLE_OPENCL) set(ENABLE_OPENCL "AUTO") endif() if(NOT DEFINED ENABLE_HIP) @@ -28,6 +34,9 @@ if(NOT DEFINED ENABLE_HIP) endif() string(TOUPPER "${ENABLE_CUDA}" ENABLE_CUDA) string(TOUPPER "${ENABLE_OPENCL}" ENABLE_OPENCL) +if(APPLE) + string(TOUPPER "${ENABLE_METAL}" ENABLE_METAL) +endif() string(TOUPPER "${ENABLE_HIP}" ENABLE_HIP) if(NOT DEFINED CMAKE_BUILD_TYPE_UPPER) string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER) @@ -112,6 +121,9 @@ function(detect_gpu_arch backend) # Detect GPU architecture, optionally filterri if(OPENCL_ENABLED OR backend STREQUAL "ALL") list(APPEND TARGET_ARCH "OPENCL") endif() + if(METAL_ENABLED OR backend STREQUAL "ALL") + list(APPEND TARGET_ARCH "METAL") + endif() set(TARGET_ARCH "${TARGET_ARCH}" PARENT_SCOPE) else() message(FATAL_ERROR "Unknown backend provided: ${backend}") @@ -149,6 +161,7 @@ if(GPUCA_DETERMINISTIC_NO_FTZ) set(GPUCA_CXX_DENORMALS_FLAGS "") set(GPUCA_CUDA_DENORMALS_FLAGS "--ftz=false") set(GPUCA_OCL_DENORMALS_FLAGS "") + set(GPUCA_METAL_DENORMALS_FLAGS "-fno-fast-math") set(GPUCA_HIP_DENORMALS_FLAGS "-fno-gpu-flush-denormals-to-zero") else() if (CMAKE_SYSTEM_NAME MATCHES Darwin OR NOT CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") @@ -158,11 +171,13 @@ else() endif() set(GPUCA_CUDA_DENORMALS_FLAGS "--ftz=true") set(GPUCA_OCL_DENORMALS_FLAGS "-cl-denorms-are-zero") + set(GPUCA_METAL_DENORMALS_FLAGS "-ffast-math") set(GPUCA_HIP_DENORMALS_FLAGS "-fgpu-flush-denormals-to-zero") endif() set(GPUCA_CXX_NO_FAST_MATH_FLAGS "-fno-fast-math -ffp-contract=off") set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--prec-div=true --prec-sqrt=true --fmad false -Xcompiler -fno-fast-math -Xcompiler -ffp-contract=off") set(GPUCA_OCL_NO_FAST_MATH_FLAGS -cl-fp32-correctly-rounded-divide-sqrt ) +set(GPUCA_METAL_NO_FAST_MATH_FLAGS "-fno-fast-math") if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_WHOLEO2}) add_definitions(-DGPUCA_DETERMINISTIC_MODE) string(APPEND CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} " ${GPUCA_CXX_NO_FAST_MATH_FLAGS}") @@ -429,6 +444,41 @@ if(ENABLE_HIP) endif() endif() +# =================================== Metal ================================== +if(ENABLE_METAL) + find_library(METAL_FRAMEWORK Metal) + find_library(COREFOUNDATION_FRAMEWORK CoreFoundation) + find_library(FOUNDATION_FRAMEWORK Foundation) + find_library(QUARTZCORE_FRAMEWORK QuartzCore) + # The frameworks are there on every macOS, but the backend needs MSL 4.1, which + # is macOS 27 and its toolchain, so ask the compiler instead of assuming. A + # library built as MSL 4.1 also only loads on macOS 27 and later. + if(NOT DEFINED GPUCA_METAL_MSL41) + set(GPUCA_METAL_PROBE "${CMAKE_CURRENT_BINARY_DIR}/metal_msl41_probe.metal") + file(WRITE "${GPUCA_METAL_PROBE}" "#include \nkernel void probe(device float* o [[buffer(0)]], uint i [[thread_position_in_grid]]) { o[i] = o[i] * 2.0f; }\n") + execute_process(COMMAND xcrun -sdk macosx metal -std=metal4.1 -c "${GPUCA_METAL_PROBE}" -o "${GPUCA_METAL_PROBE}.air" + RESULT_VARIABLE GPUCA_METAL_PROBE_RESULT OUTPUT_QUIET ERROR_QUIET) + if(GPUCA_METAL_PROBE_RESULT EQUAL 0) + set(GPUCA_METAL_MSL41 ON CACHE INTERNAL "Metal toolchain compiles MSL 4.1") + else() + set(GPUCA_METAL_MSL41 OFF CACHE INTERNAL "Metal toolchain compiles MSL 4.1") + endif() + endif() + if(METAL_FRAMEWORK AND COREFOUNDATION_FRAMEWORK AND FOUNDATION_FRAMEWORK AND QUARTZCORE_FRAMEWORK AND GPUCA_METAL_MSL41) + set(METAL_ENABLED ON) + set(METAL_FRAMEWORKS ${METAL_FRAMEWORK} ${COREFOUNDATION_FRAMEWORK} + ${FOUNDATION_FRAMEWORK} ${QUARTZCORE_FRAMEWORK}) + message(STATUS "Found Metal frameworks") + elseif(NOT ENABLE_METAL STREQUAL "AUTO") + if(NOT GPUCA_METAL_MSL41) + message(FATAL_ERROR "The Metal backend needs MSL 4.1: this toolchain rejects -std=metal4.1, and the result would need macOS 27 to run") + endif() + message(FATAL_ERROR "Metal frameworks not available") + else() + set(METAL_ENABLED OFF) + endif() +endif() + # if we end up here without a FATAL, it means we have found the "O2GPU" package set(O2GPU_FOUND TRUE) if (NOT GPUCA_FINDO2GPU_CHECK_ONLY)