Add iterators and specializations for pointer wrapper

Author: OliverRietmann

.github/workflows/standalone-benchmark.yml

@@ -73,6 +73,7 @@ jobs:
           STANDALONE_DIR: /root/standalone
           BUILD_DIR: /root/standalone/build
           ARTIFACT_FILE: /root/artifact.txt
+          ALIBUILD_O2_FORCE_GPU: 1
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v4

GPU/Common/GPUCommonAlgorithm.h

@@ -16,7 +16,6 @@
 #define GPUCOMMONALGORITHM_H
 
 #include "GPUCommonDef.h"
-#include "MemLayout.h"
 
 #if !defined(GPUCA_GPUCODE) // Could also enable custom search on the CPU, but it is not always faster, so we stick to std::sort
 #include <algorithm>

GPU/Common/GPUCommonAlgorithmThrust.h

@@ -19,7 +19,6 @@
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wshadow"
 #include <thrust/sort.h>
-#include <thrust/iterator/iterator_traits.h>
 #include <thrust/execution_policy.h>
 #include <thrust/device_ptr.h>
 #pragma GCC diagnostic pop

GPU/Common/MemLayout.h

@@ -12,29 +12,34 @@ template <class T> using const_reference = const T&;
 template <class T> using pointer = T*;
 template <class T> using const_pointer = const T*;
 
+template <class T> using reference_restrict = T& __restrict__;
+template <class T> using const_reference_restrict = const T& __restrict__;
+template <class T> using pointer_restrict = T* __restrict__;
+template <class T> using const_pointer_restrict = const T* __restrict__;
+
 template <class SF>
 struct RandomAccessAt {
     MemLayout::size_t i;
     template <class... Args>
-    constexpr SF operator()(Args& ...args) const { return {{args[i]...}}; }
+    constexpr SF operator()(Args& ...args) const { return {args[i]...}; }
     template <class... Args>
-    constexpr SF operator()(const Args& ...args) const { return {{args[i]...}}; }
+    constexpr SF operator()(const Args& ...args) const { return {args[i]...}; }
 };
 
 template <class SF>
 struct GetPointer {
     template <class... Args>
-    constexpr SF operator()(Args& ...args) const { return {{&args...}}; }
+    constexpr SF operator()(Args& ...args) const { return {&args...}; }
     template <class... Args>
-    constexpr SF operator()(const Args& ...args) const { return {{&args...}}; }
+    constexpr SF operator()(const Args& ...args) const { return {&args...}; }
 };
 
 template <class SF>
 struct AggregateConstructor {
     template <class... Args>
-    constexpr SF operator()(Args& ...args) const { return {{args...}}; }
+    constexpr SF operator()(Args& ...args) const { return {args...}; }
     template <class... Args>
-    constexpr SF operator()(const Args& ...args) const { return {{args...}}; } 
+    constexpr SF operator()(const Args& ...args) const { return {args...}; }
 };
 
 struct FirstMember {
@@ -47,20 +52,20 @@ struct FirstMember {
 template <class SF>
 struct PreIncrement {
     template <class... Args>
-    constexpr SF operator()(Args& ...args) const { return {{++args...}}; }
+    constexpr SF operator()(Args& ...args) const { return {++args...}; }
 };
 
 template <class SF>
 struct PreDecrement {
     template <class... Args>
-    constexpr SF operator()(Args& ...args) const { return {{--args...}}; }
+    constexpr SF operator()(Args& ...args) const { return {--args...}; }
 };
 
 template <class SF>
 struct Advance {
     ptrdiff_t i;
     template <class... Args>
-    constexpr SF operator()(const Args& ...args) const { return {{(args + i)...}}; }
+    constexpr SF operator()(const Args& ...args) const { return {(args + i)...}; }
 };
 
 struct CopyAssignment {
@@ -78,10 +83,12 @@ struct wrapper : public S<F> {
     constexpr wrapper() = default;
     constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
     template <template <class> class F_other>
-    constexpr wrapper(wrapper<S, F_other>& other) : Base{other.apply(AggregateConstructor<wrapper>{})} {}
+    constexpr wrapper(S<F_other>& other) : Base{other.apply(AggregateConstructor<Base>{})} {}
+    template <template <class> class F_other>
+    constexpr wrapper(const S<F_other>& other) : Base{other.apply(AggregateConstructor<Base>{})} {}
 
-    constexpr wrapper<S, reference> operator[] (size_t i) { return Base::apply(RandomAccessAt<wrapper<S, reference>>{i}); }
-    constexpr wrapper<S, const_reference> operator[] (size_t i) const { return Base::apply(RandomAccessAt<wrapper<S, const_reference>>{i}); }
+    constexpr wrapper<S, reference> operator[] (size_t i) { return Base::apply(RandomAccessAt<S<reference>>{i}); }
+    constexpr wrapper<S, const_reference> operator[] (size_t i) const { return Base::apply(RandomAccessAt<S<const_reference>>{i}); }
 
     constexpr wrapper<S, reference> operator*() { return operator[](0); }
     constexpr wrapper<S, const_reference> operator*(ptrdiff_t) const { return operator[](0); }
@@ -93,8 +100,8 @@ struct wrapper<S, value> : public S<value> {
 
     constexpr wrapper() = default;
     constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
-    constexpr wrapper(wrapper<S, reference> other) : Base(other.apply(AggregateConstructor<wrapper>{})) {}
-    constexpr wrapper(wrapper<S, const_reference> other) : Base(other.apply(AggregateConstructor<wrapper>{})) {}
+    constexpr wrapper(const S<reference>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<const_reference>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
 
     //constexpr S<MemLayout::pointer> operator& () { return apply(GetPointer<S<MemLayout::pointer>>{}); }
     //constexpr S<MemLayout::const_pointer> operator& () const { return apply(GetPointer<S<MemLayout::const_pointer>>{}); }
@@ -106,10 +113,15 @@ struct wrapper<S, reference> : public S<reference> {
 
     constexpr wrapper() = delete;
     constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
-    constexpr wrapper(wrapper<S, value>& other) : Base(other.apply(AggregateConstructor<wrapper>{})) {}
+    constexpr wrapper(S<value>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(S<reference_restrict> other) : Base(other.apply(AggregateConstructor<Base>{})) {}
 
     constexpr wrapper(const wrapper& other) = default;
 
+    constexpr wrapper& operator=(const wrapper<S, value>& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
     constexpr wrapper& operator=(const wrapper& other) {
         Base::apply(other, CopyAssignment{});
         return *this;
@@ -118,17 +130,63 @@ struct wrapper<S, reference> : public S<reference> {
         Base::apply(other, CopyAssignment{});
         return *this;
     }
+    constexpr wrapper& operator=(const wrapper<S, reference_restrict>& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
+    constexpr wrapper& operator=(const wrapper<S, const_reference_restrict>& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
+
+    constexpr wrapper(wrapper&& other) = default;
+    
+    constexpr wrapper& operator=(wrapper&& other) { return operator=(other); }
+
+    constexpr wrapper<S, pointer> operator&() { return Base::apply(GetPointer<S<pointer>>{}); }
+    //constexpr wrapper<S, const_pointer> operator&() const { return Base::apply(GetPointer<S<const_pointer>>{}); }
+    constexpr pointer<wrapper<S, reference>> operator->() { return this; }
+};
+
+template <template <template <class> class> class S>
+struct wrapper<S, reference_restrict> : public S<reference_restrict> {
+    using Base = S<reference_restrict>;
+
+    constexpr wrapper() = delete;
+    constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
+    constexpr wrapper(S<value>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(S<reference> other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    
+    constexpr wrapper(const wrapper& other) = default;
+
     constexpr wrapper& operator=(const wrapper<S, value>& other) {
         Base::apply(other, CopyAssignment{});
         return *this;
     }
+    constexpr wrapper& operator=(const wrapper& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
+    constexpr wrapper& operator=(const wrapper<S, reference>& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
+    constexpr wrapper& operator=(const wrapper<S, const_reference>& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
+    constexpr wrapper& operator=(const wrapper<S, const_reference_restrict>& other) {
+        Base::apply(other, CopyAssignment{});
+        return *this;
+    }
 
     constexpr wrapper(wrapper&& other) = default;
 
     constexpr wrapper& operator=(wrapper&& other) { return operator=(other); }
 
-    constexpr wrapper<S, pointer> operator& () { return Base::apply(GetPointer<wrapper<S, pointer>>{}); }
-    //constexpr wrapper<S, const_pointer> operator& () const { return Base::apply(GetPointer<wrapper<S, const_pointer>>{}); }
+    constexpr wrapper<S, pointer> operator&() { return Base::apply(GetPointer<S<pointer>>{}); }
+    //constexpr wrapper<S, const_pointer> operator&() const { return Base::apply(GetPointer<S<const_pointer>>{}); }
+    constexpr pointer<wrapper<S, reference>> operator->() { return this; }
 };
 
 template <template <template <class> class> class S>
@@ -137,10 +195,28 @@ struct wrapper<S, const_reference> : public S<const_reference> {
 
     constexpr wrapper() = delete;
     constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
-    constexpr wrapper(wrapper<S, value> other) : Base(other.apply(AggregateConstructor<wrapper>{})) {}
-    constexpr wrapper(wrapper<S, reference> other) : Base(other.apply(AggregateConstructor<wrapper>{})) {}
+    constexpr wrapper(const S<value>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<reference>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<reference_restrict>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<const_reference_restrict>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
 
-    constexpr wrapper<S, const_pointer> operator&() const { return Base::apply(GetPointer<wrapper<S, const_pointer>>{}); }
+    constexpr wrapper<S, const_pointer> operator&() const { return Base::apply(GetPointer<S<const_pointer>>{}); }
+    constexpr const_pointer<wrapper<S, const_reference>> operator->() const { return this; }
+};
+
+template <template <template <class> class> class S>
+struct wrapper<S, const_reference_restrict> : public S<const_reference_restrict> {
+    using Base = S<const_reference_restrict>;
+
+    constexpr wrapper() = delete;
+    constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
+    constexpr wrapper(const S<value>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<reference>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<reference_restrict>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+    constexpr wrapper(const S<const_reference>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
+
+    constexpr wrapper<S, const_pointer> operator&() const { return Base::apply(GetPointer<S<const_pointer>>{}); }
+    constexpr const_pointer<wrapper<S, const_reference>> operator->() const { return this; }
 };
 
 template <template <template <class> class> class S>
@@ -150,23 +226,25 @@ struct wrapper<S, pointer> : public S<pointer> {
     constexpr wrapper() = default;
     constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
 
-    constexpr wrapper<S, reference> operator[] (size_t i) { return Base::apply(RandomAccessAt<wrapper<S, reference>>{i}); }
-    constexpr wrapper<S, const_reference> operator[] (size_t i) const { return Base::apply(RandomAccessAt<wrapper<S, const_reference>>{i}); }
+    constexpr wrapper<S, reference> operator[] (size_t i) { return Base::apply(RandomAccessAt<S<reference>>{i}); }
+    constexpr const wrapper<S, const_reference> operator[] (size_t i) const { return Base::apply(RandomAccessAt<S<const_reference>>{i}); }
 
     constexpr wrapper<S, reference> operator*() { return operator[](0); }
     constexpr wrapper<S, const_reference> operator*() const { return operator[](0); }
+    constexpr wrapper<S, reference> operator->() { return operator[](0); }
+    constexpr wrapper<S, const_reference> operator->() const { return operator[](0); }
 
     constexpr bool operator==(const wrapper& other) const { return Base::apply(FirstMember{}) == other.apply(FirstMember{}); }
     constexpr bool operator!=(const wrapper& other) const { return !this->operator==(other); }
     constexpr bool operator<(const wrapper& other) const { return Base::apply(FirstMember{}) < other.apply(FirstMember{}); }
 
-    constexpr wrapper operator+(ptrdiff_t i) const { return Base::apply(Advance<wrapper>{i}); }
+    constexpr wrapper operator+(ptrdiff_t i) const { return Base::apply(Advance<Base>{i}); }
     constexpr wrapper operator-(ptrdiff_t i) const { return operator+(-i); }
     constexpr ptrdiff_t operator-(const wrapper& other) const { return Base::apply(FirstMember{}) - other.apply(FirstMember{}); }
 
-    constexpr wrapper& operator++() { Base::apply(PreIncrement<wrapper>{}); return *this; }
+    constexpr wrapper& operator++() { Base::apply(PreIncrement<Base>{}); return *this; }
     constexpr wrapper& operator+=(ptrdiff_t i) { return *this = *this + i; }
-    constexpr wrapper& operator--() { Base::apply(PreDecrement<wrapper>{}); return *this; }
+    constexpr wrapper& operator--() { Base::apply(PreDecrement<Base>{}); return *this; }
     constexpr wrapper& operator-=(ptrdiff_t i) { return *this = *this - i; }
 };
 
@@ -176,10 +254,24 @@ struct wrapper<S, const_pointer> : public S<const_pointer> {
 
     constexpr wrapper() = default;
     constexpr wrapper(Base b) : Base{static_cast<Base&&>(b)} {}
-    constexpr wrapper(wrapper<S, pointer> other) : Base(other.apply(AggregateConstructor<wrapper>{})) {}
+    constexpr wrapper(const S<pointer>& other) : Base(other.apply(AggregateConstructor<Base>{})) {}
 
-    constexpr wrapper<S, const_reference> operator[] (size_t i) const { return Base::apply(RandomAccessAt<wrapper<S, const_reference>>{i}); }
+    constexpr wrapper<S, const_reference> operator[] (size_t i) const { return Base::apply(RandomAccessAt<S<const_reference>>{i}); }
     constexpr wrapper<S, const_reference> operator*() const { return operator[](0); }
+    constexpr wrapper<S, const_reference> operator->() const { return operator[](0); }
+
+    constexpr bool operator==(const wrapper& other) const { return Base::apply(FirstMember{}) == other.apply(FirstMember{}); }
+    constexpr bool operator!=(const wrapper& other) const { return !this->operator==(other); }
+    constexpr bool operator<(const wrapper& other) const { return Base::apply(FirstMember{}) < other.apply(FirstMember{}); }
+
+    constexpr wrapper operator+(ptrdiff_t i) const { return Base::apply(Advance<Base>{i}); }
+    constexpr wrapper operator-(ptrdiff_t i) const { return operator+(-i); }
+    constexpr ptrdiff_t operator-(const wrapper& other) const { return Base::apply(FirstMember{}) - other.apply(FirstMember{}); }
+
+    constexpr wrapper& operator++() { Base::apply(PreIncrement<Base>{}); return *this; }
+    constexpr wrapper& operator+=(ptrdiff_t i) { return *this = *this + i; }
+    constexpr wrapper& operator--() { Base::apply(PreDecrement<Base>{}); return *this; }
+    constexpr wrapper& operator-=(ptrdiff_t i) { return *this = *this - i; }
 };
 
 enum Flag { soa, aos };
@@ -204,6 +296,10 @@ struct interface<S, F, Flag::soa> { using type = wrapper<S, F>; };
 #define MEMLAYOUT_EXPAND(m) f(m, other.m)
 
 #define MEMLAYOUT_APPLY_BINARY(STRUCT_NAME, ...)\
+    template <template <class> class F_other, class Function>\
+    constexpr STRUCT_NAME apply(STRUCT_NAME<F_other>& other, Function&& f) { return {__VA_ARGS__}; }\
+    template <template <class> class F_other, class Function>\
+    constexpr STRUCT_NAME apply(STRUCT_NAME<F_other>& other, Function&& f) const { return {__VA_ARGS__}; }\
     template <template <class> class F_other, class Function>\
     constexpr STRUCT_NAME apply(const STRUCT_NAME<F_other>& other, Function&& f) { return {__VA_ARGS__}; }\
     template <template <class> class F_other, class Function>\

GPU/GPUTracking/DataCompression/GPUTPCCompressionTrackModel.h

@@ -27,6 +27,7 @@
 
 #elif defined(GPUCA_COMPRESSION_TRACK_MODEL_SECTORTRACKER)
 #include "GPUTPCTrackParam.h"
+#include "MemLayout.h"
 
 #else // Default internal track model for compression
 #endif
@@ -121,7 +122,7 @@ class GPUTPCCompressionTrackModel
   const GPUParam* mParam;
 
 #elif defined(GPUCA_COMPRESSION_TRACK_MODEL_SECTORTRACKER)
-  GPUTPCTrackParam mTrk;
+  GPUTPCTrackParamSkeleton<MemLayout::value> mTrk;
   float mAlpha;
   const GPUParam* mParam;
 

GPU/GPUTracking/DataTypes/GPUDataTypes.h

@@ -225,7 +225,7 @@ struct GPUTrackingInOutPointers {
   const AliHLTTPCRawCluster* rawClusters[NSECTORS] = {nullptr};
   uint32_t nRawClusters[NSECTORS] = {0};
   const o2::tpc::ClusterNativeAccess* clustersNative = nullptr;
-  MemLayout::wrapper<GPUTPCTrackSkeleton, MemLayout::const_pointer> sectorTracks[NSECTORS];// = {{nullptr, nullptr, nullptr, {nullptr, nullptr, nullptr, nullptr}}};
+  MemLayout::interface<GPUTPCTrackSkeleton, MemLayout::const_pointer, GPUTPCTrackLayout>::type sectorTracks[NSECTORS];
   uint32_t nSectorTracks[NSECTORS] = {0};
   const GPUTPCHitId* sectorClusters[NSECTORS] = {nullptr};
   uint32_t nSectorClusters[NSECTORS] = {0};

GPU/GPUTracking/Global/GPUChainTracking.cxx

@@ -549,23 +549,33 @@ void GPUChainTracking::ClearIOPointers()
   new (&mIOMem) InOutMemory;
 }
 
+namespace {
+
+template <class Function>
+struct ApplyRecursive {
+    Function f;
+
+    template <class T>
+    const T * operator()(const T * & aosIOPtr, GPUChainTracking::unique_ptr_array<T>& aosIOMem) const { return f(aosIOPtr, aosIOMem); }
+
+    template <template <template <class> class> class S>
+    S<MemLayout::const_pointer> operator()(S<MemLayout::const_pointer>& soaIOPtr, S<GPUChainTracking::unique_ptr_array>& soaIOMem) const {
+        return soaIOPtr.apply(soaIOMem, ApplyRecursive{f});
+    }
+};
+
+}
+
 void GPUChainTracking::AllocateIOMemory()
 {
   for (uint32_t i = 0; i < NSECTORS; i++) {
     AllocateIOMemoryHelper(mIOPtrs.nClusterData[i], mIOPtrs.clusterData[i], mIOMem.clusterData[i]);
     AllocateIOMemoryHelper(mIOPtrs.nRawClusters[i], mIOPtrs.rawClusters[i], mIOMem.rawClusters[i]);
-    
-    //AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i], mIOMem.sectorTracks[i]);
-
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mFirstHitID, mIOMem.sectorTracks[i].mFirstHitID);
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mNHits, mIOMem.sectorTracks[i].mNHits);
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mLocalTrackId, mIOMem.sectorTracks[i].mLocalTrackId);
-
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mParam.mX, mIOMem.sectorTracks[i].mParam.mX);
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mParam.mC, mIOMem.sectorTracks[i].mParam.mC);
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mParam.mZOffset, mIOMem.sectorTracks[i].mParam.mZOffset);
-    AllocateIOMemoryHelper(mIOPtrs.nSectorTracks[i], mIOPtrs.sectorTracks[i].mParam.mP, mIOMem.sectorTracks[i].mParam.mP);
-
+    auto sectorTrackAllocator = [this, nSectorTrack = this->mIOPtrs.nSectorTracks[i]](auto& IOPtrsTrack, auto& mIOMemTrack) {
+        AllocateIOMemoryHelper(nSectorTrack, IOPtrsTrack, mIOMemTrack);
+        return IOPtrsTrack;
+    };
+    ApplyRecursive{sectorTrackAllocator}(mIOPtrs.sectorTracks[i], mIOMem.sectorTracks[i]);
     AllocateIOMemoryHelper(mIOPtrs.nSectorClusters[i], mIOPtrs.sectorClusters[i], mIOMem.sectorClusters[i]);
   }
   mIOMem.clusterNativeAccess.reset(new ClusterNativeAccess);