All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
pool.h
1 //
2 // Copyright 2019 Pixar
3 //
4 // Licensed under the Apache License, Version 2.0 (the "Apache License")
5 // with the following modification; you may not use this file except in
6 // compliance with the Apache License and the following modification to it:
7 // Section 6. Trademarks. is deleted and replaced with:
8 //
9 // 6. Trademarks. This License does not grant permission to use the trade
10 // names, trademarks, service marks, or product names of the Licensor
11 // and its affiliates, except as required to comply with Section 4(c) of
12 // the License and to reproduce the content of the NOTICE file.
13 //
14 // You may obtain a copy of the Apache License at
15 //
16 // http://www.apache.org/licenses/LICENSE-2.0
17 //
18 // Unless required by applicable law or agreed to in writing, software
19 // distributed under the Apache License with the above modification is
20 // distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21 // KIND, either express or implied. See the Apache License for the specific
22 // language governing permissions and limitations under the Apache License.
23 //
24 #ifndef SDF_POOL_H
25 #define SDF_POOL_H
26 
27 #include "pxr/pxr.h"
28 #include "pxr/usd/sdf/api.h"
29 
30 #include "pxr/base/arch/demangle.h"
31 #include "pxr/base/arch/hints.h"
32 #include "pxr/base/tf/diagnostic.h"
33 #include "pxr/base/tf/staticData.h"
34 
35 #include <tbb/concurrent_queue.h>
36 
37 #include <atomic>
38 #include <thread>
39 
40 PXR_NAMESPACE_OPEN_SCOPE
41 
42 // A helper struct for thread_local that uses nullptr initialization as a
43 // sentinel to prevent guard variable use from being invoked after first
44 // initialization.
45 template <class T>
46 struct Sdf_FastThreadLocalBase
47 {
48  static T &Get() {
49  static thread_local T *theTPtr = nullptr;
50  if (ARCH_LIKELY(theTPtr)) {
51  return *theTPtr;
52  }
53  static thread_local T theT;
54  T *p = &theT;
55  theTPtr = p;
56  return *p;
57  }
58 };
59 
60 // Fixed-size scalable pool allocator with 32-bit "handles" returned. Reserves
61 // virtual memory in big regions. It's optimized for per-thread allocations,
62 // and intended to be used for SdfPath. The Tag template argument just serves
63 // as to distinguish one pool instantiation from another with otherwise
64 // equivalent template arguments. The ElemSize argument is the size of an
65 // allocated element in bytes. It must be at least 4 bytes. The RegionBits
66 // argument determines how many contiguous "chunks" of virtual memory the pool
67 // will use. A good number for this is 8, meaning we'll have at most 256
68 // "regions" or "chunks" of contiguous virtual memory, each 2^24 times ElemSize
69 // bytes in size. To allocate from reserved chunks, each thread acquires a span
70 // to hold ElemsPerSpan elements from the range, then uses that to dole out
71 // individual allocations. When freed, allocations are placed on a thread-local
72 // free list, and eventually shared back for use by other threads when the free
73 // list gets large.
74 template <class Tag,
75  unsigned ElemSize,
76  unsigned RegionBits,
77  unsigned ElemsPerSpan=16384>
78 class Sdf_Pool
79 {
80  static_assert(ElemSize >= sizeof(uint32_t),
81  "ElemSize must be at least sizeof(uint32_t)");
82 
83 public:
84  // Number of pool elements per region.
85  static constexpr uint64_t ElemsPerRegion = 1ull << (32-RegionBits);
86 
87  // Maximum index of an element in a region.
88  static constexpr uint32_t MaxIndex = ElemsPerRegion - 1;
89 
90  // Mask to extract region number from a handle value.
91  static constexpr uint32_t RegionMask = ((1 << RegionBits)-1);
92 
93  friend struct Handle;
94  // A Handle refers to an item in the pool. It just wraps around a uint32_t
95  // that represents the item's index and the item's region.
96  struct Handle {
97  constexpr Handle() noexcept = default;
98  constexpr Handle(std::nullptr_t) noexcept : value(0) {}
99  Handle(unsigned region, uint32_t index)
100  : value((index << RegionBits) | region) {}
101  Handle &operator=(Handle const &) = default;
102  Handle &operator=(std::nullptr_t) { return *this = Handle(); }
103  inline char *GetPtr() const noexcept {
104  ARCH_PRAGMA_PUSH
105  ARCH_PRAGMA_MAYBE_UNINITIALIZED
106  return Sdf_Pool::_GetPtr(value & RegionMask, value >> RegionBits);
107  ARCH_PRAGMA_POP
108  }
109  static inline Handle GetHandle(char const *ptr) noexcept {
110  return Sdf_Pool::_GetHandle(ptr);
111  }
112  explicit operator bool() const {
113  return value != 0;
114  }
115  inline bool operator ==(Handle const &r) const noexcept {
116  return value == r.value;
117  }
118  inline bool operator !=(Handle const &r) const noexcept {
119  return value != r.value;
120  }
121  inline bool operator <(Handle const &r) const noexcept {
122  return value < r.value;
123  }
124  inline void swap(Handle &r) noexcept {
125  std::swap(value, r.value);
126  }
127  uint32_t value = 0;
128  };
129 
130 private:
131  // We maintain per-thread free lists of pool items.
132  struct _FreeList {
133  inline void Pop() {
134  char *p = head.GetPtr();
135  Handle *hp = reinterpret_cast<Handle *>(p);
136  head = *hp;
137  --size;
138  }
139  inline void Push(Handle h) {
140  ++size;
141  char *p = h.GetPtr();
142  Handle *hp = reinterpret_cast<Handle *>(p);
143  *hp = head;
144  head = h;
145  }
146  Handle head;
147  size_t size = 0;
148  };
149 
150  // A pool span represents a "chunk" of new pool space that threads allocate
151  // from when their free lists are empty. When both the free list and the
152  // pool span are exhausted, a thread will look for a shared free list, or
153  // will obtain a new chunk of pool space to use.
154  struct _PoolSpan {
155  size_t size() const { return endIndex - beginIndex; }
156  inline Handle Alloc() {
157  return Handle(region, beginIndex++);
158  }
159  inline bool empty() const {
160  return beginIndex == endIndex;
161  }
162  unsigned region;
163  uint32_t beginIndex;
164  uint32_t endIndex;
165  };
166 
167  struct _PerThreadData {
168  // Local free-list of elems returned to the pool.
169  _FreeList freeList;
170  // Contiguous range of reserved but as-yet-unalloc'd space.
171  _PoolSpan span;
172  };
173 
174  // The region state structure represents the global state of the pool. This
175  // is a pool-global structure that is used to reserve new spans of pool data
176  // by threads when needed. See the Reserve() member, and the _ReserveSpan()
177  // function that does most of the state manipulation.
178  struct _RegionState {
179  static constexpr uint32_t LockedState = ~0;
180 
181  _RegionState() = default;
182  constexpr _RegionState(unsigned region, uint32_t index)
183  : _state((index << RegionBits) | region) {}
184 
185  // Make a new state that reserves up to \p num elements. There must be
186  // space left remaining.
187  inline _RegionState Reserve(unsigned num) const;
188 
189  static constexpr _RegionState GetInitState() {
190  return _RegionState(0, 0);
191  }
192 
193  static constexpr _RegionState GetLockedState() {
194  return _RegionState(LockedState, LockedState);
195  }
196 
197  constexpr bool operator==(_RegionState other) const {
198  return _state == other._state;
199  }
200 
201  uint32_t GetIndex() const {
202  return _state >> RegionBits;
203  }
204 
205  unsigned GetRegion() const {
206  return _state & RegionMask;
207  }
208 
209  bool IsLocked() const {
210  return _state == LockedState;
211  }
212 
213  // low RegionBits bits are region id, rest are index.
214  uint32_t _state;
215  };
216 
217 public:
218  static inline Handle Allocate();
219  static inline void Free(Handle h);
220 
221 private:
222 
223  // Given a region id and index, form the pointer into the pool.
224  static inline char *_GetPtr(unsigned region, uint32_t index) {
225  return _regionStarts[region] + (index * ElemSize);
226  }
227 
228  // Given a pointer into the pool, produce its corresponding Handle. Don't
229  // do this unless you really have to, it has to do a bit of a search.
230  static inline Handle _GetHandle(char const *ptr) {
231  if (ptr) {
232  for (unsigned region = 1; region != NumRegions+1; ++region) {
233  char const *start = _regionStarts[region];
234  ptrdiff_t diff = ptr - start;
235  // Indexes start at 1 to avoid hash collisions when combining
236  // multiple pool indexes in a single hash, so strictly greater
237  // than 0 rather than greater-equal is appropriate here.
238  if (ARCH_LIKELY(start && (diff > 0) &&
239  (diff < static_cast<ptrdiff_t>(ElemsPerRegion*ElemSize)))){
240  return Handle(region,
241  static_cast<uint32_t>(diff / ElemSize));
242  }
243  }
244  }
245  return nullptr;
246  }
247 
248  // Try to take a shared free list.
249  static bool _TakeSharedFreeList(_FreeList &out) {
250  return _sharedFreeLists->try_pop(out);
251  }
252 
253  // Give a free list to be shared by other threads.
254  static void _ShareFreeList(_FreeList &in) {
255  _sharedFreeLists->push(in);
256  in = {};
257  }
258 
259  // Reserve a new span of pool space.
260  static inline void _ReserveSpan(_PoolSpan &out);
261 
262  static constexpr int NumRegions = 1 << RegionBits;
263 
264  using _ThreadData = Sdf_FastThreadLocalBase<_PerThreadData>;
265  SDF_API static _ThreadData _threadData;
266  SDF_API static char *_regionStarts[NumRegions+1];
267  SDF_API static std::atomic<_RegionState> _regionState;
268  SDF_API static TfStaticData<tbb::concurrent_queue<_FreeList>> _sharedFreeLists;
269 };
270 
271 PXR_NAMESPACE_CLOSE_SCOPE
272 
273 #endif // SDF_POOL_H
Create or return a previously created object instance of global data.
Definition: staticData.h:113
void swap(UsdStageLoadRules &l, UsdStageLoadRules &r)
Swap the contents of rules l and r.
VT_API bool operator==(VtDictionary const &, VtDictionary const &)
Equality comparison.