All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
crateFile.h
1 //
2 // Copyright 2016 Pixar
3 //
4 // Licensed under the Apache License, Version 2.0 (the "Apache License")
5 // with the following modification; you may not use this file except in
6 // compliance with the Apache License and the following modification to it:
7 // Section 6. Trademarks. is deleted and replaced with:
8 //
9 // 6. Trademarks. This License does not grant permission to use the trade
10 // names, trademarks, service marks, or product names of the Licensor
11 // and its affiliates, except as required to comply with Section 4(c) of
12 // the License and to reproduce the content of the NOTICE file.
13 //
14 // You may obtain a copy of the Apache License at
15 //
16 // http://www.apache.org/licenses/LICENSE-2.0
17 //
18 // Unless required by applicable law or agreed to in writing, software
19 // distributed under the Apache License with the above modification is
20 // distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21 // KIND, either express or implied. See the Apache License for the specific
22 // language governing permissions and limitations under the Apache License.
23 //
24 #ifndef USD_CRATEFILE_H
25 #define USD_CRATEFILE_H
26 
27 #include "pxr/pxr.h"
28 #include "pxr/usd/usd/crateData.h"
29 
30 #include "shared.h"
31 #include "crateValueInliners.h"
32 
33 #include "pxr/base/arch/fileSystem.h"
34 #include "pxr/base/tf/token.h"
35 #include "pxr/base/vt/array.h"
36 #include "pxr/base/vt/value.h"
37 #include "pxr/base/work/arenaDispatcher.h"
38 #include "pxr/usd/ar/asset.h"
39 #include "pxr/usd/sdf/assetPath.h"
40 #include "pxr/usd/sdf/path.h"
41 #include "pxr/usd/sdf/types.h"
42 
43 #include <boost/container/flat_map.hpp>
44 #include <boost/functional/hash.hpp>
45 #include <boost/intrusive_ptr.hpp>
46 
47 #include <tbb/concurrent_unordered_set.h>
48 #include <tbb/spin_rw_mutex.h>
49 
50 #include <cstdint>
51 #include <iosfwd>
52 #include <string>
53 #include <type_traits>
54 #include <typeindex>
55 #include <unordered_map>
56 #include <utility>
57 #include <vector>
58 
59 PXR_NAMESPACE_OPEN_SCOPE
60 
61 namespace Usd_CrateFile {
62 
63 using std::make_pair;
64 using std::map;
65 using std::pair;
66 using std::string;
67 using std::unordered_map;
68 using std::tuple;
69 using std::vector;
70 
71 using ArAssetSharedPtr = std::shared_ptr<ArAsset>;
72 
73 // Trait indicating trivially copyable types, a hack since gcc doesn't yet
74 // implement is_trivially_copyable correctly.
75 template <class T> struct _IsBitwiseReadWrite;
76 
77 enum class TypeEnum : int32_t;
78 
79 // Value in file representation. Consists of a 2 bytes of type information
80 // (type enum value, array bit, and inlined-value bit) and 6 bytes of data.
81 // If possible, we attempt to store certain values directly in the local
82 // data, such as ints, floats, enums, and special-case values of other types
83 // (zero vectors, identity matrices, etc). For values that aren't stored
84 // inline, the 6 data bytes are the offset from the start of the file to the
85 // value's location.
86 struct ValueRep {
87 
88  friend class CrateFile;
89 
90  ValueRep() = default;
91 
92  explicit constexpr ValueRep(uint64_t data) : data(data) {}
93 
94  constexpr ValueRep(TypeEnum t,
95  bool isInlined, bool isArray, uint64_t payload)
96  : data(_Combine(t, isInlined, isArray, payload)) {}
97 
98  static const uint64_t _IsArrayBit = 1ull << 63;
99  static const uint64_t _IsInlinedBit = 1ull << 62;
100  static const uint64_t _IsCompressedBit = 1ull << 61;
101 
102  static const uint64_t _PayloadMask = ((1ull << 48) - 1);
103 
104  inline bool IsArray() const { return data & _IsArrayBit; }
105  inline void SetIsArray() { data |= _IsArrayBit; }
106 
107  inline bool IsInlined() const { return data & _IsInlinedBit; }
108  inline void SetIsInlined() { data |= _IsInlinedBit; }
109 
110  inline bool IsCompressed() const { return data & _IsCompressedBit; }
111  inline void SetIsCompressed() { data |= _IsCompressedBit; }
112 
113  inline TypeEnum GetType() const {
114  return static_cast<TypeEnum>((data >> 48) & 0xFF);
115  }
116  inline void SetType(TypeEnum t) {
117  data &= ~(0xFFull << 48); // clear type byte in data.
118  data |= (static_cast<uint64_t>(t) << 48); // set it.
119  }
120 
121  inline uint64_t GetPayload() const {
122  return data & _PayloadMask;
123  }
124 
125  inline void SetPayload(uint64_t payload) {
126  data &= ~_PayloadMask; // clear existing payload.
127  data |= payload & _PayloadMask;
128  }
129 
130  inline uint64_t GetData() const { return data; }
131 
132  bool operator==(ValueRep other) const {
133  return data == other.data;
134  }
135  bool operator!=(ValueRep other) const {
136  return !(*this == other);
137  }
138 
139  friend inline size_t hash_value(ValueRep v) {
140  return static_cast<size_t>(v.data);
141  }
142 
143  friend std::ostream &operator<<(std::ostream &o, ValueRep rep);
144 
145 private:
146  static constexpr uint64_t
147  _Combine(TypeEnum t, bool isInlined, bool isArray, uint64_t payload) {
148  return (isArray ? _IsArrayBit : 0) |
149  (isInlined ? _IsInlinedBit : 0) |
150  (static_cast<uint64_t>(t) << 48) |
151  (payload & _PayloadMask);
152  }
153 
154  uint64_t data;
155 };
156 template <> struct _IsBitwiseReadWrite<ValueRep> : std::true_type {};
157 
158 struct TimeSamples {
159  typedef Usd_Shared<vector<double>> SharedTimes;
160 
161  TimeSamples() : valueRep(0), valuesFileOffset(0) {}
162 
163  bool IsInMemory() const {
164  return valueRep.GetData() == 0;
165  }
166 
167  // Original rep from file if read from file. This will have GetData()
168  // == 0 if not from file or if the samples have been modified.
169  ValueRep valueRep;
170 
171  // Sample times.
172  SharedTimes times;
173 
174  // Sample values, but only if we have any in-memory values. Otherwise we
175  // read from the file via valuesFileOffset.
176  vector<VtValue> values;
177 
178  // Nonzero if we're looking at values in-file.
179  int64_t valuesFileOffset;
180 
181  // Note that equality does a very shallow equality check since otherwise
182  // we'd have to pull all the values from the file.
183  bool operator==(TimeSamples const &other) const {
184  return valueRep == other.valueRep &&
185  times == other.times &&
186  values == other.values &&
187  valuesFileOffset == other.valuesFileOffset;
188  }
189 
190  inline void swap(TimeSamples &other) {
191  std::swap(valueRep, other.valueRep);
192  times.swap(other.times);
193  values.swap(other.values);
194  std::swap(valuesFileOffset, other.valuesFileOffset);
195  }
196 
197  friend size_t hash_value(TimeSamples const &ts) {
198  size_t h = 0;
199  boost::hash_combine(h, ts.valueRep);
200  boost::hash_combine(h, ts.times);
201  boost::hash_combine(h, ts.values);
202  boost::hash_combine(h, ts.valuesFileOffset);
203  return h;
204  }
205 
206  friend std::ostream &
207  operator<<(std::ostream &os, TimeSamples const &samples);
208 
209  friend inline void swap(TimeSamples &l, TimeSamples &r) { l.swap(r); }
210 };
211 
212 // Value type enum.
213 enum class TypeEnum {
214  Invalid = 0,
215 #define xx(ENUMNAME, ENUMVALUE, _unused1, _unused2) \
216  ENUMNAME = ENUMVALUE,
217 
218 #include "crateDataTypes.h"
219 
220 #undef xx
221  NumTypes
222 };
223 
224 // Index base class. Used to index various tables. Deriving adds some
225 // type-safety so we don't accidentally use one kind of index with the wrong
226 // kind of table.
227 struct Index {
228  Index() : value(~0) {}
229  explicit Index(uint32_t value) : value(value) {}
230  bool operator==(const Index &other) const { return value == other.value; }
231  bool operator!=(const Index &other) const { return !(*this == other); }
232  uint32_t value;
233 };
234 
235 inline size_t hash_value(const Index &i) { return i.value; }
236 
237 std::ostream &operator<<(std::ostream &os, const Index &i);
238 
239 // Various specific indexes.
240 struct FieldIndex : Index { using Index::Index; };
241 struct FieldSetIndex : Index { using Index::Index; };
242 struct PathIndex : Index { using Index::Index; };
243 struct StringIndex : Index { using Index::Index; };
244 struct TokenIndex : Index { using Index::Index; };
245 
246 constexpr size_t _SectionNameMaxLength = 15;
247 
248 // Compile time constant section names, enforces max length.
249 struct _SectionName {
250  template <size_t N>
251  constexpr _SectionName(char const (&a)[N]) : _p(a), _size(N-1) {
252  static_assert(N <= _SectionNameMaxLength,
253  "Section name cannot exceed _SectionNameMaxLength");
254  }
255  constexpr size_t size() const { return _size; }
256  constexpr char const *c_str() const { return _p; }
257  constexpr bool operator==(_SectionName other) const {
258  return _p == other._p; }
259  constexpr bool operator!=(_SectionName other) const {
260  return _p != other._p; }
261  bool operator==(char const *otherStr) const {
262  return !strcmp(_p, otherStr); }
263 private:
264  const char * const _p;
265  const size_t _size;
266 };
267 
268 struct _Hasher {
269  template <class T>
270  inline size_t operator()(const T &val) const {
271  return boost::hash<T>()(val);
272  }
273 };
274 
275 class CrateFile
276 {
277 public:
278  struct Version;
279 
280 private:
281  // A move-only helper struct to represent a range of data in a FILE*, with
282  // optional "ownership" of that FILE* (i.e. responsibility to fclose upon
283  // destruction).
284  struct _FileRange {
285  _FileRange() = default;
286  _FileRange(FILE *file, int64_t startOffset,
287  int64_t length, bool hasOwnership)
288  : file(file)
289  , startOffset(startOffset)
290  , length(file && length == -1 ?
291  ArchGetFileLength(file) - startOffset : length)
292  , hasOwnership(hasOwnership) {}
293  ~_FileRange();
294  _FileRange(_FileRange &&other)
295  : file(other.file)
296  , startOffset(other.startOffset)
297  , length(other.length)
298  , hasOwnership(other.hasOwnership) {
299  other.file = nullptr;
300  }
301  _FileRange &operator=(_FileRange &&other) {
302  if (this != &other) {
303  file = other.file;
304  startOffset = other.startOffset;
305  length = other.length;
306  hasOwnership = other.hasOwnership;
307  other.file = nullptr;
308  }
309  return *this;
310  }
311 
312  explicit operator bool() const {
313  return file;
314  }
315 
316  int64_t GetLength() const { return length; }
317 
318  FILE *file = nullptr;
319  int64_t startOffset = 0;
320  int64_t length = 0;
321  bool hasOwnership = false;
322  };
323 
324  // This structure pulls together the underlying ArchMutableFileMapping
325  // representing the memory-mapped file plus all the information we need to
326  // support "zero-copy" arrays where we create VtArray instances that point
327  // directly into mapped memory. It gets complicated because the
328  // external-facing SdfLayer semantics are strict value semantics. In
329  // particular, we need to support the case where a client gets a VtArray out
330  // of a .usdc file, then destroys the layer object, and even mutates the
331  // file on disk, and the fetched array has to behave correctly.
332  struct _FileMapping {
333 
334  // This is a foreign data source for VtArray that refers into a
335  // memory-mapped region, and shares in the lifetime of the mapping.
336  struct ZeroCopySource : public Vt_ArrayForeignDataSource {
337  explicit ZeroCopySource(
338  CrateFile::_FileMapping *m, void *addr, size_t numBytes);
339 
340  // XXX --------------------------------
341  // Hack for tbb bug -- types in tbb::concurrent_unordered_set
342  // must be copy constructible until version 2017 update 1. Remove
343  // this once we're on or past that version of tbb.
344  ZeroCopySource(ZeroCopySource const &other)
345  : Vt_ArrayForeignDataSource(other._detachedFn, other._refCount)
346  , _mapping(other._mapping)
347  , _addr(other._addr)
348  , _numBytes(other._numBytes)
349  {}
350  // XXX --------------------------------
351 
352  bool operator==(ZeroCopySource const &other) const;
353  bool operator!=(ZeroCopySource const &other) const {
354  return !(*this == other);
355  }
356  friend size_t tbb_hasher(ZeroCopySource const &z) {
357  size_t seed = reinterpret_cast<uintptr_t>(z._addr);
358  boost::hash_combine(seed, z._numBytes);
359  return seed;
360  }
361 
362  // Return true if the refcount is nonzero.
363  bool IsInUse() const { return _refCount; }
364 
365  // Increment count and return true if it went from 0 to 1.
366  bool NewRef() {
367  return _refCount.fetch_add(1, std::memory_order_relaxed) == 0;
368  }
369 
370  // Return the address this source refers to.
371  void *GetAddr() const { return _addr; }
372 
373  // Return the number of bytes this source refers to.
374  size_t GetNumBytes() const { return _numBytes; }
375 
376  private:
377  // Callback for VtArray foreign data source.
378  static void _Detached(Vt_ArrayForeignDataSource *selfBase);
379 
380  _FileMapping *_mapping;
381  void *_addr;
382  size_t _numBytes;
383  };
384  friend struct ZeroCopySource;
385 
386  _FileMapping() : _refCount(0) {};
387 
388  explicit _FileMapping(ArchMutableFileMapping mapping, int64_t offset=0,
389  int64_t length=-1)
390  : _refCount(0)
391  , _mapping(std::move(mapping))
392  , _start(_mapping.get() + offset)
393  , _length(length == -1 ?
394  ArchGetFileMappingLength(_mapping) : length) {}
395 
396  // Add an an externally referenced page range.
397  ZeroCopySource *
398  AddRangeReference(void *addr, size_t numBytes);
399 
400  // "Silent-store" to touch outstanding page ranges to detach them in the
401  // copy-on-write sense from their file backing and make them
402  // swap-backed. No new page ranges can be added once this is invoked.
403  void DetachReferencedRanges();
404 
405  // Return the start address of the mapped file content. Note that due
406  // to having usdc files embedded into other files (like usdz files) the
407  // map start address is NOT guaranteed to be page-aligned.
408  char *GetMapStart() const { return _start; }
409 
410  // Return the length of the relevant content range in the mapping.
411  size_t GetLength() const { return _length; }
412 
413  private:
414  friend class CrateFile;
415 
416  // This class is managed by a combination of boost::intrusive_ptr and
417  // manual reference counting -- see explicit calls to
418  // intrusive_ptr_add_ref/release in the .cpp file.
419  friend inline void
420  intrusive_ptr_add_ref(_FileMapping const *m) {
421  m->_refCount.fetch_add(1, std::memory_order_relaxed);
422  }
423  friend inline void
424  intrusive_ptr_release(_FileMapping const *m) {
425  if (m->_refCount.fetch_sub(1, std::memory_order_release) == 1) {
426  std::atomic_thread_fence(std::memory_order_acquire);
427  delete m;
428  }
429  }
430 
431  mutable std::atomic<size_t> _refCount { 0 };
432  ArchMutableFileMapping _mapping;
433  char *_start;
434  int64_t _length;
435  tbb::concurrent_unordered_set<ZeroCopySource> _outstandingRanges;
436  };
437  using _FileMappingIPtr = boost::intrusive_ptr<_FileMapping>;
438 
440 
441  // _BootStrap structure. Appears at end of file, houses version, file
442  // identifier string and offset to _TableOfContents.
443  struct _BootStrap {
444  _BootStrap();
445  explicit _BootStrap(Version const &);
446  uint8_t ident[8]; // "PXR-USDC"
447  uint8_t version[8]; // 0: major, 1: minor, 2: patch, rest unused.
448  int64_t tocOffset;
449  int64_t _reserved[8];
450  };
451 
452  struct _Section {
453  _Section() { memset(this, 0, sizeof(*this)); }
454  _Section(char const *name, int64_t start, int64_t size);
455  char name[_SectionNameMaxLength+1];
456  int64_t start, size;
457  };
458 
459  struct _TableOfContents {
460  _Section const *GetSection(_SectionName) const;
461  int64_t GetMinimumSectionStart() const;
462  vector<_Section> sections;
463  };
464 
465 public:
466  friend struct ValueRep;
467  friend struct TimeSamples;
468 
469  typedef std::pair<TfToken, VtValue> FieldValuePair;
470 
471  struct Field {
472  // This padding field accounts for a bug in an earlier implementation,
473  // where both this class and its first member derived an empty base
474  // class. The standard requires that those not have the same address so
475  // GCC & clang inserted 4 bytes for this class's empty base, causing the
476  // first member to land at offset 4. Porting to MSVC revealed this,
477  // since MSVC didn't implement this correctly and the first member
478  // landed at offset 0. To fix this, we've removed the empty base and
479  // inserted our own 4 byte padding to ensure the layout comes out the
480  // same everywhere. This doesn't actually change the overall structure
481  // size since the first member is 4 bytes and the second is 8. It's
482  // still 16 bytes however you slice it.
483  uint32_t _unused_padding_;
484 
485  Field() {}
486  Field(TokenIndex ti, ValueRep v) : tokenIndex(ti), valueRep(v) {}
487  bool operator==(const Field &other) const {
488  return tokenIndex == other.tokenIndex &&
489  valueRep == other.valueRep;
490  }
491  friend size_t hash_value(const Field &f) {
492  _Hasher h;
493  size_t result = h(f.tokenIndex);
494  boost::hash_combine(result, f.valueRep);
495  return result;
496  }
497  TokenIndex tokenIndex;
498  ValueRep valueRep;
499  };
500 
501  struct Spec_0_0_1;
502 
503  struct Spec {
504  Spec() {}
505  Spec(PathIndex pi, SdfSpecType type, FieldSetIndex fsi)
506  : pathIndex(pi), fieldSetIndex(fsi), specType(type) {}
507  Spec(Spec_0_0_1 const &);
508  bool operator==(const Spec &other) const {
509  return pathIndex == other.pathIndex &&
510  fieldSetIndex == other.fieldSetIndex &&
511  specType == other.specType;
512  }
513  bool operator!=(const Spec &other) const {
514  return !(*this == other);
515  }
516  friend size_t hash_value(Spec const &s) {
517  _Hasher h;
518  size_t result = h(s.pathIndex);
519  boost::hash_combine(result, s.fieldSetIndex);
520  boost::hash_combine(result, s.specType);
521  return result;
522  }
523  PathIndex pathIndex;
524  FieldSetIndex fieldSetIndex;
525  SdfSpecType specType;
526  };
527 
528  struct Spec_0_0_1 {
529  // This padding field accounts for a bug in this earlier implementation,
530  // where both this class and its first member derived an empty base
531  // class. The standard requires that those not have the same address so
532  // GCC & clang inserted 4 bytes for this class's empty base, causing the
533  // first member to land at offset 4. Porting to MSVC revealed this,
534  // since MSVC didn't implement this correctly and the first member
535  // landed at offset 0. To fix this, we've removed the empty base and
536  // inserted our own 4 byte padding to ensure the layout comes out the
537  // same everywhere. File version 0.1.0 revises this structure to the
538  // smaller size with no padding.
539  uint32_t _unused_padding_;
540 
541  Spec_0_0_1() {}
542  Spec_0_0_1(PathIndex pi, SdfSpecType type, FieldSetIndex fsi)
543  : pathIndex(pi), fieldSetIndex(fsi), specType(type) {}
544  Spec_0_0_1(Spec const &);
545  bool operator==(const Spec_0_0_1 &other) const {
546  return pathIndex == other.pathIndex &&
547  fieldSetIndex == other.fieldSetIndex &&
548  specType == other.specType;
549  }
550  bool operator!=(const Spec_0_0_1 &other) const {
551  return !(*this == other);
552  }
553  friend size_t hash_value(Spec_0_0_1 const &s) {
554  _Hasher h;
555  size_t result = h(s.pathIndex);
556  boost::hash_combine(result, s.fieldSetIndex);
557  boost::hash_combine(result, s.specType);
558  return result;
559  }
560  PathIndex pathIndex;
561  FieldSetIndex fieldSetIndex;
562  SdfSpecType specType;
563  };
564 
565  ~CrateFile();
566 
567  static bool CanRead(string const &assetPath);
568  static TfToken const &GetSoftwareVersionToken();
569  TfToken GetFileVersionToken() const;
570 
571  static std::unique_ptr<CrateFile> CreateNew();
572 
573  // Return nullptr on failure.
574  static std::unique_ptr<CrateFile> Open(string const &assetPath);
575 
576  // Helper for saving to a file.
577  struct Packer {
578  // Move ctor/assign.
579  Packer(Packer &&);
580  Packer &operator=(Packer &&);
581 
582  // Save the contents to disk.
583  ~Packer();
584 
585  // Pack an additional spec in the crate.
586  inline void PackSpec(const SdfPath &path, SdfSpecType type,
587  const std::vector<FieldValuePair> &fields) {
588  _crate->_AddSpec(path, type, fields);
589  }
590 
591  // Write remaining data and structural sections to disk to produce a
592  // complete file. Return true if the writing completed successfully,
593  // false otherwise.
594  bool Close();
595 
596  // Return true if this Packer is valid to use, false otherwise.
597  // Typically false when we failed to open the output file for writing.
598  explicit operator bool() const;
599 
600  private:
601  Packer(Packer const &) = delete;
602  Packer &operator=(Packer const &) = delete;
603 
604  friend class CrateFile;
605  explicit Packer(CrateFile *crate) : _crate(crate) {}
606  CrateFile *_crate;
607  };
608 
609  // Return true if this CrateFile object wasn't populated from a file, or if
610  // the given \p fileName is the file this object was populated from.
611  bool CanPackTo(string const &fileName) const;
612 
613  Packer StartPacking(string const &fileName);
614 
615  string const &GetAssetPath() const { return _assetPath; }
616 
617  inline Field const &
618  GetField(FieldIndex i) const { return _fields[i.value]; }
619 
620  inline vector<Field> const & GetFields() const { return _fields; }
621 
622  inline vector<FieldIndex> const &
623  GetFieldSets() const { return _fieldSets; }
624 
625  inline size_t GetNumUniqueFieldSets() const {
626  // Count field sets by counting terminators.
627  return count(_fieldSets.begin(), _fieldSets.end(), FieldIndex());
628  }
629 
630  inline SdfPath const &
631  GetPath(PathIndex i) const { return _paths[i.value]; }
632 
633  inline vector<SdfPath> const &GetPaths() const { return _paths; }
634 
635  inline vector<Spec> const &
636  GetSpecs() const { return _specs; }
637 
638  // Get all structural data out in \p outSpecs, \p outFields,
639  // \p outFieldSets, leave those data in this CrateFile empty.
640  inline void RemoveStructuralData(
641  vector<Spec> &outSpecs,
642  vector<Field> &outFields,
643  vector<FieldIndex> &outFieldSets) {
644  // We swap through temporaries to ensure we leave our structs empty.
645  { vector<Spec> tmp; tmp.swap(_specs); outSpecs.swap(tmp); }
646  { vector<Field> tmp; tmp.swap(_fields); outFields.swap(tmp); }
647  { vector<FieldIndex> tmp; tmp.swap(_fieldSets); outFieldSets.swap(tmp); }
648  }
649 
650  inline TfToken const &
651  GetToken(TokenIndex i) const { return _tokens[i.value]; }
652 
653  inline vector<TfToken> const &GetTokens() const { return _tokens; }
654 
655  inline std::string const &
656  GetString(StringIndex i) const {
657  return GetToken(_strings[i.value]).GetString();
658  }
659  inline vector<TokenIndex> const &GetStrings() const { return _strings; }
660 
661  vector<tuple<string, int64_t, int64_t>> GetSectionsNameStartSize() const;
662 
663  inline VtValue GetTimeSampleValue(TimeSamples const &ts, size_t i) const {
664  return ts.IsInMemory() ? ts.values[i] : _GetTimeSampleValueImpl(ts, i);
665  }
666 
667  // Make \p ts mutable in a way that can accommodate changing the size of ts.
668  inline void MakeTimeSampleTimesAndValuesMutable(TimeSamples &ts) const {
669  ts.times.MakeUnique();
670  MakeTimeSampleValuesMutable(ts);
671  }
672 
673  // Make \p ts mutable so that individual sample values may be modified, but
674  // not the number of samples.
675  inline void MakeTimeSampleValuesMutable(TimeSamples &ts) const {
676  if (!ts.IsInMemory()) {
677  _MakeTimeSampleValuesMutableImpl(ts);
678  }
679  }
680 
681  void DebugPrint() const;
682 
683  inline VtValue UnpackValue(ValueRep rep) const {
684  VtValue ret;
685  _UnpackValue(rep, &ret);
686  return ret;
687  }
688 
689  std::type_info const &GetTypeid(ValueRep rep) const;
690 
691 private:
692  explicit CrateFile(bool useMmap);
693  CrateFile(string const &assetPath, string const &fileName,
694  _FileMappingIPtr mapStart, ArAssetSharedPtr const &asset);
695  CrateFile(string const &assetPath, string const &fileName,
696  _FileRange &&inputFile, ArAssetSharedPtr const &asset);
697  CrateFile(string const &assetPath, ArAssetSharedPtr const &asset);
698 
699  CrateFile(CrateFile const &) = delete;
700  CrateFile &operator=(CrateFile const &) = delete;
701 
702  void _InitMMap();
703  void _InitPread();
704  void _InitAsset();
705 
706  static _FileMappingIPtr
707  _MmapFile(char const *fileName, FILE *file);
708  static _FileMappingIPtr
709  _MmapAsset(char const *fileName, ArAssetSharedPtr const &asset);
710 
711  class _Writer;
712  class _BufferedOutput;
713  class _ReaderBase;
714  template <class ByteStream> class _Reader;
715 
716  template <class ByteStream>
717  _Reader<ByteStream> _MakeReader(ByteStream src) const;
718 
719  template <class Fn>
720  void _WriteSection(
721  _Writer &w, _SectionName name, _TableOfContents &toc, Fn writeFn) const;
722 
723  void _AddDeferredSpecs();
724 
725  bool _Write();
726 
727  void _AddSpec(const SdfPath &path, SdfSpecType type,
728  const std::vector<FieldValuePair> &fields);
729 
730  VtValue _GetTimeSampleValueImpl(TimeSamples const &ts, size_t i) const;
731  void _MakeTimeSampleValuesMutableImpl(TimeSamples &ts) const;
732 
733  void _WriteFields(_Writer &w);
734  void _WriteFieldSets(_Writer &w);
735  void _WritePaths(_Writer &w);
736  void _WriteSpecs(_Writer &w);
737 
738  template <class Iter>
739  Iter _WritePathTree(_Writer &w, Iter cur, Iter end);
740 
741  template <class Container>
742  void _WriteCompressedPathData(_Writer &w, Container const &pathData);
743 
744  template <class Iter>
745  Iter _BuildCompressedPathDataRecursive(
746  size_t &curIndex, Iter cur, Iter end,
747  vector<uint32_t> &pathIndexes,
748  vector<int32_t> &elementTokenIndexes,
749  vector<int32_t> &jumps);
750 
751  inline void _WriteTokens(_Writer &w);
752 
753  template <class Reader>
754  void _ReadStructuralSections(Reader src, int64_t fileSize);
755 
756  template <class ByteStream>
757  static _BootStrap _ReadBootStrap(ByteStream src, int64_t fileSize);
758 
759  template <class Reader>
760  _TableOfContents _ReadTOC(Reader src, _BootStrap const &b) const;
761 
762  template <class Reader> void _PrefetchStructuralSections(Reader src) const;
763  template <class Reader> void _ReadFieldSets(Reader src);
764  template <class Reader> void _ReadFields(Reader src);
765  template <class Reader> void _ReadSpecs(Reader src);
766  template <class Reader> void _ReadStrings(Reader src);
767  template <class Reader> void _ReadTokens(Reader src);
768  template <class Reader> void _ReadPaths(Reader src);
769  template <class Header, class Reader>
770  void _ReadPathsImpl(Reader reader,
771  WorkArenaDispatcher &dispatcher,
772  SdfPath parentPath=SdfPath());
773  template <class Reader>
774  void _ReadCompressedPaths(Reader reader,
775  WorkArenaDispatcher &dispatcher);
776  void _BuildDecompressedPathsImpl(
777  std::vector<uint32_t> const &pathIndexes,
778  std::vector<int32_t> const &elementTokenIndexes,
779  std::vector<int32_t> const &jumps,
780  size_t curIndex,
781  SdfPath parentPath,
782  WorkArenaDispatcher &dispatcher);
783 
784  void _ReadRawBytes(int64_t start, int64_t size, char *buf) const;
785 
786  PathIndex _AddPath(const SdfPath &path);
787  FieldSetIndex _AddFieldSet(const std::vector<FieldIndex> &fieldIndexes);
788  FieldIndex _AddField(const FieldValuePair &fv);
789  TokenIndex _AddToken(const TfToken &token);
790  TokenIndex _GetIndexForToken(const TfToken &token) const;
791  StringIndex _AddString(const string &str);
792 
793 
795 
796  // Base class, to have a pointer type.
797  struct _ValueHandlerBase;
798  template <class, class=void> struct _ScalarValueHandlerBase;
799  template <class, class=void> struct _ArrayValueHandlerBase;
800  template <class> struct _ValueHandler;
801 
802  friend struct _ValueHandlerBase;
803  template <class, class> friend struct _ScalarValueHandlerBase;
804  template <class, class> friend struct _ArrayValueHandlerBase;
805  template <class> friend struct _ValueHandler;
806 
807  template <class T> inline _ValueHandler<T> &_GetValueHandler();
808  template <class T> inline _ValueHandler<T> const &_GetValueHandler() const;
809 
810  template <class T> inline ValueRep _PackValue(T const &v);
811  template <class T> inline ValueRep _PackValue(VtArray<T> const &v);
812  ValueRep _PackValue(VtValue const &v);
813 
814  template <class T> void _UnpackValue(ValueRep rep, T *out) const;
815  template <class T> void _UnpackValue(ValueRep rep, VtArray<T> *out) const;
816  void _UnpackValue(ValueRep rep, VtValue *result) const;
817 
818  // Functions that populate the value read/write functions.
819  template <class T> void _DoTypeRegistration();
820  void _DoAllTypeRegistrations();
821  void _DeleteValueHandlers();
822  void _ClearValueHandlerDedupTables();
823 
824  static bool _IsKnownSection(char const *name);
825 
826  struct _PackingContext;
827 
829  // Member data.
830 
832  // These tables need not persist, they must be fully regenerated from
833  // in-memory data upon Save, for example.
834 
835  // An index into the path list, plus a range of fields.
836  vector<Spec> _specs;
837 
838  // Deferred specs we write separately at the end. This is for specs that
839  // have fields that we may not know how to write until we've looked at other
840  // specs (e.g. SdfPayload version upgrades) or spces with time samples that
841  // we write time-by-time so that time-sampled data is collocated by time.
842  struct _DeferredSpec {
843  _DeferredSpec() = default;
844  _DeferredSpec(PathIndex p, SdfSpecType t,
845  vector<FieldIndex> &&of,
846  vector<FieldValuePair> &&dof,
847  vector<pair<TfToken, TimeSamples>> &&ts)
848  : path(p)
849  , specType(t)
850  , ordinaryFields(std::move(of))
851  , deferredOrdinaryFields(std::move(dof))
852  , timeSampleFields(std::move(ts)) {}
853  PathIndex path;
854  SdfSpecType specType;
855  vector<FieldIndex> ordinaryFields;
856  vector<FieldValuePair> deferredOrdinaryFields;
857  vector<pair<TfToken, TimeSamples>> timeSampleFields;
858  };
859  vector<_DeferredSpec> _deferredSpecs;
860 
861  // All unique fields.
862  vector<Field> _fields;
863 
864  // A vector of groups of fields, invalid-index terminated.
865  vector<FieldIndex> _fieldSets;
866 
868  // These tables must persist, since deferred values in the file may refer to
869  // their contents by index.
870 
871  // All unique paths.
872  vector<SdfPath> _paths;
873 
874  // TfToken to token index.
875  vector<TfToken> _tokens;
876 
877  // std::string to string index.
878  vector<TokenIndex> _strings;
879 
880  // ValueRep to vector<double> for deduplicating timesamples times in memory.
881  mutable unordered_map<
882  ValueRep, TimeSamples::SharedTimes, _Hasher> _sharedTimes;
883  mutable tbb::spin_rw_mutex _sharedTimesMutex;
884 
885  // functions to write VtValues to file by type.
886  boost::container::flat_map<
887  std::type_index, std::function<ValueRep (VtValue const &)>>
888  _packValueFunctions;
889 
890  // functions to read VtValues from file by type.
891  std::function<void (ValueRep, VtValue *)>
892  _unpackValueFunctionsPread[static_cast<int>(TypeEnum::NumTypes)];
893 
894  std::function<void (ValueRep, VtValue *)>
895  _unpackValueFunctionsMmap[static_cast<int>(TypeEnum::NumTypes)];
896 
897  std::function<void (ValueRep, VtValue *)>
898  _unpackValueFunctionsAsset[static_cast<int>(TypeEnum::NumTypes)];
899 
900  _ValueHandlerBase *_valueHandlers[static_cast<int>(TypeEnum::NumTypes)];
901 
903 
904  // Temporary -- only valid during Save().
905  std::unique_ptr<_PackingContext> _packCtx;
906 
907  _TableOfContents _toc; // only valid if we have read an asset.
908  _BootStrap _boot; // only valid if we have read an asset.
909 
910 
911  // If we're reading data from an mmap'd file, then _mmapSrc will be non-null
912  // and _preadSrc & _assetSrc will be null. Otherwise if we're reading data
913  // by pread()ing from a file obtained from an ArAsset, then _assetSrc is
914  // non-null and _preadSrc observes (but does not own/fclose) the result of
915  // _assetSrc->GetFileUnsafe(). If a Save operation is completed (via
916  // StartPacking) in this state, then _preadSrc will become an owning FILE*,
917  // and _assetSrc will be nullptr. Otherwise if _assetSrc is non-null and
918  // both _mmapSrc and _preadSrc are null, we read data from the _assetSrc via
919  // ArAsset::Read(). If all three are null then this structure was not
920  // populated from an asset.
921  _FileMappingIPtr _mmapSrc;
922  _FileRange _preadSrc;
923  ArAssetSharedPtr _assetSrc;
924 
925  std::string _assetPath; // Empty if this file data is in-memory only.
926  std::string _fileReadFrom; // The file this object was populate from, if it
927  // was populated from a file.
928 
929  std::unique_ptr<char []> _debugPageMap; // Debug page access map, see
930  // USDC_DUMP_PAGE_MAPS.
931 
932  const bool _useMmap; // If true, try to use mmap for reads, otherwise try to
933  // use pread, otherwise fall back to generalized
934  // ArAsset::Read()s.
935 };
936 
937 template <>
938 struct _IsBitwiseReadWrite<CrateFile::_BootStrap> : std::true_type {};
939 
940 template <>
941 struct _IsBitwiseReadWrite<CrateFile::_Section> : std::true_type {};
942 
943 template <>
944 struct _IsBitwiseReadWrite<CrateFile::Field> : std::true_type {};
945 
946 template <>
947 struct _IsBitwiseReadWrite<CrateFile::Spec> : std::true_type {};
948 
949 template <>
950 struct _IsBitwiseReadWrite<CrateFile::Spec_0_0_1> : std::true_type {};
951 
952 
953 } // Usd_CrateFile
954 
955 
956 PXR_NAMESPACE_CLOSE_SCOPE
957 
958 #endif // USD_CRATEFILE_H
std::string const & GetString() const
Return the string that this token represents.
Definition: token.h:209
SdfSpecType
An enum that specifies the type of an object.
Definition: types.h:91
Token for efficient comparison, assignment, and hashing of known strings.
Definition: token.h:89
void swap(UsdStageLoadRules &l, UsdStageLoadRules &r)
Swap the contents of rules l and r.
ARCH_API int64_t ArchGetFileLength(const char *fileName)
Return the length of a file in bytes.
size_t ArchGetFileMappingLength(ArchConstFileMapping const &m)
Return the length of an ArchConstFileMapping.
Definition: fileSystem.h:321
A path value used to locate objects in layers or scenegraphs.
Definition: path.h:287
GF_API std::ostream & operator<<(std::ostream &, const GfBBox3d &)
Output a GfBBox3d using the format [(range) matrix zeroArea].
This is a specialization of the WorkDispatcher that uses an isolated arena to Run() all its tasks in...
VT_API bool operator==(VtDictionary const &, VtDictionary const &)
Equality comparison.
Provides a container which may hold any type, and provides introspection and iteration over array typ...
Definition: value.h:182