Debian Codegraph @

common/serialization_utils.h

raw
1 /*
2 * serialization_utils.h
3 * ---------------------
4 * Purpose: Serializing data to and from MPTM files.
5 * Notes : (currently none)
6 * Authors: OpenMPT Devs
7 * The OpenMPT source code is released under the BSD license. Read LICENSE for more details.
8 */
9
10
11 #pragma once
12
13 #include "openmpt/all/BuildSettings.hpp"
14
15 #include "mpt/base/alloc.hpp"
16 #include "mpt/io/io.hpp"
17 #include "mpt/io/io_stdstream.hpp"
18 #include "openmpt/base/Endian.hpp"
19
20 #include "../common/mptBaseTypes.h"
21
22 #include <algorithm>
23 #include <bitset>
24 #include <ios>
25 #include <iosfwd>
26 #include <limits>
27 #include <string>
28 #include <vector>
29
30 #include <istream>
31 #include <ostream>
32
33 #include <cstddef>
34 #include <cstring>
35
36 OPENMPT_NAMESPACE_BEGIN
37
38
39 #ifdef MPT_ALL_LOGGING
40 #define SSB_LOGGING
41 #endif
42
43
44 #ifdef SSB_LOGGING
45 #define SSB_LOG(x) MPT_LOG_GLOBAL(LogDebug, "serialization", (x))
46 #else
47 #define SSB_LOG(x) do { } while(0)
48 #endif
49
50
51 namespace srlztn //SeRiaLiZaTioN
52 {
53
54
55 constexpr inline std::size_t invalidDatasize = static_cast<std::size_t>(0) - 1;
56
57
58 enum class StatusLevel : uint8
59 {
60 Failure = 0x2,
61 Note = 0x1,
62 None = 0x0,
63 };
64
65 enum class StatusMessages : uint32
66 {
67 None = 0,
68
69 // Read notes and warnings.
70 SNR_ZEROENTRYCOUNT = 0x00'00'00'01,
71 SNR_NO_ENTRYIDS_WITH_CUSTOMID_DEFINED = 0x00'00'00'02,
72 SNR_LOADING_OBJECT_WITH_LARGER_VERSION = 0x00'00'00'04,
73
74 // Read failures.
75 SNR_BADSTREAM_AFTER_MAPHEADERSEEK = 0x00'00'01'00,
76 SNR_STARTBYTE_MISMATCH = 0x00'00'02'00,
77 SNR_BADSTREAM_AT_MAP_READ = 0x00'00'04'00,
78 SNR_INSUFFICIENT_STREAM_OFFTYPE = 0x00'00'08'00,
79 SNR_OBJECTCLASS_IDMISMATCH = 0x00'00'10'00,
80 SNR_TOO_MANY_ENTRIES_TO_READ = 0x00'00'20'00,
81 SNR_INSUFFICIENT_RPOSTYPE = 0x00'00'40'00,
82
83 // Write failures.
84 SNW_INSUFFICIENT_FIXEDSIZE = 0x00'01'00'00,
85 SNW_CHANGING_IDSIZE_WITH_FIXED_IDSIZESETTING = 0x00'02'00'00,
86 SNW_DATASIZETYPE_OVERFLOW = 0x00'04'00'00,
87 SNW_MAX_WRITE_COUNT_REACHED = 0x00'08'00'00,
88 SNW_INSUFFICIENT_DATASIZETYPE = 0x00'10'00'00,
89
90 SNRW_BADGIVEN_STREAM = 0x01'00'00'00,
91 };
92
93 struct Status
94 {
95 StatusLevel level = StatusLevel::None;
96 StatusMessages messages = StatusMessages::None;
97 };
98
99 constexpr inline Status SNR_ZEROENTRYCOUNT = {StatusLevel::Note, StatusMessages::SNR_ZEROENTRYCOUNT};
100 constexpr inline Status SNR_NO_ENTRYIDS_WITH_CUSTOMID_DEFINED = {StatusLevel::Note, StatusMessages::SNR_NO_ENTRYIDS_WITH_CUSTOMID_DEFINED};
101 constexpr inline Status SNR_LOADING_OBJECT_WITH_LARGER_VERSION = {StatusLevel::Note, StatusMessages::SNR_LOADING_OBJECT_WITH_LARGER_VERSION};
102
103 constexpr inline Status SNR_BADSTREAM_AFTER_MAPHEADERSEEK = {StatusLevel::Failure, StatusMessages::SNR_BADSTREAM_AFTER_MAPHEADERSEEK};
104 constexpr inline Status SNR_STARTBYTE_MISMATCH = {StatusLevel::Failure, StatusMessages::SNR_STARTBYTE_MISMATCH};
105 constexpr inline Status SNR_BADSTREAM_AT_MAP_READ = {StatusLevel::Failure, StatusMessages::SNR_BADSTREAM_AT_MAP_READ};
106 constexpr inline Status SNR_INSUFFICIENT_STREAM_OFFTYPE = {StatusLevel::Failure, StatusMessages::SNR_INSUFFICIENT_STREAM_OFFTYPE};
107 constexpr inline Status SNR_OBJECTCLASS_IDMISMATCH = {StatusLevel::Failure, StatusMessages::SNR_OBJECTCLASS_IDMISMATCH};
108 constexpr inline Status SNR_TOO_MANY_ENTRIES_TO_READ = {StatusLevel::Failure, StatusMessages::SNR_TOO_MANY_ENTRIES_TO_READ};
109 constexpr inline Status SNR_INSUFFICIENT_RPOSTYPE = {StatusLevel::Failure, StatusMessages::SNR_INSUFFICIENT_RPOSTYPE};
110
111 constexpr inline Status SNW_INSUFFICIENT_FIXEDSIZE = {StatusLevel::Failure, StatusMessages::SNW_INSUFFICIENT_FIXEDSIZE};
112 constexpr inline Status SNW_CHANGING_IDSIZE_WITH_FIXED_IDSIZESETTING = {StatusLevel::Failure, StatusMessages::SNW_CHANGING_IDSIZE_WITH_FIXED_IDSIZESETTING};
113 constexpr inline Status SNW_DATASIZETYPE_OVERFLOW = {StatusLevel::Failure, StatusMessages::SNW_DATASIZETYPE_OVERFLOW};
114 constexpr inline Status SNW_MAX_WRITE_COUNT_REACHED = {StatusLevel::Failure, StatusMessages::SNW_MAX_WRITE_COUNT_REACHED};
115 constexpr inline Status SNW_INSUFFICIENT_DATASIZETYPE = {StatusLevel::Failure, StatusMessages::SNW_INSUFFICIENT_DATASIZETYPE};
116
117 constexpr inline Status SNRW_BADGIVEN_STREAM = {StatusLevel::Failure, StatusMessages::SNRW_BADGIVEN_STREAM};
118
119
120 enum : uint16
121 {
122 IdSizeVariable = std::numeric_limits<uint16>::max(),
123 IdSizeMaxFixedSize = (std::numeric_limits<uint8>::max() >> 1)
124 };
125
126 struct ReadEntry
127 {
128 std::size_t nIdpos = 0; // Index of id start in ID array.
129 std::streamoff rposStart = 0; // Entry start position.
130 std::size_t nSize = invalidDatasize; // Entry size.
131 uint16 nIdLength = 0; // Length of id.
132 };
133
134
135 enum Rwf
136 {
137 RwfWMapStartPosEntry, // Write. True to include data start pos entry to map.
138 RwfWMapSizeEntry, // Write. True to include data size entry to map.
139 RwfWMapDescEntry, // Write. True to include description entry to map.
140 RwfWVersionNum, // Write. True to include version numeric.
141 RwfRMapCached, // Read. True if map has been cached.
142 RwfRMapHasId, // Read. True if map has IDs
143 RwfRMapHasStartpos, // Read. True if map data start pos.
144 RwfRMapHasSize, // Read. True if map has entry size.
145 RwfRMapHasDesc, // Read. True if map has entry description.
146 RwfRTwoBytesDescChar, // Read. True if map description characters are two bytes.
147 RwfRHeaderIsRead, // Read. True when header is read.
148 RwfRwHasMap, // Read/write. True if map exists.
149
150 RwfNumFlags
151 };
152
153
154 template<class T>
155 inline void Binarywrite(std::ostream& oStrm, const T& data)
156 {
157 mpt::IO::WriteIntLE(oStrm, data);
158 }
159
160 template<>
161 inline void Binarywrite(std::ostream& oStrm, const float& data)
162 {
163 IEEE754binary32LE tmp = IEEE754binary32LE(data);
164 mpt::IO::Write(oStrm, tmp);
165 }
166
167 template<>
168 inline void Binarywrite(std::ostream& oStrm, const double& data)
169 {
170 IEEE754binary64LE tmp = IEEE754binary64LE(data);
171 mpt::IO::Write(oStrm, tmp);
172 }
173
174 template <class T>
175 inline void WriteItem(std::ostream& oStrm, const T& data)
176 {
177 static_assert(std::is_trivially_default_constructible<T>::value && std::is_trivially_copyable<T>::value, "");
178 Binarywrite(oStrm, data);
179 }
180
181 void WriteItemString(std::ostream& oStrm, const std::string &str);
182
183 template <>
184 inline void WriteItem<std::string>(std::ostream& oStrm, const std::string& str) {WriteItemString(oStrm, str);}
185
186
187 template<class T>
188 inline void Binaryread(std::istream& iStrm, T& data)
189 {
190 mpt::IO::ReadIntLE(iStrm, data);
191 }
192
193 template<>
194 inline void Binaryread(std::istream& iStrm, float& data)
195 {
196 IEEE754binary32LE tmp = IEEE754binary32LE(0.0f);
197 mpt::IO::Read(iStrm, tmp);
198 data = tmp;
199 }
200
201 template<>
202 inline void Binaryread(std::istream& iStrm, double& data)
203 {
204 IEEE754binary64LE tmp = IEEE754binary64LE(0.0);
205 mpt::IO::Read(iStrm, tmp);
206 data = tmp;
207 }
208
209 //Read only given number of bytes to the beginning of data; data bytes are memset to 0 before reading.
210 template <class T>
211 inline void Binaryread(std::istream& iStrm, T& data, const std::size_t bytecount)
212 {
213 mpt::IO::ReadBinaryTruncatedLE(iStrm, data, bytecount);
214 }
215
216 template <>
217 inline void Binaryread<float>(std::istream& iStrm, float& data, const std::size_t bytecount)
218 {
219 using T = IEEE754binary32LE;
220 mpt::IO::SeekRelative(iStrm, std::min(bytecount, sizeof(T)));
221 // There is not much we can sanely do for truncated floats,
222 // thus we ignore what we could read and return 0.
223 data = 0.0f;
224 }
225
226 template <>
227 inline void Binaryread<double>(std::istream& iStrm, double& data, const std::size_t bytecount)
228 {
229 using T = IEEE754binary64LE;
230 mpt::IO::SeekRelative(iStrm, std::min(bytecount, sizeof(T)));
231 // There is not much we can sanely do for truncated floats,
232 // thus we ignore what we could read and return 0.
233 data = 0.0;
234 }
235
236
237 template <class T>
238 inline void ReadItem(std::istream& iStrm, T& data, const std::size_t nSize)
239 {
240 static_assert(std::is_trivially_default_constructible<T>::value && std::is_trivially_copyable<T>::value, "");
241 if (nSize == sizeof(T) || nSize == invalidDatasize)
242 Binaryread(iStrm, data);
243 else
244 Binaryread(iStrm, data, nSize);
245 }
246
247 void ReadItemString(std::istream& iStrm, std::string& str, const std::size_t);
248
249 template <>
250 inline void ReadItem<std::string>(std::istream& iStrm, std::string& str, const std::size_t nSize)
251 {
252 ReadItemString(iStrm, str, nSize);
253 }
254
255
256
257 class ID
258 {
259 private:
260 std::string m_ID; // NOTE: can contain null characters ('\0')
261 public:
262 ID() = default;
263 ID(const std::string &id) : m_ID(id) { }
264 ID(const char *id) : m_ID(id ? id : "") { }
265 ID(const char * str, std::size_t len) : m_ID(str, len) { }
266 template <typename T>
267 static ID FromInt(const T &val)
268 {
269 static_assert(std::numeric_limits<T>::is_integer);
270 typename mpt::make_le<T>::type valle;
271 valle = val;
272 return ID(mpt::byte_cast<const char*>(mpt::as_raw_memory(valle).data()), mpt::as_raw_memory(valle).size());
273 }
274 #ifdef SSB_LOGGING
275 bool IsPrintable() const;
276 mpt::ustring AsString() const;
277 #endif
278 std::size_t GetSize() const
279 {
280 return m_ID.size();
281 }
282 mpt::span<const char> AsSpan() const
283 {
284 return mpt::as_span(m_ID);
285 }
286 bool operator == (const ID &other) const { return m_ID == other.m_ID; }
287 bool operator != (const ID &other) const { return m_ID != other.m_ID; }
288 };
289
290
291
292 class Ssb
293 {
294
295 protected:
296
297 Ssb();
298
299 public:
300
301 bool HasFailed() const
302 {
303 return (m_Status.level >= StatusLevel::Failure);
304 }
305
306 protected:
307
308 Status m_Status;
309
310 uint32 m_nFixedEntrySize; // Read/write: If > 0, data entries have given fixed size.
311
312 std::streamoff m_posStart; // Read/write: Stream position at the beginning of object.
313
314 uint16 m_nIdbytes; // Read/Write: Tells map ID entry size in bytes. If size is variable, value is IdSizeVariable.
315 std::size_t m_nCounter; // Read/write: Keeps count of entries written/read.
316
317 std::bitset<RwfNumFlags> m_Flags; // Read/write: Various flags.
318
319 protected:
320
321 enum : uint8 { s_DefaultFlagbyte = 0 };
322 static const char s_EntryID[3];
323
324 };
325
326
327
328 class SsbRead
329 : public Ssb
330 {
331
332 public:
333
334 using ReadIterator = std::vector<ReadEntry>::const_iterator;
335
336 SsbRead(std::istream& iStrm);
337
338 // Call this to begin reading: must be called before other read functions.
339 void BeginRead(const ID &id, const uint64& nVersion);
340
341 // After calling BeginRead(), this returns number of entries in the file.
342 std::size_t GetNumEntries() const {return m_nReadEntrycount;}
343
344 // Returns read iterator to the beginning of entries.
345 // The behaviour of read iterators is undefined if map doesn't
346 // contain entry ids or data begin positions.
347 ReadIterator GetReadBegin();
348
349 // Returns read iterator to the end(one past last) of entries.
350 ReadIterator GetReadEnd();
351
352 // Compares given id with read entry id
353 bool MatchesId(const ReadIterator& iter, const ID &id);
354
355 uint64 GetReadVersion() {return m_nReadVersion;}
356
357 // Read item using default read implementation.
358 template <class T>
359 bool ReadItem(T& obj, const ID &id) {return ReadItem(obj, id, srlztn::ReadItem<T>);}
360
361 // Read item using given function.
362 template <class T, class FuncObj>
363 bool ReadItem(T& obj, const ID &id, FuncObj);
364
365 // Read item using read iterator.
366 template <class T>
367 bool ReadIterItem(const ReadIterator& iter, T& obj) {return ReadIterItem(iter, obj, srlztn::ReadItem<T>);}
368 template <class T, class FuncObj>
369 bool ReadIterItem(const ReadIterator& iter, T& obj, FuncObj func);
370
371 private:
372
373 // Reads map to cache.
374 void CacheMap();
375
376 // Searches for entry with given ID. If found, returns pointer to corresponding entry, else
377 // returns nullptr.
378 const ReadEntry* Find(const ID &id);
379
380 // Called after reading an object.
381 void OnReadEntry(const ReadEntry* pE, const ID &id, const std::streamoff& posReadBegin);
382
383 void AddReadNote(const Status s);
384
385 #ifdef SSB_LOGGING
386 // Called after reading entry. pRe is a pointer to associated map entry if exists.
387 void LogReadEntry(const ReadEntry &pRe, const std::size_t nNum);
388 #endif
389
390 void ResetReadstatus();
391
392 private:
393
394 // mapData is a cache that facilitates faster access to the stored data
395 // without having to reparse on every access.
396 // Iterator invalidation in CacheMap() is not a problem because every code
397 // path that ever returns an iterator into mapData does CacheMap exactly once
398 // beforehand. Following calls use this already cached map. As the data is
399 // immutable when reading, there is no need to ever invalidate the cache and
400 // redo CacheMap().
401
402 std::istream& iStrm;
403
404 std::vector<char> m_Idarray; // Read: Holds entry ids.
405
406 std::vector<ReadEntry> mapData; // Read: Contains map information.
407 uint64 m_nReadVersion; // Read: Version is placed here when reading.
408 std::streamoff m_rposMapBegin; // Read: If map exists, rpos of map begin, else m_rposEndofHdrData.
409 std::streamoff m_posMapEnd; // Read: If map exists, map end position, else pos of end of hdrData.
410 std::streamoff m_posDataBegin; // Read: Data begin position.
411 std::streamoff m_rposEndofHdrData; // Read: rpos of end of header data.
412 std::size_t m_nReadEntrycount; // Read: Number of entries.
413
414 std::size_t m_nNextReadHint; // Read: Hint where to start looking for the next read entry.
415
416 };
417
418
419
420 class SsbWrite
421 : public Ssb
422 {
423
424 public:
425
426 SsbWrite(std::ostream& oStrm);
427
428 // Write header
429 void BeginWrite(const ID &id, const uint64& nVersion);
430
431 // Write item using default write implementation.
432 template <class T>
433 void WriteItem(const T& obj, const ID &id) {WriteItem(obj, id, &srlztn::WriteItem<T>);}
434
435 // Write item using given function.
436 template <class T, class FuncObj>
437 void WriteItem(const T& obj, const ID &id, FuncObj);
438
439 // Writes mapping.
440 void FinishWrite();
441
442 private:
443
444 // Called after writing an item.
445 void OnWroteItem(const ID &id, const std::streamoff& posBeforeWrite);
446
447 void AddWriteNote(const Status s);
448
449 #ifdef SSB_LOGGING
450 void LogWriteEntry(const ID &id,
451 const std::size_t nEntryNum,
452 const std::size_t nBytecount,
453 const std::streamoff rposStart);
454 #endif
455
456 // Writes mapping item to mapstream.
457 void WriteMapItem(const ID &id,
458 const std::streamoff& rposDataStart,
459 const std::size_t& nDatasize,
460 const std::string &pszDesc);
461
462 void ResetWritestatus()
463 {
464 m_Status = Status{};
465 }
466
467 private:
468
469 std::ostream& oStrm;
470
471 std::streamoff m_posEntrycount; // Write: Pos of entrycount field.
472 std::streamoff m_posMapPosField; // Write: Pos of map position field.
473 std::string m_MapStreamString; // Write: Map stream string.
474
475 };
476
477
478 template <class T, class FuncObj>
479 void SsbWrite::WriteItem(const T& obj, const ID &id, FuncObj Func)
480 {
481 const std::streamoff pos = static_cast<std::streamoff>(oStrm.tellp());
482 Func(oStrm, obj);
483 OnWroteItem(id, pos);
484 }
485
486 template <class T, class FuncObj>
487 bool SsbRead::ReadItem(T& obj, const ID &id, FuncObj Func)
488 {
489 const ReadEntry* pE = Find(id);
490 const std::streamoff pos = static_cast<std::streamoff>(iStrm.tellg());
491 const bool entryFound = (pE || !m_Flags[RwfRMapHasId]);
492 if(entryFound)
493 {
494 Func(iStrm, obj, (pE) ? (pE->nSize) : invalidDatasize);
495 }
496 OnReadEntry(pE, id, pos);
497 return entryFound;
498 }
499
500
501 template <class T, class FuncObj>
502 bool SsbRead::ReadIterItem(const ReadIterator& iter, T& obj, FuncObj func)
503 {
504 iStrm.clear();
505 if (iter->rposStart != 0)
506 iStrm.seekg(m_posStart + iter->rposStart);
507 const std::streamoff pos = static_cast<std::streamoff>(iStrm.tellg());
508 func(iStrm, obj, iter->nSize);
509 OnReadEntry(&(*iter), ID(&m_Idarray[iter->nIdpos], iter->nIdLength), pos);
510 return true;
511 }
512
513
514 inline bool SsbRead::MatchesId(const ReadIterator& iter, const ID &id)
515 {
516 if(iter->nIdpos >= m_Idarray.size())
517 {
518 return false;
519 }
520 return (id == ID(&m_Idarray[iter->nIdpos], iter->nIdLength));
521 }
522
523
524 inline SsbRead::ReadIterator SsbRead::GetReadBegin()
525 {
526 MPT_ASSERT(m_Flags[RwfRMapHasId] && (m_Flags[RwfRMapHasStartpos] || m_Flags[RwfRMapHasSize] || m_nFixedEntrySize > 0));
527 if(!m_Flags[RwfRMapCached])
528 CacheMap();
529 return mapData.begin();
530 }
531
532
533 inline SsbRead::ReadIterator SsbRead::GetReadEnd()
534 {
535 if(!m_Flags[RwfRMapCached])
536 CacheMap();
537 return mapData.end();
538 }
539
540
541 template <class T>
542 struct VectorWriter
543 {
544 VectorWriter(size_t nCount) : m_nCount(nCount) {}
545 void operator()(std::ostream &oStrm, const std::vector<T> &vec)
546 {
547 for(size_t i = 0; i < m_nCount; i++)
548 {
549 Binarywrite(oStrm, vec[i]);
550 }
551 }
552 size_t m_nCount;
553 };
554
555 template <class T>
556 struct VectorReader
557 {
558 VectorReader(size_t nCount) : m_nCount(nCount) {}
559 void operator()(std::istream& iStrm, std::vector<T> &vec, const size_t)
560 {
561 vec.resize(m_nCount);
562 for(std::size_t i = 0; i < m_nCount; ++i)
563 {
564 Binaryread(iStrm, vec[i]);
565 }
566 }
567 size_t m_nCount;
568 };
569
570 template <class T>
571 struct ArrayReader
572 {
573 ArrayReader(size_t nCount) : m_nCount(nCount) {}
574 void operator()(std::istream& iStrm, T* pData, const size_t)
575 {
576 for(std::size_t i=0; i<m_nCount; ++i)
577 {
578 Binaryread(iStrm, pData[i]);
579 }
580 }
581 size_t m_nCount;
582 };
583
584
585
586 } //namespace srlztn.
587
588
589 OPENMPT_NAMESPACE_END