Tag Parser  8.0.1
C++ library for reading and writing MP4 (iTunes), ID3, Vorbis, Opus, FLAC and Matroska tags
mp4container.cpp
Go to the documentation of this file.
1 #include "./mp4container.h"
2 #include "./mp4ids.h"
3 
4 #include "../backuphelper.h"
5 #include "../exceptions.h"
6 #include "../mediafileinfo.h"
7 
8 #include <c++utilities/conversion/stringbuilder.h>
9 #include <c++utilities/io/binaryreader.h>
10 #include <c++utilities/io/binarywriter.h>
11 #include <c++utilities/io/catchiofailure.h>
12 #include <c++utilities/io/copy.h>
13 
14 #include <unistd.h>
15 
16 #include <memory>
17 #include <numeric>
18 #include <tuple>
19 
20 using namespace std;
21 using namespace IoUtilities;
22 using namespace ConversionUtilities;
23 using namespace ChronoUtilities;
24 
25 namespace TagParser {
26 
35 Mp4Container::Mp4Container(MediaFileInfo &fileInfo, uint64 startOffset)
36  : GenericContainer<MediaFileInfo, Mp4Tag, Mp4Track, Mp4Atom>(fileInfo, startOffset)
37  , m_fragmented(false)
38 {
39 }
40 
42 {
43 }
44 
46 {
48  m_fragmented = false;
49 }
50 
52 {
53  if (m_firstElement) {
54  const Mp4Atom *mediaDataAtom = m_firstElement->siblingById(Mp4AtomIds::MediaData, diag);
55  const Mp4Atom *userDataAtom = m_firstElement->subelementByPath(diag, Mp4AtomIds::Movie, Mp4AtomIds::UserData);
56  if (mediaDataAtom && userDataAtom) {
57  return userDataAtom->startOffset() < mediaDataAtom->startOffset() ? ElementPosition::BeforeData : ElementPosition::AfterData;
58  }
59  }
60  return ElementPosition::Keep;
61 }
62 
64 {
65  if (m_firstElement) {
66  const Mp4Atom *mediaDataAtom = m_firstElement->siblingById(Mp4AtomIds::MediaData, diag);
67  const Mp4Atom *movieAtom = m_firstElement->siblingById(Mp4AtomIds::Movie, diag);
68  if (mediaDataAtom && movieAtom) {
69  return movieAtom->startOffset() < mediaDataAtom->startOffset() ? ElementPosition::BeforeData : ElementPosition::AfterData;
70  }
71  }
72  return ElementPosition::Keep;
73 }
74 
76 {
77  //const string context("parsing header of MP4 container"); will be used when generating notifications
78  m_firstElement = make_unique<Mp4Atom>(*this, startOffset());
79  m_firstElement->parse(diag);
80  auto *const ftypAtom = m_firstElement->siblingByIdIncludingThis(Mp4AtomIds::FileType, diag);
81  if (!ftypAtom) {
82  m_doctype.clear();
83  m_version = 0;
84  return;
85  }
86  stream().seekg(static_cast<iostream::off_type>(ftypAtom->dataOffset()));
87  m_doctype = reader().readString(4);
88  m_version = reader().readUInt32BE();
89 }
90 
92 {
93  const string context("parsing tags of MP4 container");
94  auto *const udtaAtom = firstElement()->subelementByPath(diag, Mp4AtomIds::Movie, Mp4AtomIds::UserData);
95  if (!udtaAtom) {
96  return;
97  }
98  auto *metaAtom = udtaAtom->childById(Mp4AtomIds::Meta, diag);
99  bool surplusMetaAtoms = false;
100  while (metaAtom) {
101  metaAtom->parse(diag);
102  m_tags.emplace_back(make_unique<Mp4Tag>());
103  try {
104  m_tags.back()->parse(*metaAtom, diag);
105  } catch (const NoDataFoundException &) {
106  m_tags.pop_back();
107  }
108  if ((metaAtom = metaAtom->siblingById(Mp4AtomIds::Meta, diag))) {
109  surplusMetaAtoms = true;
110  }
111  if (!m_tags.empty()) {
112  break;
113  }
114  }
115  if (surplusMetaAtoms) {
116  diag.emplace_back(DiagLevel::Warning, "udta atom contains multiple meta atoms. Surplus meta atoms will be ignored.", context);
117  }
118 }
119 
121 {
122  static const string context("parsing tracks of MP4 container");
123  try {
124  // get moov atom which holds track information
125  if (Mp4Atom *moovAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::Movie, diag)) {
126  // get mvhd atom which holds overall track information
127  if (Mp4Atom *mvhdAtom = moovAtom->childById(Mp4AtomIds::MovieHeader, diag)) {
128  if (mvhdAtom->dataSize() > 0) {
129  stream().seekg(static_cast<iostream::off_type>(mvhdAtom->dataOffset()));
130  byte version = reader().readByte();
131  if ((version == 1 && mvhdAtom->dataSize() >= 32) || (mvhdAtom->dataSize() >= 20)) {
132  stream().seekg(3, ios_base::cur); // skip flags
133  switch (version) {
134  case 0:
135  m_creationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt32BE());
136  m_modificationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt32BE());
137  m_timeScale = reader().readUInt32BE();
138  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt32BE()) / static_cast<double>(m_timeScale));
139  break;
140  case 1:
141  m_creationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt64BE());
142  m_modificationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt64BE());
143  m_timeScale = reader().readUInt32BE();
144  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()) / static_cast<double>(m_timeScale));
145  break;
146  default:;
147  }
148  } else {
149  diag.emplace_back(DiagLevel::Critical, "mvhd atom is truncated.", context);
150  }
151  } else {
152  diag.emplace_back(DiagLevel::Critical, "mvhd atom is empty.", context);
153  }
154  } else {
155  diag.emplace_back(DiagLevel::Critical, "mvhd atom is does not exist.", context);
156  }
157  // get mvex atom which holds default values for fragmented files
159  m_fragmented = true;
160  if (mehdAtom->dataSize() > 0) {
161  stream().seekg(static_cast<iostream::off_type>(mehdAtom->dataOffset()));
162  unsigned int durationSize = reader().readByte() == 1u ? 8u : 4u; // duration size depends on atom version
163  if (mehdAtom->dataSize() >= 4 + durationSize) {
164  stream().seekg(3, ios_base::cur); // skip flags
165  switch (durationSize) {
166  case 4u:
167  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt32BE()) / static_cast<double>(m_timeScale));
168  break;
169  case 8u:
170  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()) / static_cast<double>(m_timeScale));
171  break;
172  default:;
173  }
174  } else {
175  diag.emplace_back(DiagLevel::Warning, "mehd atom is truncated.", context);
176  }
177  }
178  }
179  // get first trak atoms which hold information for each track
180  Mp4Atom *trakAtom = moovAtom->childById(Mp4AtomIds::Track, diag);
181  int trackNum = 1;
182  while (trakAtom) {
183  try {
184  trakAtom->parse(diag);
185  } catch (const Failure &) {
186  diag.emplace_back(DiagLevel::Warning, "Unable to parse child atom of moov.", context);
187  }
188  // parse the trak atom using the Mp4Track class
189  m_tracks.emplace_back(make_unique<Mp4Track>(*trakAtom));
190  try { // try to parse header
191  m_tracks.back()->parseHeader(diag);
192  } catch (const Failure &) {
193  diag.emplace_back(DiagLevel::Critical, argsToString("Unable to parse track ", trackNum, '.'), context);
194  }
195  trakAtom = trakAtom->siblingById(Mp4AtomIds::Track, diag); // get next trak atom
196  ++trackNum;
197  }
198  // get overall duration, creation time and modification time if not determined yet
199  if (m_duration.isNull() || m_modificationTime.isNull() || m_creationTime.isNull()) {
200  for (const auto &track : tracks()) {
201  if (track->duration() > m_duration) {
203  }
206  }
207  if (track->creationTime() < m_creationTime) {
209  }
210  }
211  }
212  }
213  } catch (const Failure &) {
214  diag.emplace_back(DiagLevel::Warning, "Unable to parse moov atom.", context);
215  }
216 }
217 
219 {
220  static const string context("making MP4 container");
221  progress.updateStep("Calculating atom sizes and padding ...");
222 
223  // basic validation of original file
224  if (!isHeaderParsed()) {
225  diag.emplace_back(DiagLevel::Critical, "The header has not been parsed yet.", context);
226  throw InvalidDataException();
227  }
228 
229  // define variables needed to parse atoms of original file
230  if (!firstElement()) {
231  diag.emplace_back(DiagLevel::Critical, "No MP4 atoms could be found.", context);
232  throw InvalidDataException();
233  }
234 
235  // define variables needed to manage file layout
236  // -> whether media data is written chunk by chunk (need to write chunk by chunk if tracks have been altered)
237  const bool writeChunkByChunk = m_tracksAltered;
238  // -> whether rewrite is required (always required when forced to rewrite or when tracks have been altered)
239  bool rewriteRequired = fileInfo().isForcingRewrite() || writeChunkByChunk;
240  // -> use the preferred tag position/index position (force one wins, if both are force tag pos wins; might be changed later if none is forced)
241  ElementPosition initialNewTagPos
243  ElementPosition newTagPos = initialNewTagPos;
244  // -> current tag position (determined later)
245  ElementPosition currentTagPos;
246  // -> holds new padding (before actual data)
247  uint64 newPadding;
248  // -> holds new padding (after actual data)
249  uint64 newPaddingEnd;
250  // -> holds current offset
251  uint64 currentOffset;
252  // -> holds track information, used when writing chunk-by-chunk
253  vector<tuple<istream *, vector<uint64>, vector<uint64>>> trackInfos;
254  // -> holds offsets of media data atoms in original file, used when simply copying mdat
255  vector<int64> origMediaDataOffsets;
256  // -> holds offsets of media data atoms in new file, used when simply copying mdat
257  vector<int64> newMediaDataOffsets;
258  // -> new size of movie atom and user data atom
259  uint64 movieAtomSize, userDataAtomSize;
260  // -> track count of original file
261  const auto trackCount = this->trackCount();
262 
263  // find relevant atoms in original file
264  Mp4Atom *fileTypeAtom, *progressiveDownloadInfoAtom, *movieAtom, *firstMediaDataAtom, *firstMovieFragmentAtom /*, *userDataAtom*/;
265  Mp4Atom *level0Atom, *level1Atom, *level2Atom, *lastAtomToBeWritten;
266  try {
267  // file type atom (mandatory)
268  if ((fileTypeAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::FileType, diag))) {
269  // buffer atom
270  fileTypeAtom->makeBuffer();
271  } else {
272  // throw error if missing
273  diag.emplace_back(DiagLevel::Critical, "Mandatory \"ftyp\"-atom not found.", context);
274  throw InvalidDataException();
275  }
276 
277  // progressive download information atom (not mandatory)
278  if ((progressiveDownloadInfoAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::ProgressiveDownloadInformation, diag))) {
279  // buffer atom
280  progressiveDownloadInfoAtom->makeBuffer();
281  }
282 
283  // movie atom (mandatory)
284  if (!(movieAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::Movie, diag))) {
285  // throw error if missing
286  diag.emplace_back(DiagLevel::Critical, "Mandatory \"moov\"-atom not in the source file found.", context);
287  throw InvalidDataException();
288  }
289 
290  // movie fragment atom (indicates dash file)
291  if ((firstMovieFragmentAtom = firstElement()->siblingById(Mp4AtomIds::MovieFragment, diag))) {
292  // there is at least one movie fragment atom -> consider file being dash
293  // -> can not write chunk-by-chunk (currently)
294  if (writeChunkByChunk) {
295  diag.emplace_back(DiagLevel::Critical, "Writing chunk-by-chunk is not implemented for DASH files.", context);
296  throw NotImplementedException();
297  }
298  // -> tags must be placed at the beginning
299  newTagPos = ElementPosition::BeforeData;
300  }
301 
302  // media data atom (mandatory?)
303  // -> consider not only mdat as media data atom; consider everything not handled otherwise as media data
304  for (firstMediaDataAtom = nullptr, level0Atom = firstElement(); level0Atom; level0Atom = level0Atom->nextSibling()) {
305  level0Atom->parse(diag);
306  switch (level0Atom->id()) {
309  case Mp4AtomIds::Movie:
310  case Mp4AtomIds::Free:
311  case Mp4AtomIds::Skip:
312  continue;
313  default:
314  firstMediaDataAtom = level0Atom;
315  }
316  break;
317  }
318 
319  // determine current tag position
320  // -> since tags are nested in the movie atom its position is relevant here
321  if (firstMediaDataAtom) {
322  currentTagPos = firstMediaDataAtom->startOffset() < movieAtom->startOffset() ? ElementPosition::AfterData : ElementPosition::BeforeData;
323  if (newTagPos == ElementPosition::Keep) {
324  newTagPos = currentTagPos;
325  }
326  } else {
327  currentTagPos = ElementPosition::Keep;
328  }
329 
330  // ensure index and tags are always placed at the beginning when dealing with DASH files
331  if (firstMovieFragmentAtom) {
332  if (initialNewTagPos == ElementPosition::AfterData) {
333  diag.emplace_back(
334  DiagLevel::Warning, "Sorry, but putting index/tags at the end is not possible when dealing with DASH files.", context);
335  }
336  initialNewTagPos = newTagPos = ElementPosition::BeforeData;
337  }
338 
339  // user data atom (currently not used)
340  //userDataAtom = movieAtom->childById(Mp4AtomIds::UserData);
341 
342  } catch (const NotImplementedException &) {
343  throw;
344 
345  } catch (const Failure &) {
346  // can't ignore parsing errors here
347  diag.emplace_back(DiagLevel::Critical, "Unable to parse the overall atom structure of the source file.", context);
348  throw InvalidDataException();
349  }
350 
351  progress.stopIfAborted();
352 
353  // calculate sizes
354  // -> size of tags
355  vector<Mp4TagMaker> tagMaker;
356  uint64 tagsSize = 0;
357  tagMaker.reserve(m_tags.size());
358  for (auto &tag : m_tags) {
359  try {
360  tagMaker.emplace_back(tag->prepareMaking(diag));
361  tagsSize += tagMaker.back().requiredSize();
362  } catch (const Failure &) {
363  }
364  }
365 
366  // -> size of movie atom (contains track and tag information)
367  movieAtomSize = userDataAtomSize = 0;
368  try {
369  // add size of children
370  for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
371  for (level1Atom = level0Atom->firstChild(); level1Atom; level1Atom = level1Atom->nextSibling()) {
372  level1Atom->parse(diag);
373  switch (level1Atom->id()) {
375  try {
376  for (level2Atom = level1Atom->firstChild(); level2Atom; level2Atom = level2Atom->nextSibling()) {
377  level2Atom->parse(diag);
378  switch (level2Atom->id()) {
379  case Mp4AtomIds::Meta:
380  // ignore meta data here; it is added separately
381  break;
382  default:
383  // add size of unknown childs of the user data atom
384  userDataAtomSize += level2Atom->totalSize();
385  level2Atom->makeBuffer();
386  }
387  }
388  } catch (const Failure &) {
389  // invalid children might be ignored as not mandatory
390  diag.emplace_back(
391  DiagLevel::Critical, "Unable to parse the children of \"udta\"-atom of the source file; ignoring them.", context);
392  }
393  break;
394  case Mp4AtomIds::Track:
395  // ignore track atoms here; they are added separately
396  break;
397  default:
398  // add size of unknown childs of the movie atom
399  movieAtomSize += level1Atom->totalSize();
400  level1Atom->makeBuffer();
401  }
402  }
403  }
404 
405  // add size of meta data
406  if (userDataAtomSize += tagsSize) {
407  Mp4Atom::addHeaderSize(userDataAtomSize);
408  movieAtomSize += userDataAtomSize;
409  }
410 
411  // add size of track atoms
412  for (const auto &track : tracks()) {
413  movieAtomSize += track->requiredSize(diag);
414  }
415 
416  // add header size
417  Mp4Atom::addHeaderSize(movieAtomSize);
418  } catch (const Failure &) {
419  // can't ignore parsing errors here
420  diag.emplace_back(DiagLevel::Critical, "Unable to parse the children of \"moov\"-atom of the source file.", context);
421  throw InvalidDataException();
422  }
423 
424  progress.stopIfAborted();
425 
426  // check whether there are atoms to be voided after movie next sibling (only relevant when not rewriting)
427  if (!rewriteRequired) {
428  newPaddingEnd = 0;
429  uint64 currentSum = 0;
430  for (Mp4Atom *level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
431  level0Atom->parse(diag);
432  switch (level0Atom->id()) {
435  case Mp4AtomIds::Movie:
436  case Mp4AtomIds::Free:
437  case Mp4AtomIds::Skip:
438  // must void these if they occur "between" the media data
439  currentSum += level0Atom->totalSize();
440  break;
441  default:
442  newPaddingEnd += currentSum;
443  currentSum = 0;
444  lastAtomToBeWritten = level0Atom;
445  }
446  }
447  }
448 
449  // calculate padding if no rewrite is required; otherwise use the preferred padding
450 calculatePadding:
451  if (rewriteRequired) {
452  newPadding = (fileInfo().preferredPadding() && fileInfo().preferredPadding() < 8 ? 8 : fileInfo().preferredPadding());
453  } else {
454  // file type atom
455  currentOffset = fileTypeAtom->totalSize();
456 
457  // progressive download information atom
458  if (progressiveDownloadInfoAtom) {
459  currentOffset += progressiveDownloadInfoAtom->totalSize();
460  }
461 
462  // if writing tags before data: movie atom (contains tag)
463  switch (newTagPos) {
466  currentOffset += movieAtomSize;
467  break;
468  default:;
469  }
470 
471  // check whether there is sufficiant space before the next atom
472  if (!(rewriteRequired = firstMediaDataAtom && currentOffset > firstMediaDataAtom->startOffset())) {
473  // there is sufficiant space
474  // -> check whether the padding matches specifications
475  // min padding: says "at least ... byte should be reserved to prepend further tag info", so the padding at the end
476  // shouldn't be tanken into account (it can't be used to prepend further tag info)
477  // max padding: says "do not waste more than ... byte", so here all padding should be taken into account
478  newPadding = firstMediaDataAtom->startOffset() - currentOffset;
479  rewriteRequired = (newPadding > 0 && newPadding < 8) || newPadding < fileInfo().minPadding()
480  || (newPadding + newPaddingEnd) > fileInfo().maxPadding();
481  }
482  if (rewriteRequired) {
483  // can't put the tags before media data
484  if (!firstMovieFragmentAtom && !fileInfo().forceTagPosition() && !fileInfo().forceIndexPosition()
485  && newTagPos != ElementPosition::AfterData) {
486  // writing tag before media data is not forced, its not a DASH file and tags aren't already at the end
487  // -> try to put the tags at the end
488  newTagPos = ElementPosition::AfterData;
489  rewriteRequired = false;
490  } else {
491  // writing tag before media data is forced -> rewrite the file
492  // when rewriting anyways, ensure the preferred tag position is used
493  newTagPos = initialNewTagPos == ElementPosition::Keep ? currentTagPos : initialNewTagPos;
494  }
495  // in any case: recalculate padding
496  goto calculatePadding;
497  } else {
498  // tags can be put before the media data
499  // -> ensure newTagPos is not ElementPosition::Keep
500  if (newTagPos == ElementPosition::Keep) {
501  newTagPos = ElementPosition::BeforeData;
502  }
503  }
504  }
505 
506  // setup stream(s) for writing
507  // -> update status
508  progress.nextStepOrStop("Preparing streams ...");
509 
510  // -> define variables needed to handle output stream and backup stream (required when rewriting the file)
511  string backupPath;
512  NativeFileStream &outputStream = fileInfo().stream();
513  NativeFileStream backupStream; // create a stream to open the backup/original file for the case rewriting the file is required
514  BinaryWriter outputWriter(&outputStream);
515 
516  if (rewriteRequired) {
517  if (fileInfo().saveFilePath().empty()) {
518  // move current file to temp dir and reopen it as backupStream, recreate original file
519  try {
520  BackupHelper::createBackupFile(fileInfo().backupDirectory(), fileInfo().path(), backupPath, outputStream, backupStream);
521  // recreate original file, define buffer variables
522  outputStream.open(fileInfo().path(), ios_base::out | ios_base::binary | ios_base::trunc);
523  } catch (...) {
524  const char *what = catchIoFailure();
525  diag.emplace_back(DiagLevel::Critical, "Creation of temporary file (to rewrite the original file) failed.", context);
526  throwIoFailure(what);
527  }
528  } else {
529  // open the current file as backupStream and create a new outputStream at the specified "save file path"
530  try {
531  backupStream.exceptions(ios_base::badbit | ios_base::failbit);
532  backupStream.open(fileInfo().path(), ios_base::in | ios_base::binary);
533  fileInfo().close();
534  outputStream.open(fileInfo().saveFilePath(), ios_base::out | ios_base::binary | ios_base::trunc);
535  } catch (...) {
536  const char *what = catchIoFailure();
537  diag.emplace_back(DiagLevel::Critical, "Opening streams to write output file failed.", context);
538  throwIoFailure(what);
539  }
540  }
541 
542  // set backup stream as associated input stream since we need the original elements to write the new file
543  setStream(backupStream);
544 
545  // TODO: reduce code duplication
546 
547  } else { // !rewriteRequired
548  // ensure everything to make track atoms is buffered before altering the source file
549  for (const auto &track : tracks()) {
550  track->bufferTrackAtoms(diag);
551  }
552 
553  // reopen original file to ensure it is opened for writing
554  try {
555  fileInfo().close();
556  outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
557  } catch (...) {
558  const char *what = catchIoFailure();
559  diag.emplace_back(DiagLevel::Critical, "Opening the file with write permissions failed.", context);
560  throwIoFailure(what);
561  }
562  }
563 
564  // start actual writing
565  try {
566  // write header
567  progress.nextStepOrStop("Writing header and tags ...");
568  // -> make file type atom
569  fileTypeAtom->copyBuffer(outputStream);
570  fileTypeAtom->discardBuffer();
571  // -> make progressive download info atom
572  if (progressiveDownloadInfoAtom) {
573  progressiveDownloadInfoAtom->copyBuffer(outputStream);
574  progressiveDownloadInfoAtom->discardBuffer();
575  }
576 
577  // set input/output streams of each track
578  for (auto &track : tracks()) {
579  // ensure the track reads from the original file
580  if (&track->inputStream() == &outputStream) {
581  track->setInputStream(backupStream);
582  }
583  // ensure the track writes to the output file
584  track->setOutputStream(outputStream);
585  }
586 
587  // write movie atom / padding and media data
588  for (byte pass = 0; pass != 2; ++pass) {
589  if (newTagPos == (pass ? ElementPosition::AfterData : ElementPosition::BeforeData)) {
590  // write movie atom
591  // -> write movie atom header
592  Mp4Atom::makeHeader(movieAtomSize, Mp4AtomIds::Movie, outputWriter);
593 
594  // -> write track atoms
595  for (auto &track : tracks()) {
596  track->makeTrack(diag);
597  }
598 
599  // -> write other movie atom children
600  for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
601  for (level1Atom = level0Atom->firstChild(); level1Atom; level1Atom = level1Atom->nextSibling()) {
602  switch (level1Atom->id()) {
604  case Mp4AtomIds::Track:
605  // track and user data atoms are written separately
606  break;
607  default:
608  // write buffered data
609  level1Atom->copyBuffer(outputStream);
610  level1Atom->discardBuffer();
611  }
612  }
613  }
614 
615  // -> write user data atom
616  if (userDataAtomSize) {
617  // writer user data atom header
618  Mp4Atom::makeHeader(userDataAtomSize, Mp4AtomIds::UserData, outputWriter);
619 
620  // write other children of user data atom
621  for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
622  for (level1Atom = level0Atom->childById(Mp4AtomIds::UserData, diag); level1Atom;
623  level1Atom = level1Atom->siblingById(Mp4AtomIds::UserData, diag)) {
624  for (level2Atom = level1Atom->firstChild(); level2Atom; level2Atom = level2Atom->nextSibling()) {
625  switch (level2Atom->id()) {
626  case Mp4AtomIds::Meta:
627  break;
628  default:
629  // write buffered data
630  level2Atom->copyBuffer(outputStream);
631  level2Atom->discardBuffer();
632  }
633  }
634  }
635  }
636 
637  // write meta atom
638  for (auto &maker : tagMaker) {
639  maker.make(outputStream, diag);
640  }
641  }
642 
643  } else {
644  // write padding
645  if (newPadding) {
646  // write free atom header
647  if (newPadding < numeric_limits<uint32>::max()) {
648  outputWriter.writeUInt32BE(static_cast<uint32>(newPadding));
649  outputWriter.writeUInt32BE(Mp4AtomIds::Free);
650  newPadding -= 8;
651  } else {
652  outputWriter.writeUInt32BE(1);
653  outputWriter.writeUInt32BE(Mp4AtomIds::Free);
654  outputWriter.writeUInt64BE(newPadding);
655  newPadding -= 16;
656  }
657 
658  // write zeroes
659  for (; newPadding; --newPadding) {
660  outputStream.put(0);
661  }
662  }
663 
664  // write media data
665  if (rewriteRequired) {
666  for (level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
667  level0Atom->parse(diag);
668  switch (level0Atom->id()) {
671  case Mp4AtomIds::Movie:
672  case Mp4AtomIds::Free:
673  case Mp4AtomIds::Skip:
674  break;
676  if (writeChunkByChunk) {
677  // write actual data separately when writing chunk-by-chunk
678  break;
679  } else {
680  // store media data offsets when not writing chunk-by-chunk to be able to update chunk offset table
681  origMediaDataOffsets.push_back(static_cast<int64>(level0Atom->startOffset()));
682  newMediaDataOffsets.push_back(outputStream.tellp());
683  }
684  FALLTHROUGH;
685  default:
686  // update status
687  progress.updateStep("Writing atom: " + level0Atom->idToString());
688  // copy atom entirely and forward status update calls
689  level0Atom->copyEntirely(outputStream, diag, &progress);
690  }
691  }
692 
693  // when writing chunk-by-chunk write media data now
694  if (writeChunkByChunk) {
695  // read chunk offset and chunk size table from the old file which are required to get chunks
696  progress.updateStep("Reading chunk offsets and sizes from the original file ...");
697  trackInfos.reserve(trackCount);
698  uint64 totalChunkCount = 0;
699  uint64 totalMediaDataSize = 0;
700  for (auto &track : tracks()) {
701  progress.stopIfAborted();
702 
703  // emplace information
704  trackInfos.emplace_back(
705  &track->inputStream(), track->readChunkOffsets(fileInfo().isForcingFullParse(), diag), track->readChunkSizes(diag));
706 
707  // check whether the chunks could be parsed correctly
708  const vector<uint64> &chunkOffsetTable = get<1>(trackInfos.back());
709  const vector<uint64> &chunkSizesTable = get<2>(trackInfos.back());
710  if (track->chunkCount() != chunkOffsetTable.size() || track->chunkCount() != chunkSizesTable.size()) {
711  diag.emplace_back(DiagLevel::Critical,
712  "Chunks of track " % numberToString<uint64, string>(track->id()) + " could not be parsed correctly.", context);
713  }
714 
715  // increase total chunk count and size
716  totalChunkCount += track->chunkCount();
717  totalMediaDataSize += accumulate(chunkSizesTable.cbegin(), chunkSizesTable.cend(), 0ul);
718  }
719 
720  // write media data chunk-by-chunk
721  // -> write header of media data atom
722  Mp4Atom::addHeaderSize(totalMediaDataSize);
723  Mp4Atom::makeHeader(totalMediaDataSize, Mp4AtomIds::MediaData, outputWriter);
724 
725  // -> copy chunks
726  CopyHelper<0x2000> copyHelper;
727  uint64 chunkIndexWithinTrack = 0, totalChunksCopied = 0;
728  bool anyChunksCopied;
729  do {
730  progress.stopIfAborted();
731 
732  // copy a chunk from each track
733  anyChunksCopied = false;
734  for (size_t trackIndex = 0; trackIndex < trackCount; ++trackIndex) {
735  // get source stream and tables for current track
736  auto &trackInfo = trackInfos[trackIndex];
737  istream &sourceStream = *get<0>(trackInfo);
738  vector<uint64> &chunkOffsetTable = get<1>(trackInfo);
739  const vector<uint64> &chunkSizesTable = get<2>(trackInfo);
740 
741  // still chunks to be copied (of this track)?
742  if (chunkIndexWithinTrack < chunkOffsetTable.size() && chunkIndexWithinTrack < chunkSizesTable.size()) {
743  // copy chunk, update entry in chunk offset table
744  sourceStream.seekg(static_cast<streamoff>(chunkOffsetTable[chunkIndexWithinTrack]));
745  chunkOffsetTable[chunkIndexWithinTrack] = static_cast<uint64>(outputStream.tellp());
746  copyHelper.copy(sourceStream, outputStream, chunkSizesTable[chunkIndexWithinTrack]);
747 
748  // update counter / status
749  anyChunksCopied = true;
750  ++totalChunksCopied;
751  }
752  }
753 
754  // incrase chunk index within track, update progress percentage
755  if (!(++chunkIndexWithinTrack % 10)) {
756  progress.updateStepPercentage(static_cast<byte>(totalChunksCopied * 100 / totalChunkCount));
757  }
758 
759  } while (anyChunksCopied);
760  }
761 
762  } else {
763  // can't just skip next movie sibling
764  for (Mp4Atom *level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
765  level0Atom->parse(diag);
766  switch (level0Atom->id()) {
769  case Mp4AtomIds::Movie:
770  // must void these if they occur "between" the media data
771  outputStream.seekp(4, ios_base::cur);
772  outputWriter.writeUInt32BE(Mp4AtomIds::Free);
773  break;
774  default:
775  outputStream.seekp(static_cast<iostream::off_type>(level0Atom->totalSize()), ios_base::cur);
776  }
777  if (level0Atom == lastAtomToBeWritten) {
778  break;
779  }
780  }
781  }
782  }
783  }
784 
785  // reparse what is written so far
786  progress.updateStep("Reparsing output file ...");
787  if (rewriteRequired) {
788  // report new size
789  fileInfo().reportSizeChanged(static_cast<uint64>(outputStream.tellp()));
790  // "save as path" is now the regular path
791  if (!fileInfo().saveFilePath().empty()) {
792  fileInfo().reportPathChanged(fileInfo().saveFilePath());
793  fileInfo().setSaveFilePath(string());
794  }
795  // the outputStream needs to be reopened to be able to read again
796  outputStream.close();
797  outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
798  setStream(outputStream);
799  } else {
800  const auto newSize = static_cast<uint64>(outputStream.tellp());
801  if (newSize < fileInfo().size()) {
802  // file is smaller after the modification -> truncate
803  // -> close stream before truncating
804  outputStream.close();
805  // -> truncate file
806  if (truncate(fileInfo().path().c_str(), static_cast<iostream::off_type>(newSize)) == 0) {
807  fileInfo().reportSizeChanged(newSize);
808  } else {
809  diag.emplace_back(DiagLevel::Critical, "Unable to truncate the file.", context);
810  }
811  // -> reopen the stream again
812  outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
813  } else {
814  // file is longer after the modification -> just report new size
815  fileInfo().reportSizeChanged(newSize);
816  }
817  }
818 
819  reset();
820  try {
821  parseTracks(diag);
822  } catch (const Failure &) {
823  diag.emplace_back(DiagLevel::Critical, "Unable to reparse the new file.", context);
824  throw;
825  }
826 
827  if (rewriteRequired) {
828  // check whether track count of new file equals track count of old file
829  if (trackCount != tracks().size()) {
830  diag.emplace_back(DiagLevel::Critical,
831  argsToString("Unable to update chunk offsets (\"stco\"-atom): Number of tracks in the output file (", tracks().size(),
832  ") differs from the number of tracks in the original file (", trackCount, ")."),
833  context);
834  throw Failure();
835  }
836 
837  // update chunk offset table
838  if (writeChunkByChunk) {
839  progress.updateStep("Updating chunk offset table for each track ...");
840  for (size_t trackIndex = 0; trackIndex != trackCount; ++trackIndex) {
841  const auto &track = tracks()[trackIndex];
842  const auto &chunkOffsetTable = get<1>(trackInfos[trackIndex]);
843  if (track->chunkCount() == chunkOffsetTable.size()) {
844  track->updateChunkOffsets(chunkOffsetTable);
845  } else {
846  diag.emplace_back(DiagLevel::Critical,
847  argsToString("Unable to update chunk offsets of track ", (trackIndex + 1),
848  ": Number of chunks in the output file differs from the number of chunks in the orignal file."),
849  context);
850  throw Failure();
851  }
852  }
853  } else {
854  progress.updateStep("Updating chunk offset table for each track ...");
855  updateOffsets(origMediaDataOffsets, newMediaDataOffsets, diag);
856  }
857  }
858 
859  // flush output stream
860  outputStream.flush();
861 
862  // handle errors (which might have been occurred after renaming/creating backup file)
863  } catch (...) {
864  BackupHelper::handleFailureAfterFileModified(fileInfo(), backupPath, outputStream, backupStream, diag, context);
865  }
866 }
867 
880 void Mp4Container::updateOffsets(const std::vector<int64> &oldMdatOffsets, const std::vector<int64> &newMdatOffsets, Diagnostics &diag)
881 {
882  // do NOT invalidate the status here since this method is internally called by internalMakeFile(), just update the status
883  const string context("updating MP4 container chunk offset table");
884  if (!firstElement()) {
885  diag.emplace_back(DiagLevel::Critical, "No MP4 atoms could be found.", context);
886  throw InvalidDataException();
887  }
888  // update "base-data-offset-present" of "tfhd"-atom (NOT tested properly)
889  try {
890  for (Mp4Atom *moofAtom = firstElement()->siblingById(Mp4AtomIds::MovieFragment, diag); moofAtom;
891  moofAtom = moofAtom->siblingById(Mp4AtomIds::MovieFragment, diag)) {
892  moofAtom->parse(diag);
893  try {
894  for (Mp4Atom *trafAtom = moofAtom->childById(Mp4AtomIds::TrackFragment, diag); trafAtom;
895  trafAtom = trafAtom->siblingById(Mp4AtomIds::TrackFragment, diag)) {
896  trafAtom->parse(diag);
897  int tfhdAtomCount = 0;
898  for (Mp4Atom *tfhdAtom = trafAtom->childById(Mp4AtomIds::TrackFragmentHeader, diag); tfhdAtom;
899  tfhdAtom = tfhdAtom->siblingById(Mp4AtomIds::TrackFragmentHeader, diag)) {
900  tfhdAtom->parse(diag);
901  ++tfhdAtomCount;
902  if (tfhdAtom->dataSize() < 8) {
903  diag.emplace_back(DiagLevel::Warning, "tfhd atom is truncated.", context);
904  continue;
905  }
906  stream().seekg(static_cast<iostream::off_type>(tfhdAtom->dataOffset()) + 1);
907  uint32 flags = reader().readUInt24BE();
908  if (!(flags & 1)) {
909  continue;
910  }
911  if (tfhdAtom->dataSize() < 16) {
912  diag.emplace_back(DiagLevel::Warning, "tfhd atom (denoting base-data-offset-present) is truncated.", context);
913  continue;
914  }
915  stream().seekg(4, ios_base::cur); // skip track ID
916  uint64 off = reader().readUInt64BE();
917  for (auto iOld = oldMdatOffsets.cbegin(), iNew = newMdatOffsets.cbegin(), end = oldMdatOffsets.cend(); iOld != end;
918  ++iOld, ++iNew) {
919  if (off < static_cast<uint64>(*iOld)) {
920  continue;
921  }
922  off += static_cast<uint64>(*iNew - *iOld);
923  stream().seekp(static_cast<iostream::off_type>(tfhdAtom->dataOffset()) + 8);
924  writer().writeUInt64BE(off);
925  break;
926  }
927  }
928  switch (tfhdAtomCount) {
929  case 0:
930  diag.emplace_back(DiagLevel::Warning, "traf atom doesn't contain mandatory tfhd atom.", context);
931  break;
932  case 1:
933  break;
934  default:
935  diag.emplace_back(
936  DiagLevel::Warning, "traf atom stores multiple tfhd atoms but it should only contain exactly one tfhd atom.", context);
937  }
938  }
939  } catch (const Failure &) {
940  diag.emplace_back(DiagLevel::Critical, "Unable to parse childs of top-level atom moof.", context);
941  }
942  }
943  } catch (const Failure &) {
944  diag.emplace_back(DiagLevel::Critical, "Unable to parse top-level atom moof.", context);
945  }
946  // update each track
947  for (auto &track : tracks()) {
948  if (!track->isHeaderValid()) {
949  try {
950  track->parseHeader(diag);
951  } catch (const Failure &) {
952  diag.emplace_back(DiagLevel::Warning,
953  "The chunk offsets of track " % track->name() + " couldn't be updated because the track seems to be invalid..", context);
954  throw;
955  }
956  }
957  if (track->isHeaderValid()) {
958  try {
959  track->updateChunkOffsets(oldMdatOffsets, newMdatOffsets);
960  } catch (const Failure &) {
961  diag.emplace_back(DiagLevel::Warning, "The chunk offsets of track " % track->name() + " couldn't be updated.", context);
962  throw;
963  }
964  }
965  }
966 }
967 
968 } // namespace TagParser
std::vector< uint64 > readChunkOffsets(bool parseFragments, Diagnostics &diag)
Reads the chunk offsets from the stco atom and fragments if parseFragments is true.
Definition: mp4track.cpp:178
This exception is thrown when the an operation is invoked that has not been implemented yet...
Definition: exceptions.h:53
Implementation of TagParser::Tag for the MP4 container.
Definition: mp4tag.h:97
const std::string name() const
Returns the track name if known; otherwise returns an empty string.
size_t preferredPadding() const
Returns the padding to be written before the data block when applying changes and the file needs to b...
void updateChunkOffsets(const std::vector< int64 > &oldMdatOffsets, const std::vector< int64 > &newMdatOffsets)
Updates the chunk offsets of the track.
Definition: mp4track.cpp:896
void internalMakeFile(Diagnostics &diag, AbortableProgressFeedback &progress) override
Internally called to make the file.
void updateStepPercentage(byte stepPercentage)
Updates the current step percentage and invokes the second callback specified on construction (or the...
std::vector< uint64 > readChunkSizes(TagParser::Diagnostics &diag)
Reads the chunk sizes from the stsz (sample sizes) and stsc (samples per chunk) atom.
Definition: mp4track.cpp:509
static constexpr void addHeaderSize(uint64 &dataSize)
Adds the header size to the specified data size.
Definition: mp4atom.h:81
void copyEntirely(std::ostream &targetStream, Diagnostics &diag, AbortableProgressFeedback *progress)
Writes the entire element including all childs to the specified targetStream.
Mp4Atom * firstElement() const
Returns the first element of the file if available; otherwiese returns nullptr.
bool forceTagPosition() const
Returns whether tagPosition() is forced.
void stopIfAborted() const
Throws an OperationAbortedException if aborted.
void makeBuffer()
Buffers the element (header and data).
ImplementationType * nextSibling()
Returns the next sibling of the element.
void internalParseTracks(Diagnostics &diag) override
Internally called to parse the tracks.
ImplementationType * firstChild()
Returns the first child of the element.
void setInputStream(std::istream &stream)
Assigns another input stream.
const std::vector< std::unique_ptr< Mp4Track > > & tracks() const
Returns the tracks of the file.
STL namespace.
bool isForcingRewrite() const
Returns whether forcing rewriting (when applying changes) is enabled.
uint32 chunkCount() const
Returns the number of chunks denoted by the stco atom.
Definition: mp4track.h:227
std::iostream & stream()
Returns the related stream.
uint64 startOffset() const
Returns the start offset in the related stream.
void reset() override
Discards all parsing results.
void bufferTrackAtoms(Diagnostics &diag)
Buffers all atoms required by the makeTrack() method.
Definition: mp4track.cpp:1067
ChronoUtilities::DateTime m_creationTime
size_t maxPadding() const
Returns the maximum padding to be written before the data blocks when applying changes.
void parseTracks(Diagnostics &diag)
Parses the tracks of the file if not parsed yet.
The Mp4Atom class helps to parse MP4 files.
Definition: mp4atom.h:38
uint64 totalSize() const
Returns the total size of the element.
const ChronoUtilities::DateTime & creationTime() const
Returns the creation time if known; otherwise returns a DateTime of zero ticks.
The MediaFileInfo class allows to read and write tag information providing a container/tag format ind...
Definition: mediafileinfo.h:44
TAG_PARSER_EXPORT void createBackupFile(const std::string &backupDir, const std::string &originalPath, std::string &backupPath, IoUtilities::NativeFileStream &originalStream, IoUtilities::NativeFileStream &backupStream)
bool isHeaderParsed() const
Returns an indication whether the header has been parsed yet.
Implementation of TagParser::AbstractTrack for the MP4 container.
Definition: mp4track.h:118
void discardBuffer()
Discards buffered data.
ChronoUtilities::TimeSpan m_duration
uint64 version() const
Returns the version if known; otherwise returns 0.
Contains utility classes helping to read and write streams.
void parse(Diagnostics &diag)
Parses the header information of the element which is read from the related stream at the start offse...
void setStream(std::iostream &stream)
Sets the related stream.
void setOutputStream(std::ostream &stream)
Assigns another output stream.
IoUtilities::BinaryWriter & writer()
Returns the related BinaryWriter.
Mp4TagMaker prepareMaking(Diagnostics &diag)
Prepares making.
Definition: mp4tag.cpp:372
void internalParseHeader(Diagnostics &diag) override
Internally called to parse the header.
The exception that is thrown when the data to be parsed holds no parsable information.
Definition: exceptions.h:18
size_t minPadding() const
Returns the minimum padding to be written before the data blocks when applying changes.
std::istream & inputStream()
Returns the associated input stream.
ImplementationType * siblingById(const IdentifierType &id, Diagnostics &diag)
Returns the first sibling with the specified id.
void reportPathChanged(const std::string &newPath)
Call this function to report that the path changed.
bool forceIndexPosition() const
Returns whether indexPosition() is forced.
The AbortableProgressFeedback class provides feedback about an ongoing operation via callbacks...
void close()
A possibly opened std::fstream will be closed.
uint64 id() const
Returns the track ID if known; otherwise returns 0.
void updateStep(const std::string &step, byte stepPercentage=0)
Updates the current step and invokes the first callback specified on construction.
TAG_PARSER_EXPORT void handleFailureAfterFileModified(MediaFileInfo &mediaFileInfo, const std::string &backupPath, IoUtilities::NativeFileStream &outputStream, IoUtilities::NativeFileStream &backupStream, Diagnostics &diag, const std::string &context="making file")
ElementPosition determineIndexPosition(Diagnostics &diag) const override
Determines the position of the index.
const std::string & saveFilePath() const
Returns the "save file path" which has been set using setSaveFilePath().
ElementPosition tagPosition() const
Returns the position (in the output file) where the tag information is written when applying changes...
ElementPosition indexPosition() const
Returns the position (in the output file) where the index is written when applying changes...
void reportSizeChanged(uint64 newSize)
Call this function to report that the size changed.
ChronoUtilities::DateTime m_modificationTime
void copyBuffer(std::ostream &targetStream)
Copies buffered data to targetStream.
void makeTrack(Diagnostics &diag)
Makes the track entry ("trak"-atom) for the track.
Definition: mp4track.cpp:1157
IoUtilities::BinaryReader & reader()
Returns the related BinaryReader.
static void makeHeader(uint64 size, uint32 id, IoUtilities::BinaryWriter &writer)
Writes an MP4 atom header to the specified stream.
Definition: mp4atom.cpp:172
The exception that is thrown when the data to be parsed or to be made seems invalid and therefore can...
Definition: exceptions.h:25
void setSaveFilePath(const std::string &saveFilePath)
Sets the "save file path".
const ChronoUtilities::TimeSpan & duration() const
Returns the duration if known; otherwise returns a TimeSpan of zero ticks.
void reset() override
Discards all parsing results.
const ChronoUtilities::DateTime & modificationTime() const
Returns the time of the last modification if known; otherwise returns a DateTime of zero ticks...
void internalParseTags(Diagnostics &diag) override
Internally called to parse the tags.
uint64 startOffset() const
Returns the start offset in the related stream.
uint64 size() const
Returns size of the current file in bytes.
IoUtilities::NativeFileStream & stream()
Returns the std::fstream for the current instance.
Definition: basicfileinfo.h:79
bool isHeaderValid() const
Returns an indication whether the track header is valid.
uint64 requiredSize(Diagnostics &diag) const
Returns the number of bytes written when calling makeTrack().
Definition: mp4track.cpp:1103
ImplementationType * subelementByPath(Diagnostics &diag, IdentifierType item)
Returns the sub element for the specified path.
void parseHeader(Diagnostics &diag)
Parses technical information about the track from the header.
ElementPosition determineTagPosition(Diagnostics &diag) const override
Determines the position of the tags inside the file.
MediaFileInfo & fileInfo() const
Returns the related file info.
The class inherits from std::exception and serves as base class for exceptions thrown by the elements...
Definition: exceptions.h:11
const IdentifierType & id() const
Returns the element ID.
ImplementationType * childById(const IdentifierType &id, Diagnostics &diag)
Returns the first child with the specified id.
Contains all classes and functions of the TagInfo library.
Definition: aaccodebook.h:9
void nextStepOrStop(const std::string &step, byte stepPercentage=0)
Throws an OperationAbortedException if aborted; otherwise the data for the next step is set...
ElementPosition
Definition: settings.h:10
std::string idToString() const
Converts the specified atom ID to a printable string.
Definition: mp4atom.h:67
The Diagnostics class is a container for DiagMessage.
Definition: diagnostics.h:156
The GenericContainer class helps parsing header, track, tag and chapter information of a file...