Tag Parser  8.3.0
C++ library for reading and writing MP4 (iTunes), ID3, Vorbis, Opus, FLAC and Matroska tags
mp4container.cpp
Go to the documentation of this file.
1 #include "./mp4container.h"
2 #include "./mp4ids.h"
3 
4 #include "../backuphelper.h"
5 #include "../exceptions.h"
6 #include "../mediafileinfo.h"
7 
8 #include <c++utilities/conversion/stringbuilder.h>
9 #include <c++utilities/io/binaryreader.h>
10 #include <c++utilities/io/binarywriter.h>
11 #include <c++utilities/io/catchiofailure.h>
12 #include <c++utilities/io/copy.h>
13 
14 #include <unistd.h>
15 
16 #include <memory>
17 #include <numeric>
18 #include <tuple>
19 
20 using namespace std;
21 using namespace IoUtilities;
22 using namespace ConversionUtilities;
23 using namespace ChronoUtilities;
24 
25 namespace TagParser {
26 
35 Mp4Container::Mp4Container(MediaFileInfo &fileInfo, uint64 startOffset)
36  : GenericContainer<MediaFileInfo, Mp4Tag, Mp4Track, Mp4Atom>(fileInfo, startOffset)
37  , m_fragmented(false)
38 {
39 }
40 
42 {
43 }
44 
46 {
48  m_fragmented = false;
49 }
50 
52 {
53  if (m_firstElement) {
54  const Mp4Atom *mediaDataAtom = m_firstElement->siblingById(Mp4AtomIds::MediaData, diag);
55  const Mp4Atom *userDataAtom = m_firstElement->subelementByPath(diag, Mp4AtomIds::Movie, Mp4AtomIds::UserData);
56  if (mediaDataAtom && userDataAtom) {
57  return userDataAtom->startOffset() < mediaDataAtom->startOffset() ? ElementPosition::BeforeData : ElementPosition::AfterData;
58  }
59  }
60  return ElementPosition::Keep;
61 }
62 
64 {
65  if (m_firstElement) {
66  const Mp4Atom *mediaDataAtom = m_firstElement->siblingById(Mp4AtomIds::MediaData, diag);
67  const Mp4Atom *movieAtom = m_firstElement->siblingById(Mp4AtomIds::Movie, diag);
68  if (mediaDataAtom && movieAtom) {
69  return movieAtom->startOffset() < mediaDataAtom->startOffset() ? ElementPosition::BeforeData : ElementPosition::AfterData;
70  }
71  }
72  return ElementPosition::Keep;
73 }
74 
76 {
77  //const string context("parsing header of MP4 container"); will be used when generating notifications
78  m_firstElement = make_unique<Mp4Atom>(*this, startOffset());
79  m_firstElement->parse(diag);
80  auto *const ftypAtom = m_firstElement->siblingByIdIncludingThis(Mp4AtomIds::FileType, diag);
81  if (!ftypAtom) {
82  m_doctype.clear();
83  m_version = 0;
84  return;
85  }
86  stream().seekg(static_cast<iostream::off_type>(ftypAtom->dataOffset()));
87  m_doctype = reader().readString(4);
88  m_version = reader().readUInt32BE();
89 }
90 
92 {
93  const string context("parsing tags of MP4 container");
94  auto *const udtaAtom = firstElement()->subelementByPath(diag, Mp4AtomIds::Movie, Mp4AtomIds::UserData);
95  if (!udtaAtom) {
96  return;
97  }
98  auto *metaAtom = udtaAtom->childById(Mp4AtomIds::Meta, diag);
99  bool surplusMetaAtoms = false;
100  while (metaAtom) {
101  metaAtom->parse(diag);
102  m_tags.emplace_back(make_unique<Mp4Tag>());
103  try {
104  m_tags.back()->parse(*metaAtom, diag);
105  } catch (const NoDataFoundException &) {
106  m_tags.pop_back();
107  }
108  if ((metaAtom = metaAtom->siblingById(Mp4AtomIds::Meta, diag))) {
109  surplusMetaAtoms = true;
110  }
111  if (!m_tags.empty()) {
112  break;
113  }
114  }
115  if (surplusMetaAtoms) {
116  diag.emplace_back(DiagLevel::Warning, "udta atom contains multiple meta atoms. Surplus meta atoms will be ignored.", context);
117  }
118 }
119 
121 {
122  static const string context("parsing tracks of MP4 container");
123  try {
124  // get moov atom which holds track information
125  if (Mp4Atom *moovAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::Movie, diag)) {
126  // get mvhd atom which holds overall track information
127  if (Mp4Atom *mvhdAtom = moovAtom->childById(Mp4AtomIds::MovieHeader, diag)) {
128  if (mvhdAtom->dataSize() > 0) {
129  stream().seekg(static_cast<iostream::off_type>(mvhdAtom->dataOffset()));
130  byte version = reader().readByte();
131  if ((version == 1 && mvhdAtom->dataSize() >= 32) || (mvhdAtom->dataSize() >= 20)) {
132  stream().seekg(3, ios_base::cur); // skip flags
133  switch (version) {
134  case 0:
135  m_creationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt32BE());
136  m_modificationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt32BE());
137  m_timeScale = reader().readUInt32BE();
138  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt32BE()) / static_cast<double>(m_timeScale));
139  break;
140  case 1:
141  m_creationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt64BE());
142  m_modificationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt64BE());
143  m_timeScale = reader().readUInt32BE();
144  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()) / static_cast<double>(m_timeScale));
145  break;
146  default:;
147  }
148  } else {
149  diag.emplace_back(DiagLevel::Critical, "mvhd atom is truncated.", context);
150  }
151  } else {
152  diag.emplace_back(DiagLevel::Critical, "mvhd atom is empty.", context);
153  }
154  } else {
155  diag.emplace_back(DiagLevel::Critical, "mvhd atom is does not exist.", context);
156  }
157  // get mvex atom which holds default values for fragmented files
159  m_fragmented = true;
160  if (mehdAtom->dataSize() > 0) {
161  stream().seekg(static_cast<iostream::off_type>(mehdAtom->dataOffset()));
162  unsigned int durationSize = reader().readByte() == 1u ? 8u : 4u; // duration size depends on atom version
163  if (mehdAtom->dataSize() >= 4 + durationSize) {
164  stream().seekg(3, ios_base::cur); // skip flags
165  switch (durationSize) {
166  case 4u:
167  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt32BE()) / static_cast<double>(m_timeScale));
168  break;
169  case 8u:
170  m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()) / static_cast<double>(m_timeScale));
171  break;
172  default:;
173  }
174  } else {
175  diag.emplace_back(DiagLevel::Warning, "mehd atom is truncated.", context);
176  }
177  }
178  }
179  // get first trak atoms which hold information for each track
180  Mp4Atom *trakAtom = moovAtom->childById(Mp4AtomIds::Track, diag);
181  int trackNum = 1;
182  while (trakAtom) {
183  try {
184  trakAtom->parse(diag);
185  } catch (const Failure &) {
186  diag.emplace_back(DiagLevel::Warning, "Unable to parse child atom of moov.", context);
187  }
188  // parse the trak atom using the Mp4Track class
189  m_tracks.emplace_back(make_unique<Mp4Track>(*trakAtom));
190  try { // try to parse header
191  m_tracks.back()->parseHeader(diag);
192  } catch (const Failure &) {
193  diag.emplace_back(DiagLevel::Critical, argsToString("Unable to parse track ", trackNum, '.'), context);
194  }
195  trakAtom = trakAtom->siblingById(Mp4AtomIds::Track, diag); // get next trak atom
196  ++trackNum;
197  }
198  // get overall duration, creation time and modification time if not determined yet
199  if (m_duration.isNull() || m_modificationTime.isNull() || m_creationTime.isNull()) {
200  for (const auto &track : tracks()) {
201  if (track->duration() > m_duration) {
203  }
206  }
207  if (track->creationTime() < m_creationTime) {
209  }
210  }
211  }
212  }
213  } catch (const Failure &) {
214  diag.emplace_back(DiagLevel::Warning, "Unable to parse moov atom.", context);
215  }
216 }
217 
219 {
220  static const string context("making MP4 container");
221  progress.updateStep("Calculating atom sizes and padding ...");
222 
223  // basic validation of original file
224  if (!isHeaderParsed()) {
225  diag.emplace_back(DiagLevel::Critical, "The header has not been parsed yet.", context);
226  throw InvalidDataException();
227  }
228 
229  // define variables needed to parse atoms of original file
230  if (!firstElement()) {
231  diag.emplace_back(DiagLevel::Critical, "No MP4 atoms could be found.", context);
232  throw InvalidDataException();
233  }
234 
235  // define variables needed to manage file layout
236  // -> whether media data is written chunk by chunk (need to write chunk by chunk if tracks have been altered)
237  const bool writeChunkByChunk = m_tracksAltered;
238  // -> whether rewrite is required (always required when forced to rewrite or when tracks have been altered)
239  bool rewriteRequired = fileInfo().isForcingRewrite() || writeChunkByChunk;
240  // -> use the preferred tag position/index position (force one wins, if both are force tag pos wins; might be changed later if none is forced)
241  ElementPosition initialNewTagPos
243  ElementPosition newTagPos = initialNewTagPos;
244  // -> current tag position (determined later)
245  ElementPosition currentTagPos;
246  // -> holds new padding (before actual data)
247  uint64 newPadding;
248  // -> holds new padding (after actual data)
249  uint64 newPaddingEnd;
250  // -> holds current offset
251  uint64 currentOffset;
252  // -> holds track information, used when writing chunk-by-chunk
253  vector<tuple<istream *, vector<uint64>, vector<uint64>>> trackInfos;
254  // -> holds offsets of media data atoms in original file, used when simply copying mdat
255  vector<int64> origMediaDataOffsets;
256  // -> holds offsets of media data atoms in new file, used when simply copying mdat
257  vector<int64> newMediaDataOffsets;
258  // -> new size of movie atom and user data atom
259  uint64 movieAtomSize, userDataAtomSize;
260  // -> track count of original file
261  const auto trackCount = this->trackCount();
262 
263  // find relevant atoms in original file
264  Mp4Atom *fileTypeAtom, *progressiveDownloadInfoAtom, *movieAtom, *firstMediaDataAtom, *firstMovieFragmentAtom /*, *userDataAtom*/;
265  Mp4Atom *level0Atom, *level1Atom, *level2Atom, *lastAtomToBeWritten;
266  try {
267  // file type atom (mandatory)
268  if ((fileTypeAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::FileType, diag))) {
269  // buffer atom
270  fileTypeAtom->makeBuffer();
271  } else {
272  // throw error if missing
273  diag.emplace_back(DiagLevel::Critical, "Mandatory \"ftyp\"-atom not found.", context);
274  throw InvalidDataException();
275  }
276 
277  // progressive download information atom (not mandatory)
278  if ((progressiveDownloadInfoAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::ProgressiveDownloadInformation, diag))) {
279  // buffer atom
280  progressiveDownloadInfoAtom->makeBuffer();
281  }
282 
283  // movie atom (mandatory)
284  if (!(movieAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::Movie, diag))) {
285  // throw error if missing
286  diag.emplace_back(DiagLevel::Critical, "Mandatory \"moov\"-atom not in the source file found.", context);
287  throw InvalidDataException();
288  }
289 
290  // movie fragment atom (indicates dash file)
291  if ((firstMovieFragmentAtom = firstElement()->siblingById(Mp4AtomIds::MovieFragment, diag))) {
292  // there is at least one movie fragment atom -> consider file being dash
293  // -> can not write chunk-by-chunk (currently)
294  if (writeChunkByChunk) {
295  diag.emplace_back(DiagLevel::Critical, "Writing chunk-by-chunk is not implemented for DASH files.", context);
296  throw NotImplementedException();
297  }
298  // -> tags must be placed at the beginning
299  newTagPos = ElementPosition::BeforeData;
300  }
301 
302  // media data atom (mandatory?)
303  // -> consider not only mdat as media data atom; consider everything not handled otherwise as media data
304  for (firstMediaDataAtom = nullptr, level0Atom = firstElement(); level0Atom; level0Atom = level0Atom->nextSibling()) {
305  level0Atom->parse(diag);
306  switch (level0Atom->id()) {
309  case Mp4AtomIds::Movie:
310  case Mp4AtomIds::Free:
311  case Mp4AtomIds::Skip:
312  continue;
313  default:
314  firstMediaDataAtom = level0Atom;
315  }
316  break;
317  }
318 
319  // determine current tag position
320  // -> since tags are nested in the movie atom its position is relevant here
321  if (firstMediaDataAtom) {
322  currentTagPos = firstMediaDataAtom->startOffset() < movieAtom->startOffset() ? ElementPosition::AfterData : ElementPosition::BeforeData;
323  if (newTagPos == ElementPosition::Keep) {
324  newTagPos = currentTagPos;
325  }
326  } else {
327  currentTagPos = ElementPosition::Keep;
328  }
329 
330  // ensure index and tags are always placed at the beginning when dealing with DASH files
331  if (firstMovieFragmentAtom) {
332  if (initialNewTagPos == ElementPosition::AfterData) {
333  diag.emplace_back(
334  DiagLevel::Warning, "Sorry, but putting index/tags at the end is not possible when dealing with DASH files.", context);
335  }
336  initialNewTagPos = newTagPos = ElementPosition::BeforeData;
337  }
338 
339  // user data atom (currently not used)
340  //userDataAtom = movieAtom->childById(Mp4AtomIds::UserData);
341 
342  } catch (const NotImplementedException &) {
343  throw;
344 
345  } catch (const Failure &) {
346  // can't ignore parsing errors here
347  diag.emplace_back(DiagLevel::Critical, "Unable to parse the overall atom structure of the source file.", context);
348  throw InvalidDataException();
349  }
350 
351  progress.stopIfAborted();
352 
353  // calculate sizes
354  // -> size of tags
355  vector<Mp4TagMaker> tagMaker;
356  uint64 tagsSize = 0;
357  tagMaker.reserve(m_tags.size());
358  for (auto &tag : m_tags) {
359  try {
360  tagMaker.emplace_back(tag->prepareMaking(diag));
361  tagsSize += tagMaker.back().requiredSize();
362  } catch (const Failure &) {
363  }
364  }
365 
366  // -> size of movie atom (contains track and tag information)
367  movieAtomSize = userDataAtomSize = 0;
368  try {
369  // add size of children
370  for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
371  for (level1Atom = level0Atom->firstChild(); level1Atom; level1Atom = level1Atom->nextSibling()) {
372  level1Atom->parse(diag);
373  switch (level1Atom->id()) {
375  try {
376  for (level2Atom = level1Atom->firstChild(); level2Atom; level2Atom = level2Atom->nextSibling()) {
377  level2Atom->parse(diag);
378  switch (level2Atom->id()) {
379  case Mp4AtomIds::Meta:
380  // ignore meta data here; it is added separately
381  break;
382  default:
383  // add size of unknown childs of the user data atom
384  userDataAtomSize += level2Atom->totalSize();
385  level2Atom->makeBuffer();
386  }
387  }
388  } catch (const Failure &) {
389  // invalid children might be ignored as not mandatory
390  diag.emplace_back(
391  DiagLevel::Critical, "Unable to parse the children of \"udta\"-atom of the source file; ignoring them.", context);
392  }
393  break;
394  case Mp4AtomIds::Track:
395  // ignore track atoms here; they are added separately
396  break;
397  default:
398  // add size of unknown childs of the movie atom
399  movieAtomSize += level1Atom->totalSize();
400  level1Atom->makeBuffer();
401  }
402  }
403  }
404 
405  // add size of meta data
406  if (userDataAtomSize += tagsSize) {
407  Mp4Atom::addHeaderSize(userDataAtomSize);
408  movieAtomSize += userDataAtomSize;
409  }
410 
411  // add size of track atoms
412  for (const auto &track : tracks()) {
413  movieAtomSize += track->requiredSize(diag);
414  }
415 
416  // add header size
417  Mp4Atom::addHeaderSize(movieAtomSize);
418  } catch (const Failure &) {
419  // can't ignore parsing errors here
420  diag.emplace_back(DiagLevel::Critical, "Unable to parse the children of \"moov\"-atom of the source file.", context);
421  throw InvalidDataException();
422  }
423 
424  progress.stopIfAborted();
425 
426  // check whether there are atoms to be voided after movie next sibling (only relevant when not rewriting)
427  if (!rewriteRequired) {
428  newPaddingEnd = 0;
429  uint64 currentSum = 0;
430  for (Mp4Atom *level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
431  level0Atom->parse(diag);
432  switch (level0Atom->id()) {
435  case Mp4AtomIds::Movie:
436  case Mp4AtomIds::Free:
437  case Mp4AtomIds::Skip:
438  // must void these if they occur "between" the media data
439  currentSum += level0Atom->totalSize();
440  break;
441  default:
442  newPaddingEnd += currentSum;
443  currentSum = 0;
444  lastAtomToBeWritten = level0Atom;
445  }
446  }
447  }
448 
449  // calculate padding if no rewrite is required; otherwise use the preferred padding
450 calculatePadding:
451  if (rewriteRequired) {
452  newPadding = (fileInfo().preferredPadding() && fileInfo().preferredPadding() < 8 ? 8 : fileInfo().preferredPadding());
453  } else {
454  // file type atom
455  currentOffset = fileTypeAtom->totalSize();
456 
457  // progressive download information atom
458  if (progressiveDownloadInfoAtom) {
459  currentOffset += progressiveDownloadInfoAtom->totalSize();
460  }
461 
462  // if writing tags before data: movie atom (contains tag)
463  switch (newTagPos) {
466  currentOffset += movieAtomSize;
467  break;
468  default:;
469  }
470 
471  // check whether there is sufficiant space before the next atom
472  if (!(rewriteRequired = firstMediaDataAtom && currentOffset > firstMediaDataAtom->startOffset())) {
473  // there is sufficiant space
474  // -> check whether the padding matches specifications
475  // min padding: says "at least ... byte should be reserved to prepend further tag info", so the padding at the end
476  // shouldn't be tanken into account (it can't be used to prepend further tag info)
477  // max padding: says "do not waste more than ... byte", so here all padding should be taken into account
478  newPadding = firstMediaDataAtom->startOffset() - currentOffset;
479  rewriteRequired = (newPadding > 0 && newPadding < 8) || newPadding < fileInfo().minPadding()
480  || (newPadding + newPaddingEnd) > fileInfo().maxPadding();
481  }
482  if (rewriteRequired) {
483  // can't put the tags before media data
484  if (!firstMovieFragmentAtom && !fileInfo().forceTagPosition() && !fileInfo().forceIndexPosition()
485  && newTagPos != ElementPosition::AfterData) {
486  // writing tag before media data is not forced, its not a DASH file and tags aren't already at the end
487  // -> try to put the tags at the end
488  newTagPos = ElementPosition::AfterData;
489  rewriteRequired = false;
490  } else {
491  // writing tag before media data is forced -> rewrite the file
492  // when rewriting anyways, ensure the preferred tag position is used
493  newTagPos = initialNewTagPos == ElementPosition::Keep ? currentTagPos : initialNewTagPos;
494  }
495  // in any case: recalculate padding
496  goto calculatePadding;
497  } else {
498  // tags can be put before the media data
499  // -> ensure newTagPos is not ElementPosition::Keep
500  if (newTagPos == ElementPosition::Keep) {
501  newTagPos = ElementPosition::BeforeData;
502  }
503  }
504  }
505 
506  // setup stream(s) for writing
507  // -> update status
508  progress.nextStepOrStop("Preparing streams ...");
509 
510  // -> define variables needed to handle output stream and backup stream (required when rewriting the file)
511  string backupPath;
512  NativeFileStream &outputStream = fileInfo().stream();
513  NativeFileStream backupStream; // create a stream to open the backup/original file for the case rewriting the file is required
514  BinaryWriter outputWriter(&outputStream);
515 
516  if (rewriteRequired) {
517  if (fileInfo().saveFilePath().empty()) {
518  // move current file to temp dir and reopen it as backupStream, recreate original file
519  try {
520  BackupHelper::createBackupFile(fileInfo().backupDirectory(), fileInfo().path(), backupPath, outputStream, backupStream);
521  // recreate original file, define buffer variables
522  outputStream.open(fileInfo().path(), ios_base::out | ios_base::binary | ios_base::trunc);
523  } catch (...) {
524  const char *what = catchIoFailure();
525  diag.emplace_back(DiagLevel::Critical, "Creation of temporary file (to rewrite the original file) failed.", context);
526  throwIoFailure(what);
527  }
528  } else {
529  // open the current file as backupStream and create a new outputStream at the specified "save file path"
530  try {
531  backupStream.exceptions(ios_base::badbit | ios_base::failbit);
532  backupStream.open(fileInfo().path(), ios_base::in | ios_base::binary);
533  fileInfo().close();
534  outputStream.open(fileInfo().saveFilePath(), ios_base::out | ios_base::binary | ios_base::trunc);
535  } catch (...) {
536  const char *what = catchIoFailure();
537  diag.emplace_back(DiagLevel::Critical, "Opening streams to write output file failed.", context);
538  throwIoFailure(what);
539  }
540  }
541 
542  // set backup stream as associated input stream since we need the original elements to write the new file
543  setStream(backupStream);
544 
545  // TODO: reduce code duplication
546 
547  } else { // !rewriteRequired
548  // ensure everything to make track atoms is buffered before altering the source file
549  for (const auto &track : tracks()) {
550  track->bufferTrackAtoms(diag);
551  }
552 
553  // reopen original file to ensure it is opened for writing
554  try {
555  fileInfo().close();
556  outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
557  } catch (...) {
558  const char *what = catchIoFailure();
559  diag.emplace_back(DiagLevel::Critical, "Opening the file with write permissions failed.", context);
560  throwIoFailure(what);
561  }
562  }
563 
564  // start actual writing
565  try {
566  // write header
567  progress.nextStepOrStop("Writing header and tags ...");
568  // -> make file type atom
569  fileTypeAtom->copyBuffer(outputStream);
570  fileTypeAtom->discardBuffer();
571  // -> make progressive download info atom
572  if (progressiveDownloadInfoAtom) {
573  progressiveDownloadInfoAtom->copyBuffer(outputStream);
574  progressiveDownloadInfoAtom->discardBuffer();
575  }
576 
577  // set input/output streams of each track
578  for (auto &track : tracks()) {
579  // ensure the track reads from the original file
580  if (&track->inputStream() == &outputStream) {
581  track->setInputStream(backupStream);
582  }
583  // ensure the track writes to the output file
584  track->setOutputStream(outputStream);
585  }
586 
587  // write movie atom / padding and media data
588  for (byte pass = 0; pass != 2; ++pass) {
589  if (newTagPos == (pass ? ElementPosition::AfterData : ElementPosition::BeforeData)) {
590  // define function to write tracks
591  bool tracksWritten = false;
592  const auto writeTracks = [&] {
593  if (tracksWritten) {
594  return;
595  }
596 
597  for (auto &track : tracks()) {
598  track->makeTrack(diag);
599  }
600  tracksWritten = true;
601  };
602 
603  // define function to write user data
604  bool userDataWritten = false;
605  const auto writeUserData = [&] {
606  if (userDataWritten || !userDataAtomSize) {
607  return;
608  }
609 
610  // writer user data atom header
611  Mp4Atom::makeHeader(userDataAtomSize, Mp4AtomIds::UserData, outputWriter);
612 
613  // write children of user data atom
614  bool metaAtomWritten = false;
615  for (Mp4Atom *level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
616  for (Mp4Atom *level1Atom = level0Atom->childById(Mp4AtomIds::UserData, diag); level1Atom;
617  level1Atom = level1Atom->siblingById(Mp4AtomIds::UserData, diag)) {
618  for (Mp4Atom *level2Atom = level1Atom->firstChild(); level2Atom; level2Atom = level2Atom->nextSibling()) {
619  switch (level2Atom->id()) {
620  case Mp4AtomIds::Meta:
621  // write meta atom
622  for (auto &maker : tagMaker) {
623  maker.make(outputStream, diag);
624  }
625  metaAtomWritten = true;
626  break;
627  default:
628  // write buffered data
629  level2Atom->copyBuffer(outputStream);
630  level2Atom->discardBuffer();
631  }
632  }
633  }
634  }
635 
636  // write meta atom if not already written
637  if (!metaAtomWritten) {
638  for (auto &maker : tagMaker) {
639  maker.make(outputStream, diag);
640  }
641  }
642 
643  userDataWritten = true;
644  };
645 
646  // write movie atom
647  // -> write movie atom header
648  Mp4Atom::makeHeader(movieAtomSize, Mp4AtomIds::Movie, outputWriter);
649 
650  // -> write children of movie atom preserving the original order
651  for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
652  for (level1Atom = level0Atom->firstChild(); level1Atom; level1Atom = level1Atom->nextSibling()) {
653  switch (level1Atom->id()) {
654  case Mp4AtomIds::Track:
655  writeTracks();
656  break;
658  writeUserData();
659  break;
660  default:
661  // write buffered data
662  level1Atom->copyBuffer(outputStream);
663  level1Atom->discardBuffer();
664  }
665  }
666  }
667 
668  // -> write tracks and user data atoms if not already happened within the loop
669  writeTracks();
670  writeUserData();
671 
672  } else {
673  // write padding
674  if (newPadding) {
675  // write free atom header
676  if (newPadding < numeric_limits<uint32>::max()) {
677  outputWriter.writeUInt32BE(static_cast<uint32>(newPadding));
678  outputWriter.writeUInt32BE(Mp4AtomIds::Free);
679  newPadding -= 8;
680  } else {
681  outputWriter.writeUInt32BE(1);
682  outputWriter.writeUInt32BE(Mp4AtomIds::Free);
683  outputWriter.writeUInt64BE(newPadding);
684  newPadding -= 16;
685  }
686 
687  // write zeroes
688  for (; newPadding; --newPadding) {
689  outputStream.put(0);
690  }
691  }
692 
693  // write media data
694  if (rewriteRequired) {
695  for (level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
696  level0Atom->parse(diag);
697  switch (level0Atom->id()) {
700  case Mp4AtomIds::Movie:
701  case Mp4AtomIds::Free:
702  case Mp4AtomIds::Skip:
703  break;
705  if (writeChunkByChunk) {
706  // write actual data separately when writing chunk-by-chunk
707  break;
708  } else {
709  // store media data offsets when not writing chunk-by-chunk to be able to update chunk offset table
710  origMediaDataOffsets.push_back(static_cast<int64>(level0Atom->startOffset()));
711  newMediaDataOffsets.push_back(outputStream.tellp());
712  }
713  FALLTHROUGH;
714  default:
715  // update status
716  progress.updateStep("Writing atom: " + level0Atom->idToString());
717  // copy atom entirely and forward status update calls
718  level0Atom->copyEntirely(outputStream, diag, &progress);
719  }
720  }
721 
722  // when writing chunk-by-chunk write media data now
723  if (writeChunkByChunk) {
724  // read chunk offset and chunk size table from the old file which are required to get chunks
725  progress.updateStep("Reading chunk offsets and sizes from the original file ...");
726  trackInfos.reserve(trackCount);
727  uint64 totalChunkCount = 0;
728  uint64 totalMediaDataSize = 0;
729  for (auto &track : tracks()) {
730  progress.stopIfAborted();
731 
732  // emplace information
733  trackInfos.emplace_back(
734  &track->inputStream(), track->readChunkOffsets(fileInfo().isForcingFullParse(), diag), track->readChunkSizes(diag));
735 
736  // check whether the chunks could be parsed correctly
737  const vector<uint64> &chunkOffsetTable = get<1>(trackInfos.back());
738  const vector<uint64> &chunkSizesTable = get<2>(trackInfos.back());
739  if (track->chunkCount() != chunkOffsetTable.size() || track->chunkCount() != chunkSizesTable.size()) {
740  diag.emplace_back(DiagLevel::Critical,
741  "Chunks of track " % numberToString<uint64, string>(track->id()) + " could not be parsed correctly.", context);
742  }
743 
744  // increase total chunk count and size
745  totalChunkCount += track->chunkCount();
746  totalMediaDataSize += accumulate(chunkSizesTable.cbegin(), chunkSizesTable.cend(), 0ul);
747  }
748 
749  // write media data chunk-by-chunk
750  // -> write header of media data atom
751  Mp4Atom::addHeaderSize(totalMediaDataSize);
752  Mp4Atom::makeHeader(totalMediaDataSize, Mp4AtomIds::MediaData, outputWriter);
753 
754  // -> copy chunks
755  CopyHelper<0x2000> copyHelper;
756  uint64 chunkIndexWithinTrack = 0, totalChunksCopied = 0;
757  bool anyChunksCopied;
758  do {
759  progress.stopIfAborted();
760 
761  // copy a chunk from each track
762  anyChunksCopied = false;
763  for (size_t trackIndex = 0; trackIndex < trackCount; ++trackIndex) {
764  // get source stream and tables for current track
765  auto &trackInfo = trackInfos[trackIndex];
766  istream &sourceStream = *get<0>(trackInfo);
767  vector<uint64> &chunkOffsetTable = get<1>(trackInfo);
768  const vector<uint64> &chunkSizesTable = get<2>(trackInfo);
769 
770  // still chunks to be copied (of this track)?
771  if (chunkIndexWithinTrack < chunkOffsetTable.size() && chunkIndexWithinTrack < chunkSizesTable.size()) {
772  // copy chunk, update entry in chunk offset table
773  sourceStream.seekg(static_cast<streamoff>(chunkOffsetTable[chunkIndexWithinTrack]));
774  chunkOffsetTable[chunkIndexWithinTrack] = static_cast<uint64>(outputStream.tellp());
775  copyHelper.copy(sourceStream, outputStream, chunkSizesTable[chunkIndexWithinTrack]);
776 
777  // update counter / status
778  anyChunksCopied = true;
779  ++totalChunksCopied;
780  }
781  }
782 
783  // incrase chunk index within track, update progress percentage
784  if (!(++chunkIndexWithinTrack % 10)) {
785  progress.updateStepPercentage(static_cast<byte>(totalChunksCopied * 100 / totalChunkCount));
786  }
787 
788  } while (anyChunksCopied);
789  }
790 
791  } else {
792  // can't just skip next movie sibling
793  for (Mp4Atom *level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
794  level0Atom->parse(diag);
795  switch (level0Atom->id()) {
798  case Mp4AtomIds::Movie:
799  // must void these if they occur "between" the media data
800  outputStream.seekp(4, ios_base::cur);
801  outputWriter.writeUInt32BE(Mp4AtomIds::Free);
802  break;
803  default:
804  outputStream.seekp(static_cast<iostream::off_type>(level0Atom->totalSize()), ios_base::cur);
805  }
806  if (level0Atom == lastAtomToBeWritten) {
807  break;
808  }
809  }
810  }
811  }
812  }
813 
814  // reparse what is written so far
815  progress.updateStep("Reparsing output file ...");
816  if (rewriteRequired) {
817  // report new size
818  fileInfo().reportSizeChanged(static_cast<uint64>(outputStream.tellp()));
819  // "save as path" is now the regular path
820  if (!fileInfo().saveFilePath().empty()) {
821  fileInfo().reportPathChanged(fileInfo().saveFilePath());
822  fileInfo().setSaveFilePath(string());
823  }
824  // the outputStream needs to be reopened to be able to read again
825  outputStream.close();
826  outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
827  setStream(outputStream);
828  } else {
829  const auto newSize = static_cast<uint64>(outputStream.tellp());
830  if (newSize < fileInfo().size()) {
831  // file is smaller after the modification -> truncate
832  // -> close stream before truncating
833  outputStream.close();
834  // -> truncate file
835  if (truncate(fileInfo().path().c_str(), static_cast<iostream::off_type>(newSize)) == 0) {
836  fileInfo().reportSizeChanged(newSize);
837  } else {
838  diag.emplace_back(DiagLevel::Critical, "Unable to truncate the file.", context);
839  }
840  // -> reopen the stream again
841  outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
842  } else {
843  // file is longer after the modification -> just report new size
844  fileInfo().reportSizeChanged(newSize);
845  }
846  }
847 
848  reset();
849  try {
850  parseTracks(diag);
851  } catch (const Failure &) {
852  diag.emplace_back(DiagLevel::Critical, "Unable to reparse the new file.", context);
853  throw;
854  }
855 
856  if (rewriteRequired) {
857  // check whether track count of new file equals track count of old file
858  if (trackCount != tracks().size()) {
859  diag.emplace_back(DiagLevel::Critical,
860  argsToString("Unable to update chunk offsets (\"stco\"-atom): Number of tracks in the output file (", tracks().size(),
861  ") differs from the number of tracks in the original file (", trackCount, ")."),
862  context);
863  throw Failure();
864  }
865 
866  // update chunk offset table
867  if (writeChunkByChunk) {
868  progress.updateStep("Updating chunk offset table for each track ...");
869  for (size_t trackIndex = 0; trackIndex != trackCount; ++trackIndex) {
870  const auto &track = tracks()[trackIndex];
871  const auto &chunkOffsetTable = get<1>(trackInfos[trackIndex]);
872  if (track->chunkCount() == chunkOffsetTable.size()) {
873  track->updateChunkOffsets(chunkOffsetTable);
874  } else {
875  diag.emplace_back(DiagLevel::Critical,
876  argsToString("Unable to update chunk offsets of track ", (trackIndex + 1),
877  ": Number of chunks in the output file differs from the number of chunks in the orignal file."),
878  context);
879  throw Failure();
880  }
881  }
882  } else {
883  progress.updateStep("Updating chunk offset table for each track ...");
884  updateOffsets(origMediaDataOffsets, newMediaDataOffsets, diag);
885  }
886  }
887 
888  // flush output stream
889  outputStream.flush();
890 
891  // handle errors (which might have been occurred after renaming/creating backup file)
892  } catch (...) {
893  BackupHelper::handleFailureAfterFileModified(fileInfo(), backupPath, outputStream, backupStream, diag, context);
894  }
895 }
896 
909 void Mp4Container::updateOffsets(const std::vector<int64> &oldMdatOffsets, const std::vector<int64> &newMdatOffsets, Diagnostics &diag)
910 {
911  // do NOT invalidate the status here since this method is internally called by internalMakeFile(), just update the status
912  const string context("updating MP4 container chunk offset table");
913  if (!firstElement()) {
914  diag.emplace_back(DiagLevel::Critical, "No MP4 atoms could be found.", context);
915  throw InvalidDataException();
916  }
917  // update "base-data-offset-present" of "tfhd"-atom (NOT tested properly)
918  try {
919  for (Mp4Atom *moofAtom = firstElement()->siblingById(Mp4AtomIds::MovieFragment, diag); moofAtom;
920  moofAtom = moofAtom->siblingById(Mp4AtomIds::MovieFragment, diag)) {
921  moofAtom->parse(diag);
922  try {
923  for (Mp4Atom *trafAtom = moofAtom->childById(Mp4AtomIds::TrackFragment, diag); trafAtom;
924  trafAtom = trafAtom->siblingById(Mp4AtomIds::TrackFragment, diag)) {
925  trafAtom->parse(diag);
926  int tfhdAtomCount = 0;
927  for (Mp4Atom *tfhdAtom = trafAtom->childById(Mp4AtomIds::TrackFragmentHeader, diag); tfhdAtom;
928  tfhdAtom = tfhdAtom->siblingById(Mp4AtomIds::TrackFragmentHeader, diag)) {
929  tfhdAtom->parse(diag);
930  ++tfhdAtomCount;
931  if (tfhdAtom->dataSize() < 8) {
932  diag.emplace_back(DiagLevel::Warning, "tfhd atom is truncated.", context);
933  continue;
934  }
935  stream().seekg(static_cast<iostream::off_type>(tfhdAtom->dataOffset()) + 1);
936  uint32 flags = reader().readUInt24BE();
937  if (!(flags & 1)) {
938  continue;
939  }
940  if (tfhdAtom->dataSize() < 16) {
941  diag.emplace_back(DiagLevel::Warning, "tfhd atom (denoting base-data-offset-present) is truncated.", context);
942  continue;
943  }
944  stream().seekg(4, ios_base::cur); // skip track ID
945  uint64 off = reader().readUInt64BE();
946  for (auto iOld = oldMdatOffsets.cbegin(), iNew = newMdatOffsets.cbegin(), end = oldMdatOffsets.cend(); iOld != end;
947  ++iOld, ++iNew) {
948  if (off < static_cast<uint64>(*iOld)) {
949  continue;
950  }
951  off += static_cast<uint64>(*iNew - *iOld);
952  stream().seekp(static_cast<iostream::off_type>(tfhdAtom->dataOffset()) + 8);
953  writer().writeUInt64BE(off);
954  break;
955  }
956  }
957  switch (tfhdAtomCount) {
958  case 0:
959  diag.emplace_back(DiagLevel::Warning, "traf atom doesn't contain mandatory tfhd atom.", context);
960  break;
961  case 1:
962  break;
963  default:
964  diag.emplace_back(
965  DiagLevel::Warning, "traf atom stores multiple tfhd atoms but it should only contain exactly one tfhd atom.", context);
966  }
967  }
968  } catch (const Failure &) {
969  diag.emplace_back(DiagLevel::Critical, "Unable to parse childs of top-level atom moof.", context);
970  }
971  }
972  } catch (const Failure &) {
973  diag.emplace_back(DiagLevel::Critical, "Unable to parse top-level atom moof.", context);
974  }
975  // update each track
976  for (auto &track : tracks()) {
977  if (!track->isHeaderValid()) {
978  try {
979  track->parseHeader(diag);
980  } catch (const Failure &) {
981  diag.emplace_back(DiagLevel::Warning,
982  "The chunk offsets of track " % track->name() + " couldn't be updated because the track seems to be invalid..", context);
983  throw;
984  }
985  }
986  if (track->isHeaderValid()) {
987  try {
988  track->updateChunkOffsets(oldMdatOffsets, newMdatOffsets);
989  } catch (const Failure &) {
990  diag.emplace_back(DiagLevel::Warning, "The chunk offsets of track " % track->name() + " couldn't be updated.", context);
991  throw;
992  }
993  }
994  }
995 }
996 
997 } // namespace TagParser
std::vector< uint64 > readChunkOffsets(bool parseFragments, Diagnostics &diag)
Reads the chunk offsets from the stco atom and fragments if parseFragments is true.
Definition: mp4track.cpp:175
This exception is thrown when the an operation is invoked that has not been implemented yet.
Definition: exceptions.h:60
Implementation of TagParser::Tag for the MP4 container.
Definition: mp4tag.h:97
const std::string name() const
Returns the track name if known; otherwise returns an empty string.
size_t preferredPadding() const
Returns the padding to be written before the data block when applying changes and the file needs to b...
void updateChunkOffsets(const std::vector< int64 > &oldMdatOffsets, const std::vector< int64 > &newMdatOffsets)
Updates the chunk offsets of the track.
Definition: mp4track.cpp:895
void internalMakeFile(Diagnostics &diag, AbortableProgressFeedback &progress) override
Internally called to make the file.
void updateStepPercentage(byte stepPercentage)
Updates the current step percentage and invokes the second callback specified on construction (or the...
std::vector< uint64 > readChunkSizes(TagParser::Diagnostics &diag)
Reads the chunk sizes from the stsz (sample sizes) and stsc (samples per chunk) atom.
Definition: mp4track.cpp:508
static constexpr void addHeaderSize(uint64 &dataSize)
Adds the header size to the specified data size.
Definition: mp4atom.h:81
void copyEntirely(std::ostream &targetStream, Diagnostics &diag, AbortableProgressFeedback *progress)
Writes the entire element including all childs to the specified targetStream.
Mp4Atom * firstElement() const
Returns the first element of the file if available; otherwiese returns nullptr.
bool forceTagPosition() const
Returns whether tagPosition() is forced.
void stopIfAborted() const
Throws an OperationAbortedException if aborted.
void makeBuffer()
Buffers the element (header and data).
ImplementationType * nextSibling()
Returns the next sibling of the element.
void internalParseTracks(Diagnostics &diag) override
Internally called to parse the tracks.
ImplementationType * firstChild()
Returns the first child of the element.
void setInputStream(std::istream &stream)
Assigns another input stream.
const std::vector< std::unique_ptr< Mp4Track > > & tracks() const
Returns the tracks of the file.
bool isForcingRewrite() const
Returns whether forcing rewriting (when applying changes) is enabled.
uint32 chunkCount() const
Returns the number of chunks denoted by the stco atom.
Definition: mp4track.h:227
std::iostream & stream()
Returns the related stream.
uint64 startOffset() const
Returns the start offset in the related stream.
void reset() override
Discards all parsing results.
void bufferTrackAtoms(Diagnostics &diag)
Buffers all atoms required by the makeTrack() method.
Definition: mp4track.cpp:1068
ChronoUtilities::DateTime m_creationTime
size_t maxPadding() const
Returns the maximum padding to be written before the data blocks when applying changes.
void parseTracks(Diagnostics &diag)
Parses the tracks of the file if not parsed yet.
The Mp4Atom class helps to parse MP4 files.
Definition: mp4atom.h:38
uint64 totalSize() const
Returns the total size of the element.
const ChronoUtilities::DateTime & creationTime() const
Returns the creation time if known; otherwise returns a DateTime of zero ticks.
The MediaFileInfo class allows to read and write tag information providing a container/tag format ind...
Definition: mediafileinfo.h:44
TAG_PARSER_EXPORT void createBackupFile(const std::string &backupDir, const std::string &originalPath, std::string &backupPath, IoUtilities::NativeFileStream &originalStream, IoUtilities::NativeFileStream &backupStream)
bool isHeaderParsed() const
Returns an indication whether the header has been parsed yet.
Implementation of TagParser::AbstractTrack for the MP4 container.
Definition: mp4track.h:118
void discardBuffer()
Discards buffered data.
ChronoUtilities::TimeSpan m_duration
uint64 version() const
Returns the version if known; otherwise returns 0.
Contains utility classes helping to read and write streams.
void parse(Diagnostics &diag)
Parses the header information of the element which is read from the related stream at the start offse...
void setStream(std::iostream &stream)
Sets the related stream.
void setOutputStream(std::ostream &stream)
Assigns another output stream.
IoUtilities::BinaryWriter & writer()
Returns the related BinaryWriter.
Mp4TagMaker prepareMaking(Diagnostics &diag)
Prepares making.
Definition: mp4tag.cpp:376
void internalParseHeader(Diagnostics &diag) override
Internally called to parse the header.
The exception that is thrown when the data to be parsed holds no parsable information (e....
Definition: exceptions.h:18
size_t minPadding() const
Returns the minimum padding to be written before the data blocks when applying changes.
std::istream & inputStream()
Returns the associated input stream.
ImplementationType * siblingById(const IdentifierType &id, Diagnostics &diag)
Returns the first sibling with the specified id.
void reportPathChanged(const std::string &newPath)
Call this function to report that the path changed.
bool forceIndexPosition() const
Returns whether indexPosition() is forced.
The AbortableProgressFeedback class provides feedback about an ongoing operation via callbacks.
void close()
A possibly opened std::fstream will be closed.
uint64 id() const
Returns the track ID if known; otherwise returns 0.
void updateStep(const std::string &step, byte stepPercentage=0)
Updates the current step and invokes the first callback specified on construction.
TAG_PARSER_EXPORT void handleFailureAfterFileModified(MediaFileInfo &mediaFileInfo, const std::string &backupPath, IoUtilities::NativeFileStream &outputStream, IoUtilities::NativeFileStream &backupStream, Diagnostics &diag, const std::string &context="making file")
ElementPosition determineIndexPosition(Diagnostics &diag) const override
Determines the position of the index.
const std::string & saveFilePath() const
Returns the "save file path" which has been set using setSaveFilePath().
ElementPosition tagPosition() const
Returns the position (in the output file) where the tag information is written when applying changes.
ElementPosition indexPosition() const
Returns the position (in the output file) where the index is written when applying changes.
void reportSizeChanged(uint64 newSize)
Call this function to report that the size changed.
ChronoUtilities::DateTime m_modificationTime
void copyBuffer(std::ostream &targetStream)
Copies buffered data to targetStream.
void makeTrack(Diagnostics &diag)
Makes the track entry ("trak"-atom) for the track.
Definition: mp4track.cpp:1144
IoUtilities::BinaryReader & reader()
Returns the related BinaryReader.
static void makeHeader(uint64 size, uint32 id, IoUtilities::BinaryWriter &writer)
Writes an MP4 atom header to the specified stream.
Definition: mp4atom.cpp:172
The exception that is thrown when the data to be parsed or to be made seems invalid and therefore can...
Definition: exceptions.h:25
void setSaveFilePath(const std::string &saveFilePath)
Sets the "save file path".
const ChronoUtilities::TimeSpan & duration() const
Returns the duration if known; otherwise returns a TimeSpan of zero ticks.
void reset() override
Discards all parsing results.
const ChronoUtilities::DateTime & modificationTime() const
Returns the time of the last modification if known; otherwise returns a DateTime of zero ticks.
void internalParseTags(Diagnostics &diag) override
Internally called to parse the tags.
uint64 startOffset() const
Returns the start offset in the related stream.
uint64 size() const
Returns size of the current file in bytes.
IoUtilities::NativeFileStream & stream()
Returns the std::fstream for the current instance.
Definition: basicfileinfo.h:79
bool isHeaderValid() const
Returns an indication whether the track header is valid.
uint64 requiredSize(Diagnostics &diag) const
Returns the number of bytes written when calling makeTrack().
Definition: mp4track.cpp:1091
ImplementationType * subelementByPath(Diagnostics &diag, IdentifierType item)
Returns the sub element for the specified path.
void parseHeader(Diagnostics &diag)
Parses technical information about the track from the header.
ElementPosition determineTagPosition(Diagnostics &diag) const override
Determines the position of the tags inside the file.
MediaFileInfo & fileInfo() const
Returns the related file info.
The class inherits from std::exception and serves as base class for exceptions thrown by the elements...
Definition: exceptions.h:11
const IdentifierType & id() const
Returns the element ID.
ImplementationType * childById(const IdentifierType &id, Diagnostics &diag)
Returns the first child with the specified id.
Contains all classes and functions of the TagInfo library.
Definition: aaccodebook.h:9
void nextStepOrStop(const std::string &step, byte stepPercentage=0)
Throws an OperationAbortedException if aborted; otherwise the data for the next step is set.
ElementPosition
Definition: settings.h:10
std::string idToString() const
Converts the specified atom ID to a printable string.
Definition: mp4atom.h:67
The Diagnostics class is a container for DiagMessage.
Definition: diagnostics.h:156
The GenericContainer class helps parsing header, track, tag and chapter information of a file.