Tag Parser 10.3.1
C++ library for reading and writing MP4 (iTunes), ID3, Vorbis, Opus, FLAC and Matroska tags
mp4container.cpp
Go to the documentation of this file.
1#include "./mp4container.h"
2#include "./mp4ids.h"
3
4#include "../backuphelper.h"
5#include "../exceptions.h"
6#include "../mediafileinfo.h"
7
8#include <c++utilities/conversion/stringbuilder.h>
9#include <c++utilities/io/binaryreader.h>
10#include <c++utilities/io/binarywriter.h>
11#include <c++utilities/io/copy.h>
12
13#include <unistd.h>
14
15#include <memory>
16#include <numeric>
17#include <tuple>
18
19using namespace std;
20using namespace CppUtilities;
21
22namespace TagParser {
23
32Mp4Container::Mp4Container(MediaFileInfo &fileInfo, std::uint64_t startOffset)
33 : GenericContainer<MediaFileInfo, Mp4Tag, Mp4Track, Mp4Atom>(fileInfo, startOffset)
34 , m_fragmented(false)
35{
36}
37
39{
40}
41
43{
45 m_fragmented = false;
46}
47
49{
50 if (m_firstElement) {
51 const Mp4Atom *mediaDataAtom = m_firstElement->siblingById(Mp4AtomIds::MediaData, diag);
52 const Mp4Atom *userDataAtom = m_firstElement->subelementByPath(diag, Mp4AtomIds::Movie, Mp4AtomIds::UserData);
53 if (mediaDataAtom && userDataAtom) {
54 return userDataAtom->startOffset() < mediaDataAtom->startOffset() ? ElementPosition::BeforeData : ElementPosition::AfterData;
55 }
56 }
58}
59
61{
62 if (m_firstElement) {
63 const Mp4Atom *mediaDataAtom = m_firstElement->siblingById(Mp4AtomIds::MediaData, diag);
64 const Mp4Atom *movieAtom = m_firstElement->siblingById(Mp4AtomIds::Movie, diag);
65 if (mediaDataAtom && movieAtom) {
66 return movieAtom->startOffset() < mediaDataAtom->startOffset() ? ElementPosition::BeforeData : ElementPosition::AfterData;
67 }
68 }
70}
71
73{
74 CPP_UTILITIES_UNUSED(progress) //const string context("parsing header of MP4 container"); will be used when generating notifications
75 m_firstElement = make_unique<Mp4Atom>(*this, startOffset());
76 m_firstElement->parse(diag);
77 auto *const ftypAtom = m_firstElement->siblingByIdIncludingThis(Mp4AtomIds::FileType, diag);
78 if (!ftypAtom) {
79 m_doctype.clear();
80 m_version = 0;
81 return;
82 }
83 stream().seekg(static_cast<iostream::off_type>(ftypAtom->dataOffset()));
84 m_doctype = reader().readString(4);
85 m_version = reader().readUInt32BE();
86}
87
89{
90 CPP_UTILITIES_UNUSED(progress)
91 const string context("parsing tags of MP4 container");
93 if (!udtaAtom) {
94 return;
95 }
96 auto *metaAtom = udtaAtom->childById(Mp4AtomIds::Meta, diag);
97 bool surplusMetaAtoms = false;
98 while (metaAtom) {
99 metaAtom->parse(diag);
100 m_tags.emplace_back(make_unique<Mp4Tag>());
101 try {
102 m_tags.back()->parse(*metaAtom, diag);
103 } catch (const NoDataFoundException &) {
104 m_tags.pop_back();
105 }
106 if ((metaAtom = metaAtom->siblingById(Mp4AtomIds::Meta, diag))) {
107 surplusMetaAtoms = true;
108 }
109 if (!m_tags.empty()) {
110 break;
111 }
112 }
113 if (surplusMetaAtoms) {
114 diag.emplace_back(DiagLevel::Warning, "udta atom contains multiple meta atoms. Surplus meta atoms will be ignored.", context);
115 }
116}
117
119{
120 static const string context("parsing tracks of MP4 container");
121 try {
122 // get moov atom which holds track information
123 if (Mp4Atom *moovAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::Movie, diag)) {
124 // get mvhd atom which holds overall track information
125 if (Mp4Atom *mvhdAtom = moovAtom->childById(Mp4AtomIds::MovieHeader, diag)) {
126 if (mvhdAtom->dataSize() > 0) {
127 stream().seekg(static_cast<iostream::off_type>(mvhdAtom->dataOffset()));
128 std::uint8_t version = reader().readByte();
129 if ((version == 1 && mvhdAtom->dataSize() >= 32) || (mvhdAtom->dataSize() >= 20)) {
130 stream().seekg(3, ios_base::cur); // skip flags
131 switch (version) {
132 case 0:
133 m_creationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt32BE());
134 m_modificationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(reader().readUInt32BE());
135 m_timeScale = reader().readUInt32BE();
136 m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt32BE()) / static_cast<double>(m_timeScale));
137 break;
138 case 1:
139 m_creationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()));
140 m_modificationTime = DateTime::fromDate(1904, 1, 1) + TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()));
141 m_timeScale = reader().readUInt32BE();
142 m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()) / static_cast<double>(m_timeScale));
143 break;
144 default:;
145 }
146 } else {
147 diag.emplace_back(DiagLevel::Critical, "mvhd atom is truncated.", context);
148 }
149 } else {
150 diag.emplace_back(DiagLevel::Critical, "mvhd atom is empty.", context);
151 }
152 } else {
153 diag.emplace_back(DiagLevel::Critical, "mvhd atom is does not exist.", context);
154 }
155 // get mvex atom which holds default values for fragmented files
157 m_fragmented = true;
158 if (mehdAtom->dataSize() > 0) {
159 stream().seekg(static_cast<iostream::off_type>(mehdAtom->dataOffset()));
160 unsigned int durationSize = reader().readByte() == 1u ? 8u : 4u; // duration size depends on atom version
161 if (mehdAtom->dataSize() >= 4 + durationSize) {
162 stream().seekg(3, ios_base::cur); // skip flags
163 switch (durationSize) {
164 case 4u:
165 m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt32BE()) / static_cast<double>(m_timeScale));
166 break;
167 case 8u:
168 m_duration = TimeSpan::fromSeconds(static_cast<double>(reader().readUInt64BE()) / static_cast<double>(m_timeScale));
169 break;
170 default:;
171 }
172 } else {
173 diag.emplace_back(DiagLevel::Warning, "mehd atom is truncated.", context);
174 }
175 }
176 }
177 // get first trak atoms which hold information for each track
178 Mp4Atom *trakAtom = moovAtom->childById(Mp4AtomIds::Track, diag);
179 int trackNum = 1;
180 while (trakAtom) {
181 try {
182 trakAtom->parse(diag);
183 } catch (const Failure &) {
184 diag.emplace_back(DiagLevel::Warning, "Unable to parse child atom of moov.", context);
185 }
186 // parse the trak atom using the Mp4Track class
187 m_tracks.emplace_back(make_unique<Mp4Track>(*trakAtom));
188 try { // try to parse header
189 m_tracks.back()->parseHeader(diag, progress);
190 } catch (const Failure &) {
191 diag.emplace_back(DiagLevel::Critical, argsToString("Unable to parse track ", trackNum, '.'), context);
192 }
193 trakAtom = trakAtom->siblingById(Mp4AtomIds::Track, diag); // get next trak atom
194 ++trackNum;
195 }
196 // get overall duration, creation time and modification time if not determined yet
197 if (m_duration.isNull() || m_modificationTime.isNull() || m_creationTime.isNull()) {
198 for (const auto &track : tracks()) {
199 if (track->duration() > m_duration) {
201 }
204 }
207 }
208 }
209 }
210 }
211 } catch (const Failure &) {
212 diag.emplace_back(DiagLevel::Warning, "Unable to parse moov atom.", context);
213 }
214}
215
217{
218 static const string context("making MP4 container");
219 progress.updateStep("Calculating atom sizes and padding ...");
220
221 // basic validation of original file
222 if (!isHeaderParsed()) {
223 diag.emplace_back(DiagLevel::Critical, "The header has not been parsed yet.", context);
224 throw InvalidDataException();
225 }
226
227 // define variables needed to parse atoms of original file
228 if (!firstElement()) {
229 diag.emplace_back(DiagLevel::Critical, "No MP4 atoms could be found.", context);
230 throw InvalidDataException();
231 }
232
233 // define variables needed to manage file layout
234 // -> whether media data is written chunk by chunk (need to write chunk by chunk if tracks have been altered)
235 const bool writeChunkByChunk = m_tracksAltered;
236 // -> whether rewrite is required (always required when forced to rewrite or when tracks have been altered)
237 bool rewriteRequired = fileInfo().isForcingRewrite() || writeChunkByChunk;
238 // -> use the preferred tag position/index position (force one wins, if both are force tag pos wins; might be changed later if none is forced)
239 ElementPosition initialNewTagPos
241 ElementPosition newTagPos = initialNewTagPos;
242 // -> current tag position (determined later)
243 ElementPosition currentTagPos;
244 // -> holds new padding (before actual data)
245 std::uint64_t newPadding;
246 // -> holds new padding (after actual data)
247 std::uint64_t newPaddingEnd;
248 // -> holds current offset
249 std::uint64_t currentOffset;
250 // -> holds track information, used when writing chunk-by-chunk
251 vector<tuple<istream *, vector<std::uint64_t>, vector<std::uint64_t>>> trackInfos;
252 // -> holds offsets of media data atoms in original file, used when simply copying mdat
253 vector<std::int64_t> origMediaDataOffsets;
254 // -> holds offsets of media data atoms in new file, used when simply copying mdat
255 vector<std::int64_t> newMediaDataOffsets;
256 // -> new size of movie atom and user data atom
257 std::uint64_t movieAtomSize, userDataAtomSize;
258 // -> track count of original file
259 const auto trackCount = this->trackCount();
260
261 // find relevant atoms in original file
262 Mp4Atom *fileTypeAtom, *progressiveDownloadInfoAtom, *movieAtom, *firstMediaDataAtom, *firstMovieFragmentAtom /*, *userDataAtom*/;
263 Mp4Atom *level0Atom, *level1Atom, *level2Atom, *lastAtomToBeWritten;
264 try {
265 // file type atom (mandatory)
266 if ((fileTypeAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::FileType, diag))) {
267 // buffer atom
268 fileTypeAtom->makeBuffer();
269 } else {
270 // throw error if missing
271 diag.emplace_back(DiagLevel::Critical, "Mandatory \"ftyp\"-atom not found.", context);
272 throw InvalidDataException();
273 }
274
275 // progressive download information atom (not mandatory)
276 if ((progressiveDownloadInfoAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::ProgressiveDownloadInformation, diag))) {
277 // buffer atom
278 progressiveDownloadInfoAtom->makeBuffer();
279 }
280
281 // movie atom (mandatory)
282 if (!(movieAtom = firstElement()->siblingByIdIncludingThis(Mp4AtomIds::Movie, diag))) {
283 // throw error if missing
284 diag.emplace_back(DiagLevel::Critical, "Mandatory \"moov\"-atom not in the source file found.", context);
285 throw InvalidDataException();
286 }
287
288 // movie fragment atom (indicates dash file)
289 if ((firstMovieFragmentAtom = firstElement()->siblingById(Mp4AtomIds::MovieFragment, diag))) {
290 // there is at least one movie fragment atom -> consider file being dash
291 // -> can not write chunk-by-chunk (currently)
292 if (writeChunkByChunk) {
293 diag.emplace_back(DiagLevel::Critical, "Writing chunk-by-chunk is not implemented for DASH files.", context);
295 }
296 // -> tags must be placed at the beginning
297 newTagPos = ElementPosition::BeforeData;
298 }
299
300 // media data atom (mandatory?)
301 // -> consider not only mdat as media data atom; consider everything not handled otherwise as media data
302 for (firstMediaDataAtom = nullptr, level0Atom = firstElement(); level0Atom; level0Atom = level0Atom->nextSibling()) {
303 level0Atom->parse(diag);
304 switch (level0Atom->id()) {
308 case Mp4AtomIds::Free:
309 case Mp4AtomIds::Skip:
310 continue;
311 default:
312 firstMediaDataAtom = level0Atom;
313 }
314 break;
315 }
316
317 // determine current tag position
318 // -> since tags are nested in the movie atom its position is relevant here
319 if (firstMediaDataAtom) {
320 currentTagPos = firstMediaDataAtom->startOffset() < movieAtom->startOffset() ? ElementPosition::AfterData : ElementPosition::BeforeData;
321 if (newTagPos == ElementPosition::Keep) {
322 newTagPos = currentTagPos;
323 }
324 } else {
325 currentTagPos = ElementPosition::Keep;
326 }
327
328 // ensure index and tags are always placed at the beginning when dealing with DASH files
329 if (firstMovieFragmentAtom) {
330 if (initialNewTagPos == ElementPosition::AfterData) {
331 diag.emplace_back(
332 DiagLevel::Warning, "Sorry, but putting index/tags at the end is not possible when dealing with DASH files.", context);
333 }
334 initialNewTagPos = newTagPos = ElementPosition::BeforeData;
335 }
336
337 // user data atom (currently not used)
338 //userDataAtom = movieAtom->childById(Mp4AtomIds::UserData);
339
340 } catch (const NotImplementedException &) {
341 throw;
342
343 } catch (const Failure &) {
344 // can't ignore parsing errors here
345 diag.emplace_back(DiagLevel::Critical, "Unable to parse the overall atom structure of the source file.", context);
346 throw InvalidDataException();
347 }
348
349 progress.stopIfAborted();
350
351 // calculate sizes
352 // -> size of tags
353 vector<Mp4TagMaker> tagMaker;
354 std::uint64_t tagsSize = 0;
355 tagMaker.reserve(m_tags.size());
356 for (auto &tag : m_tags) {
357 try {
358 tagMaker.emplace_back(tag->prepareMaking(diag));
359 tagsSize += tagMaker.back().requiredSize();
360 } catch (const Failure &) {
361 }
362 }
363
364 // -> size of movie atom (contains track and tag information)
365 movieAtomSize = userDataAtomSize = 0;
366 try {
367 // add size of children
368 for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
369 for (level1Atom = level0Atom->firstChild(); level1Atom; level1Atom = level1Atom->nextSibling()) {
370 level1Atom->parse(diag);
371 switch (level1Atom->id()) {
373 try {
374 for (level2Atom = level1Atom->firstChild(); level2Atom; level2Atom = level2Atom->nextSibling()) {
375 level2Atom->parse(diag);
376 switch (level2Atom->id()) {
377 case Mp4AtomIds::Meta:
378 // ignore meta data here; it is added separately
379 break;
380 default:
381 // add size of unknown children of the user data atom
382 userDataAtomSize += level2Atom->totalSize();
383 level2Atom->makeBuffer();
384 }
385 }
386 } catch (const Failure &) {
387 // invalid children might be ignored as not mandatory
388 diag.emplace_back(
389 DiagLevel::Critical, "Unable to parse the children of \"udta\"-atom of the source file; ignoring them.", context);
390 }
391 break;
393 // ignore track atoms here; they are added separately
394 break;
395 default:
396 // add size of unknown children of the movie atom
397 movieAtomSize += level1Atom->totalSize();
398 level1Atom->makeBuffer();
399 }
400 }
401 }
402
403 // add size of meta data
404 if (userDataAtomSize += tagsSize) {
405 Mp4Atom::addHeaderSize(userDataAtomSize);
406 movieAtomSize += userDataAtomSize;
407 }
408
409 // add size of track atoms
410 for (const auto &track : tracks()) {
411 movieAtomSize += track->requiredSize(diag);
412 }
413
414 // add header size
415 Mp4Atom::addHeaderSize(movieAtomSize);
416 } catch (const Failure &) {
417 // can't ignore parsing errors here
418 diag.emplace_back(DiagLevel::Critical, "Unable to parse the children of \"moov\"-atom of the source file.", context);
419 throw InvalidDataException();
420 }
421
422 progress.stopIfAborted();
423
424 // check whether there are atoms to be voided after movie next sibling (only relevant when not rewriting)
425 if (!rewriteRequired) {
426 newPaddingEnd = 0;
427 std::uint64_t currentSum = 0;
428 for (level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
429 level0Atom->parse(diag);
430 switch (level0Atom->id()) {
434 case Mp4AtomIds::Free:
435 case Mp4AtomIds::Skip:
436 // must void these if they occur "between" the media data
437 currentSum += level0Atom->totalSize();
438 break;
439 default:
440 newPaddingEnd += currentSum;
441 currentSum = 0;
442 lastAtomToBeWritten = level0Atom;
443 }
444 }
445 }
446
447 // calculate padding if no rewrite is required; otherwise use the preferred padding
448calculatePadding:
449 if (rewriteRequired) {
450 newPadding = (fileInfo().preferredPadding() && fileInfo().preferredPadding() < 8 ? 8 : fileInfo().preferredPadding());
451 } else {
452 // file type atom
453 currentOffset = fileTypeAtom->totalSize();
454
455 // progressive download information atom
456 if (progressiveDownloadInfoAtom) {
457 currentOffset += progressiveDownloadInfoAtom->totalSize();
458 }
459
460 // if writing tags before data: movie atom (contains tag)
461 switch (newTagPos) {
464 currentOffset += movieAtomSize;
465 break;
466 default:;
467 }
468
469 // check whether there is sufficiant space before the next atom
470 if (!(rewriteRequired = firstMediaDataAtom && currentOffset > firstMediaDataAtom->startOffset())) {
471 // there is sufficiant space
472 // -> check whether the padding matches specifications
473 // min padding: says "at least ... byte should be reserved to prepend further tag info", so the padding at the end
474 // shouldn't be tanken into account (it can't be used to prepend further tag info)
475 // max padding: says "do not waste more than ... byte", so here all padding should be taken into account
476 newPadding = firstMediaDataAtom->startOffset() - currentOffset;
477 rewriteRequired = (newPadding > 0 && newPadding < 8) || newPadding < fileInfo().minPadding()
478 || (newPadding + newPaddingEnd) > fileInfo().maxPadding();
479 }
480 if (rewriteRequired) {
481 // can't put the tags before media data
482 if (!firstMovieFragmentAtom && !fileInfo().forceTagPosition() && !fileInfo().forceIndexPosition()
483 && newTagPos != ElementPosition::AfterData) {
484 // writing tag before media data is not forced, its not a DASH file and tags aren't already at the end
485 // -> try to put the tags at the end
486 newTagPos = ElementPosition::AfterData;
487 rewriteRequired = false;
488 } else {
489 // writing tag before media data is forced -> rewrite the file
490 // when rewriting anyways, ensure the preferred tag position is used
491 newTagPos = initialNewTagPos == ElementPosition::Keep ? currentTagPos : initialNewTagPos;
492 }
493 // in any case: recalculate padding
494 goto calculatePadding;
495 } else {
496 // tags can be put before the media data
497 // -> ensure newTagPos is not ElementPosition::Keep
498 if (newTagPos == ElementPosition::Keep) {
499 newTagPos = ElementPosition::BeforeData;
500 }
501 }
502 }
503
504 // setup stream(s) for writing
505 // -> update status
506 progress.nextStepOrStop("Preparing streams ...");
507
508 // -> define variables needed to handle output stream and backup stream (required when rewriting the file)
509 string originalPath = fileInfo().path(), backupPath;
510 NativeFileStream &outputStream = fileInfo().stream();
511 NativeFileStream backupStream; // create a stream to open the backup/original file for the case rewriting the file is required
512 BinaryWriter outputWriter(&outputStream);
513
514 if (rewriteRequired) {
515 if (fileInfo().saveFilePath().empty()) {
516 // move current file to temp dir and reopen it as backupStream, recreate original file
517 try {
518 BackupHelper::createBackupFileCanonical(fileInfo().backupDirectory(), originalPath, backupPath, outputStream, backupStream);
519 // recreate original file, define buffer variables
520 outputStream.open(originalPath, ios_base::out | ios_base::binary | ios_base::trunc);
521 } catch (const std::ios_base::failure &failure) {
522 diag.emplace_back(
523 DiagLevel::Critical, argsToString("Creation of temporary file (to rewrite the original file) failed: ", failure.what()), context);
524 throw;
525 }
526 } else {
527 // open the current file as backupStream and create a new outputStream at the specified "save file path"
528 try {
529 backupStream.exceptions(ios_base::badbit | ios_base::failbit);
530 backupStream.open(BasicFileInfo::pathForOpen(fileInfo().path()).data(), ios_base::in | ios_base::binary);
531 fileInfo().close();
532 outputStream.open(BasicFileInfo::pathForOpen(fileInfo().saveFilePath()).data(), ios_base::out | ios_base::binary | ios_base::trunc);
533 } catch (const std::ios_base::failure &failure) {
534 diag.emplace_back(DiagLevel::Critical, argsToString("Opening streams to write output file failed: ", failure.what()), context);
535 throw;
536 }
537 }
538
539 // set backup stream as associated input stream since we need the original elements to write the new file
540 setStream(backupStream);
541
542 // TODO: reduce code duplication
543
544 } else { // !rewriteRequired
545 // ensure everything to make track atoms is buffered before altering the source file
546 for (const auto &track : tracks()) {
547 track->bufferTrackAtoms(diag);
548 }
549
550 // reopen original file to ensure it is opened for writing
551 try {
552 fileInfo().close();
553 outputStream.open(fileInfo().path(), ios_base::in | ios_base::out | ios_base::binary);
554 } catch (const std::ios_base::failure &failure) {
555 diag.emplace_back(DiagLevel::Critical, argsToString("Opening the file with write permissions failed: ", failure.what()), context);
556 throw;
557 }
558 }
559
560 // start actual writing
561 try {
562 // write header
563 progress.nextStepOrStop("Writing header and tags ...");
564 // -> make file type atom
565 fileTypeAtom->copyBuffer(outputStream);
566 fileTypeAtom->discardBuffer();
567 // -> make progressive download info atom
568 if (progressiveDownloadInfoAtom) {
569 progressiveDownloadInfoAtom->copyBuffer(outputStream);
570 progressiveDownloadInfoAtom->discardBuffer();
571 }
572
573 // set input/output streams of each track
574 for (auto &track : tracks()) {
575 // ensure the track reads from the original file
576 if (&track->inputStream() == &outputStream) {
577 track->setInputStream(backupStream);
578 }
579 // ensure the track writes to the output file
580 track->setOutputStream(outputStream);
581 }
582
583 // write movie atom / padding and media data
584 for (std::uint8_t pass = 0; pass != 2; ++pass) {
585 if (newTagPos == (pass ? ElementPosition::AfterData : ElementPosition::BeforeData)) {
586 // define function to write tracks
587 auto tracksWritten = false;
588 const auto writeTracks = [this, &diag, &tracksWritten] {
589 if (tracksWritten) {
590 return;
591 }
592 for (auto &track : tracks()) {
593 track->makeTrack(diag);
594 }
595 tracksWritten = true;
596 };
597
598 // define function to write user data
599 auto userDataWritten = false;
600 auto writeUserData = [level0Atom, level1Atom, level2Atom, movieAtom, &userDataWritten, userDataAtomSize, &outputStream, &outputWriter,
601 &tagMaker, &diag]() mutable {
602 if (userDataWritten || !userDataAtomSize) {
603 return;
604 }
605
606 // writer user data atom header
607 Mp4Atom::makeHeader(userDataAtomSize, Mp4AtomIds::UserData, outputWriter);
608
609 // write children of user data atom
610 bool metaAtomWritten = false;
611 for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
612 for (level1Atom = level0Atom->childById(Mp4AtomIds::UserData, diag); level1Atom;
613 level1Atom = level1Atom->siblingById(Mp4AtomIds::UserData, diag)) {
614 for (level2Atom = level1Atom->firstChild(); level2Atom; level2Atom = level2Atom->nextSibling()) {
615 switch (level2Atom->id()) {
616 case Mp4AtomIds::Meta:
617 // write meta atom
618 for (auto &maker : tagMaker) {
619 maker.make(outputStream, diag);
620 }
621 metaAtomWritten = true;
622 break;
623 default:
624 // write buffered data
625 level2Atom->copyBuffer(outputStream);
626 level2Atom->discardBuffer();
627 }
628 }
629 }
630 }
631
632 // write meta atom if not already written
633 if (!metaAtomWritten) {
634 for (auto &maker : tagMaker) {
635 maker.make(outputStream, diag);
636 }
637 }
638
639 userDataWritten = true;
640 };
641
642 // write movie atom
643 // -> write movie atom header
644 Mp4Atom::makeHeader(movieAtomSize, Mp4AtomIds::Movie, outputWriter);
645
646 // -> write children of movie atom preserving the original order
647 for (level0Atom = movieAtom; level0Atom; level0Atom = level0Atom->siblingById(Mp4AtomIds::Movie, diag)) {
648 for (level1Atom = level0Atom->firstChild(); level1Atom; level1Atom = level1Atom->nextSibling()) {
649 switch (level1Atom->id()) {
651 writeTracks();
652 break;
654 writeUserData();
655 break;
656 default:
657 // write buffered data
658 level1Atom->copyBuffer(outputStream);
659 level1Atom->discardBuffer();
660 }
661 }
662 }
663
664 // -> write tracks and user data atoms if not already happened within the loop
665 writeTracks();
666 writeUserData();
667
668 } else {
669 // write padding
670 if (newPadding) {
671 // write free atom header
672 if (newPadding < numeric_limits<std::uint32_t>::max()) {
673 outputWriter.writeUInt32BE(static_cast<std::uint32_t>(newPadding));
674 outputWriter.writeUInt32BE(Mp4AtomIds::Free);
675 newPadding -= 8;
676 } else {
677 outputWriter.writeUInt32BE(1);
678 outputWriter.writeUInt32BE(Mp4AtomIds::Free);
679 outputWriter.writeUInt64BE(newPadding);
680 newPadding -= 16;
681 }
682
683 // write zeroes
684 for (; newPadding; --newPadding) {
685 outputStream.put(0);
686 }
687 }
688
689 // write media data
690 if (rewriteRequired) {
691 for (level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
692 level0Atom->parse(diag);
693 switch (level0Atom->id()) {
697 case Mp4AtomIds::Free:
698 case Mp4AtomIds::Skip:
699 break;
701 if (writeChunkByChunk) {
702 // write actual data separately when writing chunk-by-chunk
703 break;
704 } else {
705 // store media data offsets when not writing chunk-by-chunk to be able to update chunk offset table
706 origMediaDataOffsets.push_back(static_cast<std::int64_t>(level0Atom->startOffset()));
707 newMediaDataOffsets.push_back(outputStream.tellp());
708 }
709 [[fallthrough]];
710 default:
711 // update status
712 progress.updateStep("Writing atom: " + level0Atom->idToString());
713 // copy atom entirely and forward status update calls
714 level0Atom->copyEntirely(outputStream, diag, &progress);
715 }
716 }
717
718 // when writing chunk-by-chunk write media data now
719 if (writeChunkByChunk) {
720 // read chunk offset and chunk size table from the old file which are required to get chunks
721 progress.updateStep("Reading chunk offsets and sizes from the original file ...");
722 trackInfos.reserve(trackCount);
723 std::uint64_t totalChunkCount = 0;
724 std::uint64_t totalMediaDataSize = 0;
725 for (auto &track : tracks()) {
726 progress.stopIfAborted();
727
728 // emplace information
729 trackInfos.emplace_back(
730 &track->inputStream(), track->readChunkOffsets(fileInfo().isForcingFullParse(), diag), track->readChunkSizes(diag));
731
732 // check whether the chunks could be parsed correctly
733 const vector<std::uint64_t> &chunkOffsetTable = get<1>(trackInfos.back());
734 const vector<std::uint64_t> &chunkSizesTable = get<2>(trackInfos.back());
735 if (track->chunkCount() != chunkOffsetTable.size() || track->chunkCount() != chunkSizesTable.size()) {
736 diag.emplace_back(DiagLevel::Critical,
737 "Chunks of track " % numberToString<std::uint64_t, string>(track->id()) + " could not be parsed correctly.",
738 context);
739 }
740
741 // increase total chunk count and size
742 totalChunkCount += track->chunkCount();
743 totalMediaDataSize += accumulate(chunkSizesTable.cbegin(), chunkSizesTable.cend(), 0ul);
744 }
745
746 // write media data chunk-by-chunk
747 // -> write header of media data atom
748 Mp4Atom::addHeaderSize(totalMediaDataSize);
749 Mp4Atom::makeHeader(totalMediaDataSize, Mp4AtomIds::MediaData, outputWriter);
750
751 // -> copy chunks
752 CopyHelper<0x2000> copyHelper;
753 std::uint64_t chunkIndexWithinTrack = 0, totalChunksCopied = 0;
754 bool anyChunksCopied;
755 do {
756 progress.stopIfAborted();
757
758 // copy a chunk from each track
759 anyChunksCopied = false;
760 for (size_t trackIndex = 0; trackIndex < trackCount; ++trackIndex) {
761 // get source stream and tables for current track
762 auto &trackInfo = trackInfos[trackIndex];
763 istream &sourceStream = *get<0>(trackInfo);
764 vector<std::uint64_t> &chunkOffsetTable = get<1>(trackInfo);
765 const vector<std::uint64_t> &chunkSizesTable = get<2>(trackInfo);
766
767 // still chunks to be copied (of this track)?
768 if (chunkIndexWithinTrack < chunkOffsetTable.size() && chunkIndexWithinTrack < chunkSizesTable.size()) {
769 // copy chunk, update entry in chunk offset table
770 sourceStream.seekg(static_cast<streamoff>(chunkOffsetTable[chunkIndexWithinTrack]));
771 chunkOffsetTable[chunkIndexWithinTrack] = static_cast<std::uint64_t>(outputStream.tellp());
772 copyHelper.copy(sourceStream, outputStream, chunkSizesTable[chunkIndexWithinTrack]);
773
774 // update counter / status
775 anyChunksCopied = true;
776 ++totalChunksCopied;
777 }
778 }
779
780 // incrase chunk index within track, update progress percentage
781 if (!(++chunkIndexWithinTrack % 10)) {
782 progress.updateStepPercentage(static_cast<std::uint8_t>(totalChunksCopied * 100 / totalChunkCount));
783 }
784
785 } while (anyChunksCopied);
786 }
787
788 } else {
789 // can't just skip next movie sibling
790 for (level0Atom = firstMediaDataAtom; level0Atom; level0Atom = level0Atom->nextSibling()) {
791 level0Atom->parse(diag);
792 switch (level0Atom->id()) {
796 // must void these if they occur "between" the media data
797 outputStream.seekp(4, ios_base::cur);
798 outputWriter.writeUInt32BE(Mp4AtomIds::Free);
799 break;
800 default:
801 outputStream.seekp(static_cast<iostream::off_type>(level0Atom->totalSize()), ios_base::cur);
802 }
803 if (level0Atom == lastAtomToBeWritten) {
804 break;
805 }
806 }
807 }
808 }
809 }
810
811 // reparse what is written so far
812 progress.updateStep("Reparsing output file ...");
813 if (rewriteRequired) {
814 // report new size
815 fileInfo().reportSizeChanged(static_cast<std::uint64_t>(outputStream.tellp()));
816 // "save as path" is now the regular path
817 if (!fileInfo().saveFilePath().empty()) {
818 fileInfo().reportPathChanged(fileInfo().saveFilePath());
819 fileInfo().setSaveFilePath(string());
820 }
821 // the outputStream needs to be reopened to be able to read again
822 outputStream.close();
823 outputStream.open(BasicFileInfo::pathForOpen(fileInfo().path()).data(), ios_base::in | ios_base::out | ios_base::binary);
824 setStream(outputStream);
825 } else {
826 const auto newSize = static_cast<std::uint64_t>(outputStream.tellp());
827 if (newSize < fileInfo().size()) {
828 // file is smaller after the modification -> truncate
829 // -> close stream before truncating
830 outputStream.close();
831 // -> truncate file
832 if (truncate(BasicFileInfo::pathForOpen(fileInfo().path()).data(), static_cast<iostream::off_type>(newSize)) == 0) {
833 fileInfo().reportSizeChanged(newSize);
834 } else {
835 diag.emplace_back(DiagLevel::Critical, "Unable to truncate the file.", context);
836 }
837 // -> reopen the stream again
838 outputStream.open(BasicFileInfo::pathForOpen(fileInfo().path()).data(), ios_base::in | ios_base::out | ios_base::binary);
839 } else {
840 // file is longer after the modification -> just report new size
841 fileInfo().reportSizeChanged(newSize);
842 }
843 }
844
845 reset();
846 try {
847 parseTracks(diag, progress);
848 } catch (const OperationAbortedException &) {
849 throw;
850 } catch (const Failure &) {
851 diag.emplace_back(DiagLevel::Critical, "Unable to reparse the new file.", context);
852 throw;
853 }
854
855 if (rewriteRequired) {
856 // check whether the track count of the new file equals the track count of old file
857 if (trackCount != tracks().size()) {
858 diag.emplace_back(DiagLevel::Critical,
859 argsToString("Unable to update chunk offsets (\"stco\"/\"co64\"-atom): Number of tracks in the output file (", tracks().size(),
860 ") differs from the number of tracks in the original file (", trackCount, ")."),
861 context);
862 throw Failure();
863 }
864
865 // update chunk offset table
866 if (writeChunkByChunk) {
867 progress.updateStep("Updating chunk offset table for each track ...");
868 for (size_t trackIndex = 0; trackIndex != trackCount; ++trackIndex) {
869 const auto &track = tracks()[trackIndex];
870 const auto &chunkOffsetTable = get<1>(trackInfos[trackIndex]);
871 if (track->chunkCount() == chunkOffsetTable.size()) {
872 track->updateChunkOffsets(chunkOffsetTable);
873 } else {
874 diag.emplace_back(DiagLevel::Critical,
875 argsToString("Unable to update chunk offsets of track ", (trackIndex + 1),
876 ": Number of chunks in the output file differs from the number of chunks in the original file."),
877 context);
878 throw Failure();
879 }
880 }
881 } else {
882 progress.updateStep("Updating chunk offset table for each track ...");
883 updateOffsets(origMediaDataOffsets, newMediaDataOffsets, diag, progress);
884 }
885 }
886
887 // prevent deferring final write operations (to catch and handle possible errors here)
888 outputStream.flush();
889
890 // handle errors (which might have been occurred after renaming/creating backup file)
891 } catch (...) {
892 BackupHelper::handleFailureAfterFileModifiedCanonical(fileInfo(), originalPath, backupPath, outputStream, backupStream, diag, context);
893 }
894}
895
908void Mp4Container::updateOffsets(const std::vector<std::int64_t> &oldMdatOffsets, const std::vector<std::int64_t> &newMdatOffsets, Diagnostics &diag,
910{
911 // do NOT invalidate the status here since this method is internally called by internalMakeFile(), just update the status
912 const string context("updating MP4 container chunk offset table");
913 if (!firstElement()) {
914 diag.emplace_back(DiagLevel::Critical, "No MP4 atoms could be found.", context);
915 throw InvalidDataException();
916 }
917 // update "base-data-offset-present" of "tfhd"-atom (NOT tested properly)
918 try {
919 for (Mp4Atom *moofAtom = firstElement()->siblingById(Mp4AtomIds::MovieFragment, diag); moofAtom;
920 moofAtom = moofAtom->siblingById(Mp4AtomIds::MovieFragment, diag)) {
921 moofAtom->parse(diag);
922 try {
923 for (Mp4Atom *trafAtom = moofAtom->childById(Mp4AtomIds::TrackFragment, diag); trafAtom;
924 trafAtom = trafAtom->siblingById(Mp4AtomIds::TrackFragment, diag)) {
925 trafAtom->parse(diag);
926 int tfhdAtomCount = 0;
927 for (Mp4Atom *tfhdAtom = trafAtom->childById(Mp4AtomIds::TrackFragmentHeader, diag); tfhdAtom;
928 tfhdAtom = tfhdAtom->siblingById(Mp4AtomIds::TrackFragmentHeader, diag)) {
929 tfhdAtom->parse(diag);
930 ++tfhdAtomCount;
931 if (tfhdAtom->dataSize() < 8) {
932 diag.emplace_back(DiagLevel::Warning, "tfhd atom is truncated.", context);
933 continue;
934 }
935 stream().seekg(static_cast<iostream::off_type>(tfhdAtom->dataOffset()) + 1);
936 std::uint32_t flags = reader().readUInt24BE();
937 if (!(flags & 1)) {
938 continue;
939 }
940 if (tfhdAtom->dataSize() < 16) {
941 diag.emplace_back(DiagLevel::Warning, "tfhd atom (denoting base-data-offset-present) is truncated.", context);
942 continue;
943 }
944 stream().seekg(4, ios_base::cur); // skip track ID
945 std::uint64_t off = reader().readUInt64BE();
946 for (auto iOld = oldMdatOffsets.cbegin(), iNew = newMdatOffsets.cbegin(), end = oldMdatOffsets.cend(); iOld != end;
947 ++iOld, ++iNew) {
948 if (off < static_cast<std::uint64_t>(*iOld)) {
949 continue;
950 }
951 off += static_cast<std::uint64_t>(*iNew - *iOld);
952 stream().seekp(static_cast<iostream::off_type>(tfhdAtom->dataOffset()) + 8);
953 writer().writeUInt64BE(off);
954 break;
955 }
956 }
957 switch (tfhdAtomCount) {
958 case 0:
959 diag.emplace_back(DiagLevel::Warning, "traf atom doesn't contain mandatory tfhd atom.", context);
960 break;
961 case 1:
962 break;
963 default:
964 diag.emplace_back(
965 DiagLevel::Warning, "traf atom stores multiple tfhd atoms but it should only contain exactly one tfhd atom.", context);
966 }
967 }
968 } catch (const Failure &) {
969 diag.emplace_back(DiagLevel::Critical, "Unable to parse children of top-level atom moof.", context);
970 }
971 }
972 } catch (const Failure &) {
973 diag.emplace_back(DiagLevel::Critical, "Unable to parse top-level atom moof.", context);
974 }
975 // update each track
976 for (auto &track : tracks()) {
977 if (!track->isHeaderValid()) {
978 try {
979 track->parseHeader(diag, progress);
980 } catch (const Failure &) {
981 diag.emplace_back(DiagLevel::Warning,
982 "The chunk offsets of track " % track->name() + " couldn't be updated because the track seems to be invalid..", context);
983 throw;
984 }
985 }
986 if (track->isHeaderValid()) {
987 try {
988 track->updateChunkOffsets(oldMdatOffsets, newMdatOffsets);
989 } catch (const Failure &) {
990 diag.emplace_back(DiagLevel::Warning, "The chunk offsets of track " % track->name() + " couldn't be updated.", context);
991 throw;
992 }
993 }
994 }
995}
996
997} // namespace TagParser
The AbortableProgressFeedback class provides feedback about an ongoing operation via callbacks.
void stopIfAborted() const
Throws an OperationAbortedException if aborted.
void nextStepOrStop(const std::string &step, std::uint8_t stepPercentage=0)
Throws an OperationAbortedException if aborted; otherwise the data for the next step is set.
CppUtilities::DateTime m_modificationTime
std::iostream & stream()
Returns the related stream.
std::uint64_t startOffset() const
Returns the start offset in the related stream.
void parseTracks(Diagnostics &diag, AbortableProgressFeedback &progress)
Parses the tracks of the file if not parsed yet.
std::uint64_t version() const
Returns the version if known; otherwise returns 0.
bool isHeaderParsed() const
Returns an indication whether the header has been parsed yet.
void setStream(std::iostream &stream)
Sets the related stream.
CppUtilities::BinaryWriter & writer()
Returns the related BinaryWriter.
CppUtilities::BinaryReader & reader()
Returns the related BinaryReader.
CppUtilities::DateTime m_creationTime
CppUtilities::TimeSpan m_duration
std::uint64_t id() const
Returns the track ID if known; otherwise returns 0.
const CppUtilities::DateTime & modificationTime() const
Returns the time of the last modification if known; otherwise returns a DateTime of zero ticks.
const CppUtilities::DateTime & creationTime() const
Returns the creation time if known; otherwise returns a DateTime of zero ticks.
std::istream & inputStream()
Returns the associated input stream.
void parseHeader(Diagnostics &diag, AbortableProgressFeedback &progress)
Parses technical information about the track from the header.
bool isHeaderValid() const
Returns an indication whether the track header is valid.
void setOutputStream(std::ostream &stream)
Assigns another output stream.
void setInputStream(std::istream &stream)
Assigns another input stream.
const CppUtilities::TimeSpan & duration() const
Returns the duration if known; otherwise returns a TimeSpan of zero ticks.
const std::string name() const
Returns the track name if known; otherwise returns an empty string.
void reportPathChanged(std::string_view newPath)
Call this function to report that the path changed.
const std::string & path() const
Returns the path of the current file.
std::uint64_t size() const
Returns size of the current file in bytes.
CppUtilities::NativeFileStream & stream()
Returns the std::fstream for the current instance.
Definition: basicfileinfo.h:85
void close()
A possibly opened std::fstream will be closed.
static std::string_view pathForOpen(std::string_view url)
Returns removes the "file:/" prefix from url to be able to pass it to functions like open(),...
void reportSizeChanged(std::uint64_t newSize)
Call this function to report that the size changed.
void updateStep(const std::string &step, std::uint8_t stepPercentage=0)
Updates the current step and invokes the first callback specified on construction.
void updateStepPercentage(std::uint8_t stepPercentage)
Updates the current step percentage and invokes the second callback specified on construction (or the...
The Diagnostics class is a container for DiagMessage.
Definition: diagnostics.h:156
The class inherits from std::exception and serves as base class for exceptions thrown by the elements...
Definition: exceptions.h:11
The GenericContainer class helps parsing header, track, tag and chapter information of a file.
const std::vector< std::unique_ptr< Mp4Track > > & tracks() const
Returns the tracks of the file.
MediaFileInfo & fileInfo() const
Returns the related file info.
Mp4Atom * firstElement() const
Returns the first element of the file if available; otherwiese returns nullptr.
void reset() override
Discards all parsing results.
std::uint64_t startOffset() const
Returns the start offset in the related stream.
void discardBuffer()
Discards buffered data.
void copyEntirely(std::ostream &targetStream, Diagnostics &diag, AbortableProgressFeedback *progress)
Writes the entire element including all children to the specified targetStream.
const IdentifierType & id() const
Returns the element ID.
void copyBuffer(std::ostream &targetStream)
Copies buffered data to targetStream.
ImplementationType * childById(const IdentifierType &id, Diagnostics &diag)
Returns the first child with the specified id.
ImplementationType * nextSibling()
Returns the next sibling of the element.
ImplementationType * firstChild()
Returns the first child of the element.
ImplementationType * subelementByPath(Diagnostics &diag, IdentifierType item)
Returns the sub element for the specified path.
std::uint64_t totalSize() const
Returns the total size of the element.
void parse(Diagnostics &diag)
Parses the header information of the element which is read from the related stream at the start offse...
void makeBuffer()
Buffers the element (header and data).
ImplementationType * siblingById(const IdentifierType &id, Diagnostics &diag)
Returns the first sibling with the specified id.
The exception that is thrown when the data to be parsed or to be made seems invalid and therefore can...
Definition: exceptions.h:25
The MediaFileInfo class allows to read and write tag information providing a container/tag format ind...
Definition: mediafileinfo.h:75
bool isForcingRewrite() const
Returns whether forcing rewriting (when applying changes) is enabled.
void setSaveFilePath(std::string_view saveFilePath)
Sets the "save file path".
bool forceIndexPosition() const
Returns whether indexPosition() is forced.
std::size_t maxPadding() const
Returns the maximum padding to be written before the data blocks when applying changes.
std::size_t preferredPadding() const
Returns the padding to be written before the data block when applying changes and the file needs to b...
std::size_t minPadding() const
Returns the minimum padding to be written before the data blocks when applying changes.
ElementPosition tagPosition() const
Returns the position (in the output file) where the tag information is written when applying changes.
bool forceTagPosition() const
Returns whether tagPosition() is forced.
ElementPosition indexPosition() const
Returns the position (in the output file) where the index is written when applying changes.
The Mp4Atom class helps to parse MP4 files.
Definition: mp4atom.h:38
static constexpr void addHeaderSize(std::uint64_t &dataSize)
Adds the header size to the specified data size.
Definition: mp4atom.h:81
std::string idToString() const
Converts the specified atom ID to a printable string.
Definition: mp4atom.h:67
static void makeHeader(std::uint64_t size, std::uint32_t id, CppUtilities::BinaryWriter &writer)
Writes an MP4 atom header to the specified stream.
Definition: mp4atom.cpp:171
ElementPosition determineIndexPosition(Diagnostics &diag) const override
Determines the position of the index.
ElementPosition determineTagPosition(Diagnostics &diag) const override
Determines the position of the tags inside the file.
void reset() override
Discards all parsing results.
Mp4Container(MediaFileInfo &fileInfo, std::uint64_t startOffset)
Constructs a new container for the specified fileInfo at the specified startOffset.
void internalParseHeader(Diagnostics &diag, AbortableProgressFeedback &progress) override
Internally called to parse the header.
void internalParseTags(Diagnostics &diag, AbortableProgressFeedback &progress) override
Internally called to parse the tags.
void internalParseTracks(Diagnostics &diag, AbortableProgressFeedback &progress) override
Internally called to parse the tracks.
void internalMakeFile(Diagnostics &diag, AbortableProgressFeedback &progress) override
Internally called to make the file.
Implementation of TagParser::Tag for the MP4 container.
Definition: mp4tag.h:97
Mp4TagMaker prepareMaking(Diagnostics &diag)
Prepares making.
Definition: mp4tag.cpp:394
Implementation of TagParser::AbstractTrack for the MP4 container.
Definition: mp4track.h:118
std::uint32_t chunkCount() const
Returns the number of chunks denoted by the stco atom.
Definition: mp4track.h:229
std::vector< std::uint64_t > readChunkSizes(TagParser::Diagnostics &diag)
Reads the chunk sizes from the stsz (sample sizes) and stsc (samples per chunk) atom.
Definition: mp4track.cpp:507
void updateChunkOffsets(const std::vector< std::int64_t > &oldMdatOffsets, const std::vector< std::int64_t > &newMdatOffsets)
Updates the chunk offsets of the track.
Definition: mp4track.cpp:890
std::uint64_t requiredSize(Diagnostics &diag) const
Returns the number of bytes written when calling makeTrack().
Definition: mp4track.cpp:1086
std::vector< std::uint64_t > readChunkOffsets(bool parseFragments, Diagnostics &diag)
Reads the chunk offsets from the stco atom and fragments if parseFragments is true.
Definition: mp4track.cpp:173
void bufferTrackAtoms(Diagnostics &diag)
Buffers all atoms required by the makeTrack() method.
Definition: mp4track.cpp:1063
void makeTrack(Diagnostics &diag)
Makes the track entry ("trak"-atom) for the track.
Definition: mp4track.cpp:1139
The exception that is thrown when the data to be parsed holds no parsable information (e....
Definition: exceptions.h:18
This exception is thrown when the an operation is invoked that has not been implemented yet.
Definition: exceptions.h:60
The exception that is thrown when an operation has been stopped and thus not successfully completed b...
Definition: exceptions.h:46
TAG_PARSER_EXPORT void handleFailureAfterFileModifiedCanonical(MediaFileInfo &fileInfo, const std::string &originalPath, const std::string &backupPath, CppUtilities::NativeFileStream &outputStream, CppUtilities::NativeFileStream &backupStream, Diagnostics &diag, const std::string &context="making file")
Handles a failure/abort which occurred after the file has been modified.
TAG_PARSER_EXPORT void createBackupFileCanonical(const std::string &backupDir, std::string &originalPath, std::string &backupPath, CppUtilities::NativeFileStream &originalStream, CppUtilities::NativeFileStream &backupStream)
Creates a backup file like createBackupFile() but canonicalizes originalPath before doing the backup.
@ ProgressiveDownloadInformation
Definition: mp4ids.h:52
Contains all classes and functions of the TagInfo library.
Definition: aaccodebook.h:10
ElementPosition
Definition: settings.h:13