1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
11 #include "libavcodec/avcodec.h"
14 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
16 #define INT64_MIN (-9223372036854775807LL-1)
18 #define AC3_SAMPLES_PER_FRAME 1536
24 int64_t next_start; /* start time of next output frame */
25 int64_t next_pts; /* start time of next input frame */
26 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
27 int drop_count; /* count of 'time went backwards' drops */
39 struct hb_work_private_s
42 int busy; // bitmask with one bit for each active input
43 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
44 // appropriate bit is cleared when input gets
45 // an eof buf. syncWork returns done when all
48 hb_subtitle_t * subtitle;
50 int64_t next_start; /* start time of next output frame */
51 int64_t next_pts; /* start time of next input frame */
52 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
53 int drop_count; /* count of 'time went backwards' drops */
54 int drops; /* frames dropped to make a cbr video stream */
55 int dups; /* frames duplicated to make a cbr video stream */
59 int chap_mark; /* to propagate chapter mark across a drop */
60 hb_buffer_t * cur; /* The next picture to process */
63 hb_sync_audio_t sync_audio[8];
66 uint64_t st_counts[4];
71 /***********************************************************************
73 **********************************************************************/
74 static void InitAudio( hb_work_object_t * w, int i );
75 static void SyncVideo( hb_work_object_t * w );
76 static void SyncAudio( hb_work_object_t * w, int i );
77 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
78 static void UpdateState( hb_work_object_t * w );
80 /***********************************************************************
82 ***********************************************************************
83 * Initialize the work object
84 **********************************************************************/
85 int syncInit( hb_work_object_t * w, hb_job_t * job )
87 hb_title_t * title = job->title;
88 hb_chapter_t * chapter;
91 hb_work_private_t * pv;
93 pv = calloc( 1, sizeof( hb_work_private_t ) );
97 pv->pts_offset = INT64_MIN;
99 /* Calculate how many video frames we are expecting */
100 if (job->pts_to_stop)
102 duration = job->pts_to_stop + 90000;
107 for( i = job->chapter_start; i <= job->chapter_end; i++ )
109 chapter = hb_list_item( title->list_chapter, i - 1 );
110 duration += chapter->duration;
113 /* 1 second safety so we're sure we won't miss anything */
115 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
117 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
120 /* Initialize libsamplerate for every audio track we have */
121 if ( ! job->indepth_scan )
123 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
125 pv->busy |= ( 1 << (i + 1) );
130 /* Get subtitle info, if any */
131 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
136 /***********************************************************************
138 ***********************************************************************
140 **********************************************************************/
141 void syncClose( hb_work_object_t * w )
143 hb_work_private_t * pv = w->private_data;
144 hb_job_t * job = pv->job;
145 hb_title_t * title = job->title;
146 hb_audio_t * audio = NULL;
151 hb_buffer_close( &pv->cur );
154 hb_log( "sync: got %d frames, %d expected",
155 pv->count_frames, pv->count_frames_max );
157 if (pv->drops || pv->dups )
159 hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
162 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
164 audio = hb_list_item( title->list_audio, i );
165 if( audio->config.out.codec == HB_ACODEC_AC3 )
167 free( pv->sync_audio[i].ac3_buf );
171 src_delete( pv->sync_audio[i].state );
176 w->private_data = NULL;
179 /***********************************************************************
181 ***********************************************************************
182 * The root routine of this work abject
184 * The way this works is that we are syncing the audio to the PTS of
185 * the last video that we processed. That's why we skip the audio sync
186 * if we haven't got a valid PTS from the video yet.
188 **********************************************************************/
189 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
190 hb_buffer_t ** unused2 )
192 hb_work_private_t * pv = w->private_data;
198 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
200 if ( pv->busy & ( 1 << (i + 1) ) )
204 return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
207 hb_work_object_t hb_sync =
216 static void InitAudio( hb_work_object_t * w, int i )
218 hb_work_private_t * pv = w->private_data;
219 hb_job_t * job = pv->job;
220 hb_title_t * title = job->title;
221 hb_sync_audio_t * sync;
223 sync = &pv->sync_audio[i];
224 sync->audio = hb_list_item( title->list_audio, i );
226 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
228 /* Have a silent AC-3 frame ready in case we have to fill a
234 codec = avcodec_find_encoder( CODEC_ID_AC3 );
235 c = avcodec_alloc_context();
237 c->bit_rate = sync->audio->config.in.bitrate;
238 c->sample_rate = sync->audio->config.in.samplerate;
239 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
241 if( avcodec_open( c, codec ) < 0 )
243 hb_log( "sync: avcodec_open failed" );
247 zeros = calloc( AC3_SAMPLES_PER_FRAME *
248 sizeof( short ) * c->channels, 1 );
249 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
250 sync->audio->config.in.samplerate / 8;
251 sync->ac3_buf = malloc( sync->ac3_size );
253 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
254 zeros ) != sync->ac3_size )
256 hb_log( "sync: avcodec_encode_audio failed" );
265 /* Initialize libsamplerate */
267 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
268 sync->data.end_of_input = 0;
272 /***********************************************************************
274 ***********************************************************************
276 **********************************************************************/
277 static void SyncVideo( hb_work_object_t * w )
279 hb_work_private_t * pv = w->private_data;
280 hb_buffer_t * cur, * next, * sub = NULL;
281 hb_job_t * job = pv->job;
283 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
285 /* We haven't even got a frame yet */
291 /* we got an end-of-stream. Feed it downstream & signal that we're done. */
292 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
297 /* At this point we have a frame to process. Let's check
298 1) if we will be able to push into the fifo ahead
299 2) if the next frame is there already, since we need it to
300 compute the duration of the current frame*/
301 while( !hb_fifo_is_full( job->fifo_sync ) &&
302 ( next = hb_fifo_see( job->fifo_raw ) ) )
304 hb_buffer_t * buf_tmp;
306 if( next->size == 0 )
308 /* we got an end-of-stream. Feed it downstream & signal that
309 * we're done. Note that this means we drop the final frame of
310 * video (we don't know its duration). On DVDs the final frame
311 * is often strange and dropping it seems to be a good idea. */
312 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
316 if( pv->pts_offset == INT64_MIN )
318 /* This is our first frame */
320 if ( cur->start != 0 )
323 * The first pts from a dvd should always be zero but
324 * can be non-zero with a transport or program stream since
325 * we're not guaranteed to start on an IDR frame. If we get
326 * a non-zero initial PTS extend its duration so it behaves
327 * as if it started at zero so that our audio timing will
330 hb_log( "sync: first pts is %lld", cur->start );
335 if( cur->new_chap ) {
336 hb_log("sync got new chapter %d", cur->new_chap );
340 * since the first frame is always 0 and the upstream reader code
341 * is taking care of adjusting for pts discontinuities, we just have
342 * to deal with the next frame's start being in the past. This can
343 * happen when the PTS is adjusted after data loss but video frame
344 * reordering causes some frames with the old clock to appear after
345 * the clock change. This creates frames that overlap in time which
346 * looks to us like time going backward. The downstream muxing code
347 * can deal with overlaps of up to a frame time but anything larger
348 * we handle by dropping frames here.
350 if ( (int64_t)( next->start - cur->start ) <= 0 )
352 if ( pv->first_drop == 0 )
354 pv->first_drop = next->start;
357 buf_tmp = hb_fifo_get( job->fifo_raw );
358 if ( buf_tmp->new_chap )
360 // don't drop a chapter mark when we drop the buffer
361 pv->chap_mark = buf_tmp->new_chap;
363 hb_buffer_close( &buf_tmp );
366 if ( pv->first_drop )
368 hb_log( "sync: video time didn't advance - dropped %d frames "
369 "(delta %d ms, current %lld, next %lld, dur %d)",
370 pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
371 cur->start, next->start, (int)( next->start - cur->start ) );
377 * Track the video sequence number localy so that we can sync the audio
378 * to it using the sequence number as well as the PTS.
380 pv->video_sequence = cur->sequence;
382 /* Look for a subtitle for this frame */
386 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
388 /* If two subtitles overlap, make the first one stop
389 when the second one starts */
390 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
391 if( sub2 && sub->stop > sub2->start )
392 sub->stop = sub2->start;
394 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
395 // sub, cur->sequence, sub->sequence);
397 if( sub->sequence > cur->sequence )
400 * The video is behind where we are, so wait until
401 * it catches up to the same reader point on the
402 * DVD. Then our PTS should be in the same region
409 if( sub->stop > cur->start ) {
411 * The stop time is in the future, so fall through
412 * and we'll deal with it in the next block of
419 * The subtitle is older than this picture, trash it
421 sub = hb_fifo_get( pv->subtitle->fifo_raw );
422 hb_buffer_close( &sub );
426 * There is a valid subtitle, is it time to display it?
430 if( sub->stop > sub->start)
433 * Normal subtitle which ends after it starts, check to
434 * see that the current video is between the start and end.
436 if( cur->start > sub->start &&
437 cur->start < sub->stop )
440 * We should be playing this, so leave the
443 * fall through to display
445 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
448 * Subtitle is on for less than three seconds, extend
449 * the time that it is displayed to make it easier
450 * to read. Make it 3 seconds or until the next
451 * subtitle is displayed.
453 * This is in response to Indochine which only
454 * displays subs for 1 second - too fast to read.
456 sub->stop = sub->start + ( 3 * 90000 );
458 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
460 if( sub2 && sub->stop > sub2->start )
462 sub->stop = sub2->start;
469 * Defer until the play point is within the subtitle
477 * The end of the subtitle is less than the start, this is a
478 * sign of a PTS discontinuity.
480 if( sub->start > cur->start )
483 * we haven't reached the start time yet, or
484 * we have jumped backwards after having
485 * already started this subtitle.
487 if( cur->start < sub->stop )
490 * We have jumped backwards and so should
491 * continue displaying this subtitle.
493 * fall through to display.
499 * Defer until the play point is within the subtitle
505 * Play this subtitle as the start is greater than our
508 * fall through to display/
516 if ( job->mux & HB_MUX_AVI || job->cfr )
519 * The concept of variable frame rate video was a bit too advanced
520 * for Microsoft so AVI doesn't support it. Since almost all dvd
521 * video is VFR we have to convert it to constant frame rate to
522 * put it in an AVI container. So here we duplicate, drop and
523 * otherwise trash video frames to appease the gods of Redmond.
526 /* mpeg durations are exact when expressed in ticks of the
527 * 27MHz System clock but not in HB's 90KHz PTS clock. To avoid
528 * a truncation bias that will eventually cause the audio to desync
529 * we compute the duration of the next frame using 27MHz ticks
530 * then truncate it to 90KHz. */
531 duration = ( (int64_t)(pv->count_frames + 1 ) * job->vrate_base ) / 300 -
534 /* We don't want the input & output clocks to be exactly in phase
535 * otherwise small variations in the time will cause us to think
536 * we're a full frame off & there will be lots of drops and dups.
537 * We offset the input clock by half the duration so it's maximally
538 * out of phase with the output clock. */
539 if( cur->start < pv->next_start - ( duration >> 1 ) )
541 /* current frame too old - drop it */
544 pv->chap_mark = cur->new_chap;
546 hb_buffer_close( &cur );
547 pv->cur = cur = hb_fifo_get( job->fifo_raw );
548 pv->next_pts = next->start;
553 if( next->start > pv->next_start + duration + ( duration >> 1 ) )
555 /* next frame too far ahead - dup current frame */
556 buf_tmp = hb_buffer_init( cur->size );
557 hb_buffer_copy_settings( buf_tmp, cur );
558 memcpy( buf_tmp->data, cur->data, cur->size );
559 buf_tmp->sequence = cur->sequence;
564 /* this frame in our time window & doesn't need to be duped */
566 pv->cur = cur = hb_fifo_get( job->fifo_raw );
567 pv->next_pts = next->start;
573 * Adjust the pts of the current frame so that it's contiguous
574 * with the previous frame. The start time of the current frame
575 * has to be the end time of the previous frame and the stop
576 * time has to be the start of the next frame. We don't
577 * make any adjustments to the source timestamps other than removing
578 * the clock offsets (which also removes pts discontinuities).
579 * This means we automatically encode at the source's frame rate.
580 * MP2 uses an implicit duration (frames end when the next frame
581 * starts) but more advanced containers like MP4 use an explicit
582 * duration. Since we're looking ahead one frame we set the
583 * explicit stop time from the start time of the next frame.
586 pv->cur = cur = hb_fifo_get( job->fifo_raw );
587 pv->next_pts = cur->start;
588 duration = cur->start - buf_tmp->start;
591 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
592 duration, buf_tmp->start, next->start );
596 buf_tmp->start = pv->next_start;
597 pv->next_start += duration;
598 buf_tmp->stop = pv->next_start;
602 // we have a pending chapter mark from a recent drop - put it on this
603 // buffer (this may make it one frame late but we can't do any better).
604 buf_tmp->new_chap = pv->chap_mark;
608 /* If we have a subtitle for this picture, copy it */
609 /* FIXME: we should avoid this memcpy */
612 buf_tmp->sub = hb_buffer_init( sub->size );
613 buf_tmp->sub->x = sub->x;
614 buf_tmp->sub->y = sub->y;
615 buf_tmp->sub->width = sub->width;
616 buf_tmp->sub->height = sub->height;
617 memcpy( buf_tmp->sub->data, sub->data, sub->size );
620 /* Push the frame to the renderer */
621 hb_fifo_push( job->fifo_sync, buf_tmp );
626 /* Make sure we won't get more frames then expected */
627 if( pv->count_frames >= pv->count_frames_max * 2)
629 hb_log( "sync: got too many frames (%d), exiting early",
632 // Drop an empty buffer into our output to ensure that things
633 // get flushed all the way out.
634 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
641 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
642 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
644 int64_t start = sync->next_start;
645 int64_t duration = buf->stop - buf->start;
647 sync->next_pts += duration;
649 if( audio->config.in.samplerate == audio->config.out.samplerate ||
650 audio->config.out.codec == HB_ACODEC_AC3 ||
651 audio->config.out.codec == HB_ACODEC_DCA )
654 * If we don't have to do sample rate conversion or this audio is
655 * pass-thru just send the input buffer downstream after adjusting
656 * its timestamps to make the output stream continuous.
661 /* Not pass-thru - do sample rate conversion */
662 int count_in, count_out;
663 hb_buffer_t * buf_raw = buf;
664 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
667 count_in = buf_raw->size / channel_count;
669 * When using stupid rates like 44.1 there will always be some
670 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
671 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
672 * the error will build up over time and eventually the audio will
673 * substantially lag the video. libsamplerate will keep track of the
674 * fractional sample & give it to us when appropriate if we give it
675 * an extra sample of space in the output buffer.
677 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
679 sync->data.input_frames = count_in;
680 sync->data.output_frames = count_out;
681 sync->data.src_ratio = (double)audio->config.out.samplerate /
682 (double)audio->config.in.samplerate;
684 buf = hb_buffer_init( count_out * channel_count );
685 sync->data.data_in = (float *) buf_raw->data;
686 sync->data.data_out = (float *) buf->data;
687 if( src_process( sync->state, &sync->data ) )
689 /* XXX If this happens, we're screwed */
690 hb_log( "sync: audio %d src_process failed", i );
692 hb_buffer_close( &buf_raw );
694 buf->size = sync->data.output_frames_gen * channel_count;
695 duration = ( sync->data.output_frames_gen * 90000 ) /
696 audio->config.out.samplerate;
698 buf->frametype = HB_FRAME_AUDIO;
700 buf->stop = start + duration;
701 sync->next_start = start + duration;
702 hb_fifo_push( fifo, buf );
705 /***********************************************************************
707 ***********************************************************************
709 **********************************************************************/
710 static void SyncAudio( hb_work_object_t * w, int i )
712 hb_work_private_t * pv = w->private_data;
713 hb_job_t * job = pv->job;
714 hb_sync_audio_t * sync = &pv->sync_audio[i];
715 hb_audio_t * audio = sync->audio;
719 if( audio->config.out.codec == HB_ACODEC_AC3 )
721 fifo = audio->priv.fifo_out;
725 fifo = audio->priv.fifo_sync;
728 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
730 /* if the next buffer is an eof send it downstream */
731 if ( buf->size <= 0 )
733 buf = hb_fifo_get( audio->priv.fifo_raw );
734 hb_fifo_push( fifo, buf );
735 pv->busy &=~ (1 << (i + 1) );
738 if ( (int64_t)( buf->start - sync->next_pts ) < 0 )
740 // audio time went backwards.
741 // If our output clock is more than a half frame ahead of the
742 // input clock drop this frame to move closer to sync.
743 // Otherwise drop frames until the input clock matches the output clock.
744 if ( sync->first_drop || sync->next_start - buf->start > 90*15 )
746 // Discard data that's in the past.
747 if ( sync->first_drop == 0 )
749 sync->first_drop = sync->next_pts;
752 buf = hb_fifo_get( audio->priv.fifo_raw );
753 hb_buffer_close( &buf );
756 sync->next_pts = buf->start;
758 if ( sync->first_drop )
760 // we were dropping old data but input buf time is now current
761 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
762 "(next %lld, current %lld)", i,
763 (int)( sync->next_pts - sync->first_drop ) / 90,
764 sync->drop_count, sync->first_drop, sync->next_pts );
765 sync->first_drop = 0;
766 sync->drop_count = 0;
767 sync->next_pts = buf->start;
769 if ( buf->start - sync->next_pts >= (90 * 70) )
771 if ( buf->start - sync->next_pts > (90000LL * 60) )
773 // there's a gap of more than a minute between the last
774 // frame and this. assume we got a corrupted timestamp
775 // and just drop the next buf.
776 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
777 " start %lld, next %lld",
778 (int)((buf->start - sync->next_pts) / (90000*60)),
779 i, buf->start, sync->next_pts );
780 buf = hb_fifo_get( audio->priv.fifo_raw );
781 hb_buffer_close( &buf );
785 * there's a gap of at least 70ms between the last
786 * frame we processed & the next. Fill it with silence.
788 hb_log( "sync: adding %d ms of silence to audio %d"
789 " start %lld, next %lld",
790 (int)((buf->start - sync->next_pts) / 90),
791 i, buf->start, sync->next_pts );
792 InsertSilence( w, i, buf->start - sync->next_pts );
797 * When we get here we've taken care of all the dups and gaps in the
798 * audio stream and are ready to inject the next input frame into
801 buf = hb_fifo_get( audio->priv.fifo_raw );
802 OutputAudioFrame( job, audio, buf, sync, fifo, i );
806 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
808 hb_work_private_t * pv = w->private_data;
809 hb_job_t *job = pv->job;
810 hb_sync_audio_t *sync = &pv->sync_audio[i];
814 // to keep pass-thru and regular audio in sync we generate silence in
815 // AC3 frame-sized units. If the silence duration isn't an integer multiple
816 // of the AC3 frame duration we will truncate or round up depending on
817 // which minimizes the timing error.
818 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
819 sync->audio->config.in.samplerate;
820 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
822 while ( --frame_count >= 0 )
824 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
826 buf = hb_buffer_init( sync->ac3_size );
827 buf->start = sync->next_pts;
828 buf->stop = buf->start + frame_dur;
829 memcpy( buf->data, sync->ac3_buf, buf->size );
830 fifo = sync->audio->priv.fifo_out;
834 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
835 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
836 sync->audio->config.out.mixdown) );
837 buf->start = sync->next_pts;
838 buf->stop = buf->start + frame_dur;
839 memset( buf->data, 0, buf->size );
840 fifo = sync->audio->priv.fifo_sync;
842 OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
846 static void UpdateState( hb_work_object_t * w )
848 hb_work_private_t * pv = w->private_data;
851 if( !pv->count_frames )
853 pv->st_first = hb_get_date();
857 if( hb_get_date() > pv->st_dates[3] + 1000 )
859 memmove( &pv->st_dates[0], &pv->st_dates[1],
860 3 * sizeof( uint64_t ) );
861 memmove( &pv->st_counts[0], &pv->st_counts[1],
862 3 * sizeof( uint64_t ) );
863 pv->st_dates[3] = hb_get_date();
864 pv->st_counts[3] = pv->count_frames;
867 #define p state.param.working
868 state.state = HB_STATE_WORKING;
869 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
870 if( p.progress > 1.0 )
874 p.rate_cur = 1000.0 *
875 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
876 (float) ( pv->st_dates[3] - pv->st_dates[0] );
877 if( hb_get_date() > pv->st_first + 4000 )
880 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
881 (float) ( pv->st_dates[3] - pv->st_first );
882 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
884 p.hours = eta / 3600;
885 p.minutes = ( eta % 3600 ) / 60;
886 p.seconds = eta % 60;
897 hb_set_state( pv->job->h, &state );