1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
11 #include "ffmpeg/avcodec.h"
14 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
16 #define INT64_MIN (-9223372036854775807LL-1)
18 #define AC3_SAMPLES_PER_FRAME 1536
24 int64_t next_start; /* start time of next output frame */
25 int64_t next_pts; /* start time of next input frame */
26 int64_t start_silence; /* if we're inserting silence, the time we started */
27 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
28 int drop_count; /* count of 'time went backwards' drops */
40 struct hb_work_private_s
46 hb_subtitle_t * subtitle;
48 int64_t next_start; /* start time of next output frame */
49 int64_t next_pts; /* start time of next input frame */
50 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
51 int drop_count; /* count of 'time went backwards' drops */
52 int drops; /* frames dropped to make a cbr video stream */
53 int dups; /* frames duplicated to make a cbr video stream */
57 int chap_mark; /* to propagate chapter mark across a drop */
58 hb_buffer_t * cur; /* The next picture to process */
61 hb_sync_audio_t sync_audio[8];
64 uint64_t st_counts[4];
69 /***********************************************************************
71 **********************************************************************/
72 static void InitAudio( hb_work_object_t * w, int i );
73 static int SyncVideo( hb_work_object_t * w );
74 static void SyncAudio( hb_work_object_t * w, int i );
75 static int NeedSilence( hb_work_object_t * w, hb_audio_t *, int i );
76 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
77 static void UpdateState( hb_work_object_t * w );
79 /***********************************************************************
81 ***********************************************************************
82 * Initialize the work object
83 **********************************************************************/
84 int syncInit( hb_work_object_t * w, hb_job_t * job )
86 hb_title_t * title = job->title;
87 hb_chapter_t * chapter;
90 hb_work_private_t * pv;
92 pv = calloc( 1, sizeof( hb_work_private_t ) );
96 pv->pts_offset = INT64_MIN;
99 /* Calculate how many video frames we are expecting */
101 for( i = job->chapter_start; i <= job->chapter_end; i++ )
103 chapter = hb_list_item( title->list_chapter, i - 1 );
104 duration += chapter->duration;
107 /* 1 second safety so we're sure we won't miss anything */
108 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
110 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
112 /* Initialize libsamplerate for every audio track we have */
113 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
118 /* Get subtitle info, if any */
119 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
121 pv->video_sequence = 0;
126 /***********************************************************************
128 ***********************************************************************
130 **********************************************************************/
131 void syncClose( hb_work_object_t * w )
133 hb_work_private_t * pv = w->private_data;
134 hb_job_t * job = pv->job;
135 hb_title_t * title = job->title;
136 hb_audio_t * audio = NULL;
141 hb_buffer_close( &pv->cur );
144 hb_log( "sync: got %d frames, %d expected",
145 pv->count_frames, pv->count_frames_max );
147 if (pv->drops || pv->dups )
149 hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
152 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
154 if ( pv->sync_audio[i].start_silence )
156 hb_log( "sync: added %d ms of silence to audio %d",
157 (int)((pv->sync_audio[i].next_pts -
158 pv->sync_audio[i].start_silence) / 90), i );
161 audio = hb_list_item( title->list_audio, i );
162 if( audio->config.out.codec == HB_ACODEC_AC3 )
164 free( pv->sync_audio[i].ac3_buf );
168 src_delete( pv->sync_audio[i].state );
173 w->private_data = NULL;
176 /***********************************************************************
178 ***********************************************************************
179 * The root routine of this work abject
181 * The way this works is that we are syncing the audio to the PTS of
182 * the last video that we processed. That's why we skip the audio sync
183 * if we haven't got a valid PTS from the video yet.
185 **********************************************************************/
186 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
187 hb_buffer_t ** unused2 )
189 hb_work_private_t * pv = w->private_data;
192 /* If we ever got a video frame, handle audio now */
193 if( pv->pts_offset != INT64_MIN )
195 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
202 return SyncVideo( w );
205 hb_work_object_t hb_sync =
214 static void InitAudio( hb_work_object_t * w, int i )
216 hb_work_private_t * pv = w->private_data;
217 hb_job_t * job = pv->job;
218 hb_title_t * title = job->title;
219 hb_sync_audio_t * sync;
221 sync = &pv->sync_audio[i];
222 sync->audio = hb_list_item( title->list_audio, i );
224 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
226 /* Have a silent AC-3 frame ready in case we have to fill a
232 codec = avcodec_find_encoder( CODEC_ID_AC3 );
233 c = avcodec_alloc_context();
235 c->bit_rate = sync->audio->config.in.bitrate;
236 c->sample_rate = sync->audio->config.in.samplerate;
237 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
239 if( avcodec_open( c, codec ) < 0 )
241 hb_log( "sync: avcodec_open failed" );
245 zeros = calloc( AC3_SAMPLES_PER_FRAME *
246 sizeof( short ) * c->channels, 1 );
247 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
248 sync->audio->config.in.samplerate / 8;
249 sync->ac3_buf = malloc( sync->ac3_size );
251 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
252 zeros ) != sync->ac3_size )
254 hb_log( "sync: avcodec_encode_audio failed" );
263 /* Initialize libsamplerate */
265 sync->state = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
266 sync->data.end_of_input = 0;
270 /***********************************************************************
272 ***********************************************************************
274 **********************************************************************/
275 static int SyncVideo( hb_work_object_t * w )
277 hb_work_private_t * pv = w->private_data;
278 hb_buffer_t * cur, * next, * sub = NULL;
279 hb_job_t * job = pv->job;
286 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
288 /* We haven't even got a frame yet */
292 if( cur->size == 0 && pv->pts_offset == INT64_MIN )
294 /* we got an end-of-stream with no video frames (happens during
295 * an indepth_scan). Feed the eos downstream & signal that we're done. */
296 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
301 /* At this point we have a frame to process. Let's check
302 1) if we will be able to push into the fifo ahead
303 2) if the next frame is there already, since we need it to
304 compute the duration of the current frame*/
305 while( !hb_fifo_is_full( job->fifo_sync ) &&
306 ( next = hb_fifo_see( job->fifo_raw ) ) )
308 hb_buffer_t * buf_tmp;
310 if( next->size == 0 )
312 // we got the empty buffer that signals end-of-stream
313 // note that we're done but continue to the end of this
314 // loop so that the final frame gets processed.
316 next->start = pv->next_pts + 90*30;
319 if( pv->pts_offset == INT64_MIN )
321 /* This is our first frame */
323 if ( cur->start != 0 )
326 * The first pts from a dvd should always be zero but
327 * can be non-zero with a transport or program stream since
328 * we're not guaranteed to start on an IDR frame. If we get
329 * a non-zero initial PTS extend its duration so it behaves
330 * as if it started at zero so that our audio timing will
333 hb_log( "sync: first pts is %lld", cur->start );
339 * since the first frame is always 0 and the upstream reader code
340 * is taking care of adjusting for pts discontinuities, we just have
341 * to deal with the next frame's start being in the past. This can
342 * happen when the PTS is adjusted after data loss but video frame
343 * reordering causes some frames with the old clock to appear after
344 * the clock change. This creates frames that overlap in time which
345 * looks to us like time going backward. The downstream muxing code
346 * can deal with overlaps of up to a frame time but anything larger
347 * we handle by dropping frames here.
349 if ( (int64_t)( next->start - pv->next_pts ) <= 0 )
351 if ( pv->first_drop == 0 )
353 pv->first_drop = next->start;
356 buf_tmp = hb_fifo_get( job->fifo_raw );
357 if ( buf_tmp->new_chap )
359 // don't drop a chapter mark when we drop the buffer
360 pv->chap_mark = buf_tmp->new_chap;
362 hb_buffer_close( &buf_tmp );
365 if ( pv->first_drop )
367 hb_log( "sync: video time didn't advance - dropped %d frames "
368 "(delta %d ms, current %lld, next %lld)",
369 pv->drop_count, (int)( pv->next_pts - pv->first_drop ) / 90,
370 pv->next_pts, pv->first_drop );
376 * Track the video sequence number localy so that we can sync the audio
377 * to it using the sequence number as well as the PTS.
379 pv->video_sequence = cur->sequence;
381 /* Look for a subtitle for this frame */
385 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
387 /* If two subtitles overlap, make the first one stop
388 when the second one starts */
389 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
390 if( sub2 && sub->stop > sub2->start )
391 sub->stop = sub2->start;
393 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
394 // sub, cur->sequence, sub->sequence);
396 if( sub->sequence > cur->sequence )
399 * The video is behind where we are, so wait until
400 * it catches up to the same reader point on the
401 * DVD. Then our PTS should be in the same region
408 if( sub->stop > cur->start ) {
410 * The stop time is in the future, so fall through
411 * and we'll deal with it in the next block of
418 * The subtitle is older than this picture, trash it
420 sub = hb_fifo_get( pv->subtitle->fifo_raw );
421 hb_buffer_close( &sub );
425 * There is a valid subtitle, is it time to display it?
429 if( sub->stop > sub->start)
432 * Normal subtitle which ends after it starts, check to
433 * see that the current video is between the start and end.
435 if( cur->start > sub->start &&
436 cur->start < sub->stop )
439 * We should be playing this, so leave the
442 * fall through to display
444 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
447 * Subtitle is on for less than three seconds, extend
448 * the time that it is displayed to make it easier
449 * to read. Make it 3 seconds or until the next
450 * subtitle is displayed.
452 * This is in response to Indochine which only
453 * displays subs for 1 second - too fast to read.
455 sub->stop = sub->start + ( 3 * 90000 );
457 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
459 if( sub2 && sub->stop > sub2->start )
461 sub->stop = sub2->start;
468 * Defer until the play point is within the subtitle
476 * The end of the subtitle is less than the start, this is a
477 * sign of a PTS discontinuity.
479 if( sub->start > cur->start )
482 * we haven't reached the start time yet, or
483 * we have jumped backwards after having
484 * already started this subtitle.
486 if( cur->start < sub->stop )
489 * We have jumped backwards and so should
490 * continue displaying this subtitle.
492 * fall through to display.
498 * Defer until the play point is within the subtitle
504 * Play this subtitle as the start is greater than our
507 * fall through to display/
515 if ( job->mux & HB_MUX_AVI )
518 * The concept of variable frame rate video was a bit too advanced
519 * for Microsoft so AVI doesn't support it. Since almost all dvd
520 * video is VFR we have to convert it to constant frame rate to
521 * put it in an AVI container. So here we duplicate, drop and
522 * otherwise trash video frames to appease the gods of Redmond.
525 /* mpeg durations are exact when expressed in ticks of the
526 * 27MHz System clock but not in HB's 90KHz PTS clock. To avoid
527 * a truncation bias that will eventually cause the audio to desync
528 * we compute the duration of the next frame using 27MHz ticks
529 * then truncate it to 90KHz. */
530 duration = ( (int64_t)(pv->count_frames + 1 ) * job->vrate_base ) / 300 -
533 /* We don't want the input & output clocks to be exactly in phase
534 * otherwise small variations in the time will cause us to think
535 * we're a full frame off & there will be lots of drops and dups.
536 * We offset the input clock by half the duration so it's maximally
537 * out of phase with the output clock. */
538 if( cur->start < pv->next_start - ( duration >> 1 ) )
540 /* current frame too old - drop it */
543 pv->chap_mark = cur->new_chap;
545 hb_buffer_close( &cur );
546 pv->cur = cur = hb_fifo_get( job->fifo_raw );
547 pv->next_pts = next->start;
552 if( next->start > pv->next_start + duration + ( duration >> 1 ) )
554 /* next frame too far ahead - dup current frame */
555 buf_tmp = hb_buffer_init( cur->size );
556 hb_buffer_copy_settings( buf_tmp, cur );
557 memcpy( buf_tmp->data, cur->data, cur->size );
558 buf_tmp->sequence = cur->sequence;
563 /* this frame in our time window & doesn't need to be duped */
565 pv->cur = cur = hb_fifo_get( job->fifo_raw );
566 pv->next_pts = next->start;
572 * Adjust the pts of the current frame so that it's contiguous
573 * with the previous frame. The start time of the current frame
574 * has to be the end time of the previous frame and the stop
575 * time has to be the start of the next frame. We don't
576 * make any adjustments to the source timestamps other than removing
577 * the clock offsets (which also removes pts discontinuities).
578 * This means we automatically encode at the source's frame rate.
579 * MP2 uses an implicit duration (frames end when the next frame
580 * starts) but more advanced containers like MP4 use an explicit
581 * duration. Since we're looking ahead one frame we set the
582 * explicit stop time from the start time of the next frame.
585 pv->cur = cur = hb_fifo_get( job->fifo_raw );
586 pv->next_pts = next->start;
587 duration = next->start - buf_tmp->start;
590 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
591 duration, buf_tmp->start, next->start );
595 buf_tmp->start = pv->next_start;
596 pv->next_start += duration;
597 buf_tmp->stop = pv->next_start;
601 // we have a pending chapter mark from a recent drop - put it on this
602 // buffer (this may make it one frame late but we can't do any better).
603 buf_tmp->new_chap = pv->chap_mark;
607 /* If we have a subtitle for this picture, copy it */
608 /* FIXME: we should avoid this memcpy */
611 buf_tmp->sub = hb_buffer_init( sub->size );
612 buf_tmp->sub->x = sub->x;
613 buf_tmp->sub->y = sub->y;
614 buf_tmp->sub->width = sub->width;
615 buf_tmp->sub->height = sub->height;
616 memcpy( buf_tmp->sub->data, sub->data, sub->size );
619 /* Push the frame to the renderer */
620 hb_fifo_push( job->fifo_sync, buf_tmp );
625 /* Make sure we won't get more frames then expected */
626 if( pv->count_frames >= pv->count_frames_max * 2)
628 hb_log( "sync: got too many frames (%d), exiting early", pv->count_frames );
634 hb_buffer_close( &pv->cur );
636 // Drop an empty buffer into our output to ensure that things
637 // get flushed all the way out.
638 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
645 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
646 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
648 int64_t start = sync->next_start;
649 int64_t duration = buf->stop - buf->start;
651 sync->next_pts += duration;
653 if( audio->config.in.samplerate == audio->config.out.samplerate ||
654 audio->config.out.codec == HB_ACODEC_AC3 ||
655 audio->config.out.codec == HB_ACODEC_DCA )
658 * If we don't have to do sample rate conversion or this audio is
659 * pass-thru just send the input buffer downstream after adjusting
660 * its timestamps to make the output stream continuous.
665 /* Not pass-thru - do sample rate conversion */
666 int count_in, count_out;
667 hb_buffer_t * buf_raw = buf;
668 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
671 count_in = buf_raw->size / channel_count;
673 * When using stupid rates like 44.1 there will always be some
674 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
675 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
676 * the error will build up over time and eventually the audio will
677 * substantially lag the video. libsamplerate will keep track of the
678 * fractional sample & give it to us when appropriate if we give it
679 * an extra sample of space in the output buffer.
681 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
683 sync->data.input_frames = count_in;
684 sync->data.output_frames = count_out;
685 sync->data.src_ratio = (double)audio->config.out.samplerate /
686 (double)audio->config.in.samplerate;
688 buf = hb_buffer_init( count_out * channel_count );
689 sync->data.data_in = (float *) buf_raw->data;
690 sync->data.data_out = (float *) buf->data;
691 if( src_process( sync->state, &sync->data ) )
693 /* XXX If this happens, we're screwed */
694 hb_log( "sync: audio %d src_process failed", i );
696 hb_buffer_close( &buf_raw );
698 buf->size = sync->data.output_frames_gen * channel_count;
699 duration = ( sync->data.output_frames_gen * 90000 ) /
700 audio->config.out.samplerate;
702 buf->frametype = HB_FRAME_AUDIO;
704 buf->stop = start + duration;
705 sync->next_start = start + duration;
706 hb_fifo_push( fifo, buf );
709 /***********************************************************************
711 ***********************************************************************
713 **********************************************************************/
714 static void SyncAudio( hb_work_object_t * w, int i )
716 hb_work_private_t * pv = w->private_data;
717 hb_job_t * job = pv->job;
718 hb_sync_audio_t * sync = &pv->sync_audio[i];
719 hb_audio_t * audio = sync->audio;
723 if( audio->config.out.codec == HB_ACODEC_AC3 )
725 fifo = audio->priv.fifo_out;
729 fifo = audio->priv.fifo_sync;
732 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
734 if ( (int64_t)( buf->start - sync->next_pts ) < 0 )
737 * audio time went backwards by more than a frame time (this can
738 * happen when we reset the PTS because of lost data).
739 * Discard data that's in the past.
741 if ( sync->first_drop == 0 )
743 sync->first_drop = buf->start;
746 buf = hb_fifo_get( audio->priv.fifo_raw );
747 hb_buffer_close( &buf );
750 if ( sync->first_drop )
752 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
753 "(next %lld, current %lld)", i,
754 (int)( sync->next_pts - sync->first_drop ) / 90,
755 sync->drop_count, sync->first_drop, sync->next_pts );
756 sync->first_drop = 0;
757 sync->drop_count = 0;
759 if ( buf->start - sync->next_pts >= (90 * 70) )
762 * there's a gap of at least 70ms between the last
763 * frame we processed & the next. Fill it with silence.
765 hb_log( "sync: adding %d ms of silence to audio %d"
766 " start %lld, next %lld",
767 (int)((buf->start - sync->next_pts) / 90),
768 i, buf->start, sync->next_pts );
769 InsertSilence( w, i, buf->start - sync->next_pts );
774 * When we get here we've taken care of all the dups and gaps in the
775 * audio stream and are ready to inject the next input frame into
778 buf = hb_fifo_get( audio->priv.fifo_raw );
779 OutputAudioFrame( job, audio, buf, sync, fifo, i );
782 if( NeedSilence( w, audio, i ) )
784 InsertSilence( w, i, (90000 * AC3_SAMPLES_PER_FRAME) /
785 sync->audio->config.in.samplerate );
789 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio, int i )
791 hb_work_private_t * pv = w->private_data;
792 hb_job_t * job = pv->job;
793 hb_sync_audio_t * sync = &pv->sync_audio[i];
795 if( hb_fifo_size( audio->priv.fifo_in ) ||
796 hb_fifo_size( audio->priv.fifo_raw ) ||
797 hb_fifo_size( audio->priv.fifo_sync ) ||
798 hb_fifo_size( audio->priv.fifo_out ) )
800 /* We have some audio, we are fine */
804 /* No audio left in fifos */
806 if( hb_thread_has_exited( job->reader ) )
808 /* We might miss some audio to complete encoding and muxing
810 if ( sync->start_silence == 0 )
812 hb_log("sync: reader has exited, adding silence to audio %d", i);
813 sync->start_silence = sync->next_pts;
820 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
822 hb_work_private_t * pv = w->private_data;
823 hb_job_t *job = pv->job;
824 hb_sync_audio_t *sync = &pv->sync_audio[i];
828 // to keep pass-thru and regular audio in sync we generate silence in
829 // AC3 frame-sized units. If the silence duration isn't an integer multiple
830 // of the AC3 frame duration we will truncate or round up depending on
831 // which minimizes the timing error.
832 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
833 sync->audio->config.in.samplerate;
834 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
836 while ( --frame_count >= 0 )
838 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
840 buf = hb_buffer_init( sync->ac3_size );
841 buf->start = sync->next_pts;
842 buf->stop = buf->start + frame_dur;
843 memcpy( buf->data, sync->ac3_buf, buf->size );
844 fifo = sync->audio->priv.fifo_out;
848 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
849 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
850 sync->audio->config.out.mixdown) );
851 buf->start = sync->next_pts;
852 buf->stop = buf->start + frame_dur;
853 memset( buf->data, 0, buf->size );
854 fifo = sync->audio->priv.fifo_sync;
856 OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
860 static void UpdateState( hb_work_object_t * w )
862 hb_work_private_t * pv = w->private_data;
865 if( !pv->count_frames )
867 pv->st_first = hb_get_date();
871 if( hb_get_date() > pv->st_dates[3] + 1000 )
873 memmove( &pv->st_dates[0], &pv->st_dates[1],
874 3 * sizeof( uint64_t ) );
875 memmove( &pv->st_counts[0], &pv->st_counts[1],
876 3 * sizeof( uint64_t ) );
877 pv->st_dates[3] = hb_get_date();
878 pv->st_counts[3] = pv->count_frames;
881 #define p state.param.working
882 state.state = HB_STATE_WORKING;
883 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
884 if( p.progress > 1.0 )
888 p.rate_cur = 1000.0 *
889 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
890 (float) ( pv->st_dates[3] - pv->st_dates[0] );
891 if( hb_get_date() > pv->st_first + 4000 )
894 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
895 (float) ( pv->st_dates[3] - pv->st_first );
896 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
898 p.hours = eta / 3600;
899 p.minutes = ( eta % 3600 ) / 60;
900 p.seconds = eta % 60;
911 hb_set_state( pv->job->h, &state );