1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
23 int64_t next_start; /* start time of next output frame */
24 int64_t next_pts; /* start time of next input frame */
25 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
26 int drop_count; /* count of 'time went backwards' drops */
38 struct hb_work_private_s
41 int busy; // bitmask with one bit for each active input
42 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
43 // appropriate bit is cleared when input gets
44 // an eof buf. syncWork returns done when all
47 hb_subtitle_t * subtitle;
49 int64_t next_start; /* start time of next output frame */
50 int64_t next_pts; /* start time of next input frame */
51 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
52 int drop_count; /* count of 'time went backwards' drops */
53 int drops; /* frames dropped to make a cbr video stream */
54 int dups; /* frames duplicated to make a cbr video stream */
58 int chap_mark; /* to propagate chapter mark across a drop */
59 hb_buffer_t * cur; /* The next picture to process */
62 hb_sync_audio_t sync_audio[8];
65 uint64_t st_counts[4];
70 /***********************************************************************
72 **********************************************************************/
73 static void InitAudio( hb_work_object_t * w, int i );
74 static void SyncVideo( hb_work_object_t * w );
75 static void SyncAudio( hb_work_object_t * w, int i );
76 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
77 static void UpdateState( hb_work_object_t * w );
79 /***********************************************************************
81 ***********************************************************************
82 * Initialize the work object
83 **********************************************************************/
84 int syncInit( hb_work_object_t * w, hb_job_t * job )
86 hb_title_t * title = job->title;
87 hb_chapter_t * chapter;
90 hb_work_private_t * pv;
92 pv = calloc( 1, sizeof( hb_work_private_t ) );
96 pv->pts_offset = INT64_MIN;
98 /* Calculate how many video frames we are expecting */
101 duration = job->pts_to_stop + 90000;
106 for( i = job->chapter_start; i <= job->chapter_end; i++ )
108 chapter = hb_list_item( title->list_chapter, i - 1 );
109 duration += chapter->duration;
112 /* 1 second safety so we're sure we won't miss anything */
114 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
116 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
119 /* Initialize libsamplerate for every audio track we have */
120 if ( ! job->indepth_scan )
122 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
124 pv->busy |= ( 1 << (i + 1) );
129 /* Get subtitle info, if any */
130 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
135 /***********************************************************************
137 ***********************************************************************
139 **********************************************************************/
140 void syncClose( hb_work_object_t * w )
142 hb_work_private_t * pv = w->private_data;
143 hb_job_t * job = pv->job;
144 hb_title_t * title = job->title;
145 hb_audio_t * audio = NULL;
150 hb_buffer_close( &pv->cur );
153 hb_log( "sync: got %d frames, %d expected",
154 pv->count_frames, pv->count_frames_max );
156 if (pv->drops || pv->dups )
158 hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
161 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
163 audio = hb_list_item( title->list_audio, i );
164 if( audio->config.out.codec == HB_ACODEC_AC3 )
166 free( pv->sync_audio[i].ac3_buf );
170 src_delete( pv->sync_audio[i].state );
175 w->private_data = NULL;
178 /***********************************************************************
180 ***********************************************************************
181 * The root routine of this work abject
183 * The way this works is that we are syncing the audio to the PTS of
184 * the last video that we processed. That's why we skip the audio sync
185 * if we haven't got a valid PTS from the video yet.
187 **********************************************************************/
188 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
189 hb_buffer_t ** unused2 )
191 hb_work_private_t * pv = w->private_data;
197 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
199 if ( pv->busy & ( 1 << (i + 1) ) )
203 return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
206 hb_work_object_t hb_sync =
215 static void InitAudio( hb_work_object_t * w, int i )
217 hb_work_private_t * pv = w->private_data;
218 hb_job_t * job = pv->job;
219 hb_title_t * title = job->title;
220 hb_sync_audio_t * sync;
222 sync = &pv->sync_audio[i];
223 sync->audio = hb_list_item( title->list_audio, i );
225 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
227 /* Have a silent AC-3 frame ready in case we have to fill a
233 codec = avcodec_find_encoder( CODEC_ID_AC3 );
234 c = avcodec_alloc_context();
236 c->bit_rate = sync->audio->config.in.bitrate;
237 c->sample_rate = sync->audio->config.in.samplerate;
238 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
240 if( hb_avcodec_open( c, codec ) < 0 )
242 hb_log( "sync: avcodec_open failed" );
246 zeros = calloc( AC3_SAMPLES_PER_FRAME *
247 sizeof( short ) * c->channels, 1 );
248 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
249 sync->audio->config.in.samplerate / 8;
250 sync->ac3_buf = malloc( sync->ac3_size );
252 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
253 zeros ) != sync->ac3_size )
255 hb_log( "sync: avcodec_encode_audio failed" );
259 hb_avcodec_close( c );
264 /* Initialize libsamplerate */
266 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
267 sync->data.end_of_input = 0;
271 /***********************************************************************
273 ***********************************************************************
275 **********************************************************************/
276 static void SyncVideo( hb_work_object_t * w )
278 hb_work_private_t * pv = w->private_data;
279 hb_buffer_t * cur, * next, * sub = NULL;
280 hb_job_t * job = pv->job;
282 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
284 /* We haven't even got a frame yet */
290 /* we got an end-of-stream. Feed it downstream & signal that we're done. */
291 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
296 /* At this point we have a frame to process. Let's check
297 1) if we will be able to push into the fifo ahead
298 2) if the next frame is there already, since we need it to
299 compute the duration of the current frame*/
300 while( !hb_fifo_is_full( job->fifo_sync ) &&
301 ( next = hb_fifo_see( job->fifo_raw ) ) )
303 hb_buffer_t * buf_tmp;
305 if( next->size == 0 )
307 /* we got an end-of-stream. Feed it downstream & signal that
308 * we're done. Note that this means we drop the final frame of
309 * video (we don't know its duration). On DVDs the final frame
310 * is often strange and dropping it seems to be a good idea. */
311 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
315 if( pv->pts_offset == INT64_MIN )
317 /* This is our first frame */
319 if ( cur->start != 0 )
322 * The first pts from a dvd should always be zero but
323 * can be non-zero with a transport or program stream since
324 * we're not guaranteed to start on an IDR frame. If we get
325 * a non-zero initial PTS extend its duration so it behaves
326 * as if it started at zero so that our audio timing will
329 hb_log( "sync: first pts is %lld", cur->start );
334 if( cur->new_chap ) {
335 hb_log("sync got new chapter %d", cur->new_chap );
339 * since the first frame is always 0 and the upstream reader code
340 * is taking care of adjusting for pts discontinuities, we just have
341 * to deal with the next frame's start being in the past. This can
342 * happen when the PTS is adjusted after data loss but video frame
343 * reordering causes some frames with the old clock to appear after
344 * the clock change. This creates frames that overlap in time which
345 * looks to us like time going backward. The downstream muxing code
346 * can deal with overlaps of up to a frame time but anything larger
347 * we handle by dropping frames here.
349 if ( (int64_t)( next->start - cur->start ) <= 0 )
351 if ( pv->first_drop == 0 )
353 pv->first_drop = next->start;
356 buf_tmp = hb_fifo_get( job->fifo_raw );
357 if ( buf_tmp->new_chap )
359 // don't drop a chapter mark when we drop the buffer
360 pv->chap_mark = buf_tmp->new_chap;
362 hb_buffer_close( &buf_tmp );
365 if ( pv->first_drop )
367 hb_log( "sync: video time didn't advance - dropped %d frames "
368 "(delta %d ms, current %lld, next %lld, dur %d)",
369 pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
370 cur->start, next->start, (int)( next->start - cur->start ) );
376 * Track the video sequence number localy so that we can sync the audio
377 * to it using the sequence number as well as the PTS.
379 pv->video_sequence = cur->sequence;
381 /* Look for a subtitle for this frame */
385 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
387 /* If two subtitles overlap, make the first one stop
388 when the second one starts */
389 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
390 if( sub2 && sub->stop > sub2->start )
391 sub->stop = sub2->start;
393 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
394 // sub, cur->sequence, sub->sequence);
396 if( sub->sequence > cur->sequence )
399 * The video is behind where we are, so wait until
400 * it catches up to the same reader point on the
401 * DVD. Then our PTS should be in the same region
408 if( sub->stop > cur->start ) {
410 * The stop time is in the future, so fall through
411 * and we'll deal with it in the next block of
418 * The subtitle is older than this picture, trash it
420 sub = hb_fifo_get( pv->subtitle->fifo_raw );
421 hb_buffer_close( &sub );
425 * There is a valid subtitle, is it time to display it?
429 if( sub->stop > sub->start)
432 * Normal subtitle which ends after it starts, check to
433 * see that the current video is between the start and end.
435 if( cur->start > sub->start &&
436 cur->start < sub->stop )
439 * We should be playing this, so leave the
442 * fall through to display
444 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
447 * Subtitle is on for less than three seconds, extend
448 * the time that it is displayed to make it easier
449 * to read. Make it 3 seconds or until the next
450 * subtitle is displayed.
452 * This is in response to Indochine which only
453 * displays subs for 1 second - too fast to read.
455 sub->stop = sub->start + ( 3 * 90000 );
457 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
459 if( sub2 && sub->stop > sub2->start )
461 sub->stop = sub2->start;
468 * Defer until the play point is within the subtitle
476 * The end of the subtitle is less than the start, this is a
477 * sign of a PTS discontinuity.
479 if( sub->start > cur->start )
482 * we haven't reached the start time yet, or
483 * we have jumped backwards after having
484 * already started this subtitle.
486 if( cur->start < sub->stop )
489 * We have jumped backwards and so should
490 * continue displaying this subtitle.
492 * fall through to display.
498 * Defer until the play point is within the subtitle
504 * Play this subtitle as the start is greater than our
507 * fall through to display/
515 if ( job->mux & HB_MUX_AVI || job->cfr )
518 * The concept of variable frame rate video was a bit too advanced
519 * for Microsoft so AVI doesn't support it. Since almost all dvd
520 * video is VFR we have to convert it to constant frame rate to
521 * put it in an AVI container. So here we duplicate, drop and
522 * otherwise trash video frames to appease the gods of Redmond.
525 /* mpeg durations are exact when expressed in ticks of the
526 * 27MHz System clock but not in HB's 90KHz PTS clock. To avoid
527 * a truncation bias that will eventually cause the audio to desync
528 * we compute the duration of the next frame using 27MHz ticks
529 * then truncate it to 90KHz. */
530 duration = ( (int64_t)(pv->count_frames + 1 ) * job->vrate_base ) / 300 -
533 /* We don't want the input & output clocks to be exactly in phase
534 * otherwise small variations in the time will cause us to think
535 * we're a full frame off & there will be lots of drops and dups.
536 * We offset the input clock by half the duration so it's maximally
537 * out of phase with the output clock. */
538 if( cur->start < pv->next_start - ( duration >> 1 ) )
540 /* current frame too old - drop it */
543 pv->chap_mark = cur->new_chap;
545 hb_buffer_close( &cur );
546 pv->cur = cur = hb_fifo_get( job->fifo_raw );
547 pv->next_pts = next->start;
552 if( next->start > pv->next_start + duration + ( duration >> 1 ) )
554 /* next frame too far ahead - dup current frame */
555 buf_tmp = hb_buffer_init( cur->size );
556 hb_buffer_copy_settings( buf_tmp, cur );
557 memcpy( buf_tmp->data, cur->data, cur->size );
558 buf_tmp->sequence = cur->sequence;
563 /* this frame in our time window & doesn't need to be duped */
565 pv->cur = cur = hb_fifo_get( job->fifo_raw );
566 pv->next_pts = next->start;
572 * Adjust the pts of the current frame so that it's contiguous
573 * with the previous frame. The start time of the current frame
574 * has to be the end time of the previous frame and the stop
575 * time has to be the start of the next frame. We don't
576 * make any adjustments to the source timestamps other than removing
577 * the clock offsets (which also removes pts discontinuities).
578 * This means we automatically encode at the source's frame rate.
579 * MP2 uses an implicit duration (frames end when the next frame
580 * starts) but more advanced containers like MP4 use an explicit
581 * duration. Since we're looking ahead one frame we set the
582 * explicit stop time from the start time of the next frame.
585 pv->cur = cur = hb_fifo_get( job->fifo_raw );
586 pv->next_pts = cur->start;
587 duration = cur->start - buf_tmp->start;
590 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
591 duration, buf_tmp->start, next->start );
595 buf_tmp->start = pv->next_start;
596 pv->next_start += duration;
597 buf_tmp->stop = pv->next_start;
601 // we have a pending chapter mark from a recent drop - put it on this
602 // buffer (this may make it one frame late but we can't do any better).
603 buf_tmp->new_chap = pv->chap_mark;
607 /* If we have a subtitle for this picture, copy it */
608 /* FIXME: we should avoid this memcpy */
611 buf_tmp->sub = hb_buffer_init( sub->size );
612 buf_tmp->sub->x = sub->x;
613 buf_tmp->sub->y = sub->y;
614 buf_tmp->sub->width = sub->width;
615 buf_tmp->sub->height = sub->height;
616 memcpy( buf_tmp->sub->data, sub->data, sub->size );
619 /* Push the frame to the renderer */
620 hb_fifo_push( job->fifo_sync, buf_tmp );
625 /* Make sure we won't get more frames then expected */
626 if( pv->count_frames >= pv->count_frames_max * 2)
628 hb_log( "sync: got too many frames (%d), exiting early",
631 // Drop an empty buffer into our output to ensure that things
632 // get flushed all the way out.
633 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
640 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
641 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
643 int64_t start = sync->next_start;
644 int64_t duration = buf->stop - buf->start;
646 sync->next_pts += duration;
648 if( audio->config.in.samplerate == audio->config.out.samplerate ||
649 audio->config.out.codec == HB_ACODEC_AC3 ||
650 audio->config.out.codec == HB_ACODEC_DCA )
653 * If we don't have to do sample rate conversion or this audio is
654 * pass-thru just send the input buffer downstream after adjusting
655 * its timestamps to make the output stream continuous.
660 /* Not pass-thru - do sample rate conversion */
661 int count_in, count_out;
662 hb_buffer_t * buf_raw = buf;
663 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
666 count_in = buf_raw->size / channel_count;
668 * When using stupid rates like 44.1 there will always be some
669 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
670 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
671 * the error will build up over time and eventually the audio will
672 * substantially lag the video. libsamplerate will keep track of the
673 * fractional sample & give it to us when appropriate if we give it
674 * an extra sample of space in the output buffer.
676 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
678 sync->data.input_frames = count_in;
679 sync->data.output_frames = count_out;
680 sync->data.src_ratio = (double)audio->config.out.samplerate /
681 (double)audio->config.in.samplerate;
683 buf = hb_buffer_init( count_out * channel_count );
684 sync->data.data_in = (float *) buf_raw->data;
685 sync->data.data_out = (float *) buf->data;
686 if( src_process( sync->state, &sync->data ) )
688 /* XXX If this happens, we're screwed */
689 hb_log( "sync: audio %d src_process failed", i );
691 hb_buffer_close( &buf_raw );
693 buf->size = sync->data.output_frames_gen * channel_count;
694 duration = ( sync->data.output_frames_gen * 90000 ) /
695 audio->config.out.samplerate;
697 buf->frametype = HB_FRAME_AUDIO;
699 buf->stop = start + duration;
700 sync->next_start = start + duration;
701 hb_fifo_push( fifo, buf );
704 /***********************************************************************
706 ***********************************************************************
708 **********************************************************************/
709 static void SyncAudio( hb_work_object_t * w, int i )
711 hb_work_private_t * pv = w->private_data;
712 hb_job_t * job = pv->job;
713 hb_sync_audio_t * sync = &pv->sync_audio[i];
714 hb_audio_t * audio = sync->audio;
718 if( audio->config.out.codec == HB_ACODEC_AC3 )
720 fifo = audio->priv.fifo_out;
724 fifo = audio->priv.fifo_sync;
727 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
729 /* if the next buffer is an eof send it downstream */
730 if ( buf->size <= 0 )
732 buf = hb_fifo_get( audio->priv.fifo_raw );
733 hb_fifo_push( fifo, buf );
734 pv->busy &=~ (1 << (i + 1) );
737 if ( (int64_t)( buf->start - sync->next_pts ) < 0 )
739 // audio time went backwards.
740 // If our output clock is more than a half frame ahead of the
741 // input clock drop this frame to move closer to sync.
742 // Otherwise drop frames until the input clock matches the output clock.
743 if ( sync->first_drop || sync->next_start - buf->start > 90*15 )
745 // Discard data that's in the past.
746 if ( sync->first_drop == 0 )
748 sync->first_drop = sync->next_pts;
751 buf = hb_fifo_get( audio->priv.fifo_raw );
752 hb_buffer_close( &buf );
755 sync->next_pts = buf->start;
757 if ( sync->first_drop )
759 // we were dropping old data but input buf time is now current
760 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
761 "(next %lld, current %lld)", i,
762 (int)( sync->next_pts - sync->first_drop ) / 90,
763 sync->drop_count, sync->first_drop, sync->next_pts );
764 sync->first_drop = 0;
765 sync->drop_count = 0;
766 sync->next_pts = buf->start;
768 if ( buf->start - sync->next_pts >= (90 * 70) )
770 if ( buf->start - sync->next_pts > (90000LL * 60) )
772 // there's a gap of more than a minute between the last
773 // frame and this. assume we got a corrupted timestamp
774 // and just drop the next buf.
775 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
776 " start %lld, next %lld",
777 (int)((buf->start - sync->next_pts) / (90000*60)),
778 i, buf->start, sync->next_pts );
779 buf = hb_fifo_get( audio->priv.fifo_raw );
780 hb_buffer_close( &buf );
784 * there's a gap of at least 70ms between the last
785 * frame we processed & the next. Fill it with silence.
787 hb_log( "sync: adding %d ms of silence to audio %d"
788 " start %lld, next %lld",
789 (int)((buf->start - sync->next_pts) / 90),
790 i, buf->start, sync->next_pts );
791 InsertSilence( w, i, buf->start - sync->next_pts );
796 * When we get here we've taken care of all the dups and gaps in the
797 * audio stream and are ready to inject the next input frame into
800 buf = hb_fifo_get( audio->priv.fifo_raw );
801 OutputAudioFrame( job, audio, buf, sync, fifo, i );
805 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
807 hb_work_private_t * pv = w->private_data;
808 hb_job_t *job = pv->job;
809 hb_sync_audio_t *sync = &pv->sync_audio[i];
813 // to keep pass-thru and regular audio in sync we generate silence in
814 // AC3 frame-sized units. If the silence duration isn't an integer multiple
815 // of the AC3 frame duration we will truncate or round up depending on
816 // which minimizes the timing error.
817 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
818 sync->audio->config.in.samplerate;
819 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
821 while ( --frame_count >= 0 )
823 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
825 buf = hb_buffer_init( sync->ac3_size );
826 buf->start = sync->next_pts;
827 buf->stop = buf->start + frame_dur;
828 memcpy( buf->data, sync->ac3_buf, buf->size );
829 fifo = sync->audio->priv.fifo_out;
833 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
834 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
835 sync->audio->config.out.mixdown) );
836 buf->start = sync->next_pts;
837 buf->stop = buf->start + frame_dur;
838 memset( buf->data, 0, buf->size );
839 fifo = sync->audio->priv.fifo_sync;
841 OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
845 static void UpdateState( hb_work_object_t * w )
847 hb_work_private_t * pv = w->private_data;
850 if( !pv->count_frames )
852 pv->st_first = hb_get_date();
856 if( hb_get_date() > pv->st_dates[3] + 1000 )
858 memmove( &pv->st_dates[0], &pv->st_dates[1],
859 3 * sizeof( uint64_t ) );
860 memmove( &pv->st_counts[0], &pv->st_counts[1],
861 3 * sizeof( uint64_t ) );
862 pv->st_dates[3] = hb_get_date();
863 pv->st_counts[3] = pv->count_frames;
866 #define p state.param.working
867 state.state = HB_STATE_WORKING;
868 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
869 if( p.progress > 1.0 )
873 p.rate_cur = 1000.0 *
874 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
875 (float) ( pv->st_dates[3] - pv->st_dates[0] );
876 if( hb_get_date() > pv->st_first + 4000 )
879 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
880 (float) ( pv->st_dates[3] - pv->st_first );
881 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
883 p.hours = eta / 3600;
884 p.minutes = ( eta % 3600 ) / 60;
885 p.seconds = eta % 60;
896 hb_set_state( pv->job->h, &state );