1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
11 #include "libavcodec/avcodec.h"
14 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
16 #define INT64_MIN (-9223372036854775807LL-1)
18 #define AC3_SAMPLES_PER_FRAME 1536
24 int64_t next_start; /* start time of next output frame */
25 int64_t next_pts; /* start time of next input frame */
26 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
27 int drop_count; /* count of 'time went backwards' drops */
39 struct hb_work_private_s
42 int busy; // bitmask with one bit for each active input
43 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
44 // appropriate bit is cleared when input gets
45 // an eof buf. syncWork returns done when all
48 hb_subtitle_t * subtitle;
50 int64_t next_start; /* start time of next output frame */
51 int64_t next_pts; /* start time of next input frame */
52 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
53 int drop_count; /* count of 'time went backwards' drops */
54 int drops; /* frames dropped to make a cbr video stream */
55 int dups; /* frames duplicated to make a cbr video stream */
59 int chap_mark; /* to propagate chapter mark across a drop */
60 hb_buffer_t * cur; /* The next picture to process */
63 hb_sync_audio_t sync_audio[8];
66 uint64_t st_counts[4];
71 /***********************************************************************
73 **********************************************************************/
74 static void InitAudio( hb_work_object_t * w, int i );
75 static void SyncVideo( hb_work_object_t * w );
76 static void SyncAudio( hb_work_object_t * w, int i );
77 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
78 static void UpdateState( hb_work_object_t * w );
80 /***********************************************************************
82 ***********************************************************************
83 * Initialize the work object
84 **********************************************************************/
85 int syncInit( hb_work_object_t * w, hb_job_t * job )
87 hb_title_t * title = job->title;
88 hb_chapter_t * chapter;
91 hb_work_private_t * pv;
93 pv = calloc( 1, sizeof( hb_work_private_t ) );
97 pv->pts_offset = INT64_MIN;
99 /* Calculate how many video frames we are expecting */
101 for( i = job->chapter_start; i <= job->chapter_end; i++ )
103 chapter = hb_list_item( title->list_chapter, i - 1 );
104 duration += chapter->duration;
107 /* 1 second safety so we're sure we won't miss anything */
108 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
110 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
113 /* Initialize libsamplerate for every audio track we have */
114 if ( ! job->indepth_scan )
116 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
118 pv->busy |= ( 1 << (i + 1) );
123 /* Get subtitle info, if any */
124 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
129 /***********************************************************************
131 ***********************************************************************
133 **********************************************************************/
134 void syncClose( hb_work_object_t * w )
136 hb_work_private_t * pv = w->private_data;
137 hb_job_t * job = pv->job;
138 hb_title_t * title = job->title;
139 hb_audio_t * audio = NULL;
144 hb_buffer_close( &pv->cur );
147 hb_log( "sync: got %d frames, %d expected",
148 pv->count_frames, pv->count_frames_max );
150 if (pv->drops || pv->dups )
152 hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
155 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
157 audio = hb_list_item( title->list_audio, i );
158 if( audio->config.out.codec == HB_ACODEC_AC3 )
160 free( pv->sync_audio[i].ac3_buf );
164 src_delete( pv->sync_audio[i].state );
169 w->private_data = NULL;
172 /***********************************************************************
174 ***********************************************************************
175 * The root routine of this work abject
177 * The way this works is that we are syncing the audio to the PTS of
178 * the last video that we processed. That's why we skip the audio sync
179 * if we haven't got a valid PTS from the video yet.
181 **********************************************************************/
182 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
183 hb_buffer_t ** unused2 )
185 hb_work_private_t * pv = w->private_data;
191 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
193 if ( pv->busy & ( 1 << (i + 1) ) )
197 return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
200 hb_work_object_t hb_sync =
209 static void InitAudio( hb_work_object_t * w, int i )
211 hb_work_private_t * pv = w->private_data;
212 hb_job_t * job = pv->job;
213 hb_title_t * title = job->title;
214 hb_sync_audio_t * sync;
216 sync = &pv->sync_audio[i];
217 sync->audio = hb_list_item( title->list_audio, i );
219 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
221 /* Have a silent AC-3 frame ready in case we have to fill a
227 codec = avcodec_find_encoder( CODEC_ID_AC3 );
228 c = avcodec_alloc_context();
230 c->bit_rate = sync->audio->config.in.bitrate;
231 c->sample_rate = sync->audio->config.in.samplerate;
232 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
234 if( avcodec_open( c, codec ) < 0 )
236 hb_log( "sync: avcodec_open failed" );
240 zeros = calloc( AC3_SAMPLES_PER_FRAME *
241 sizeof( short ) * c->channels, 1 );
242 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
243 sync->audio->config.in.samplerate / 8;
244 sync->ac3_buf = malloc( sync->ac3_size );
246 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
247 zeros ) != sync->ac3_size )
249 hb_log( "sync: avcodec_encode_audio failed" );
258 /* Initialize libsamplerate */
260 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
261 sync->data.end_of_input = 0;
265 /***********************************************************************
267 ***********************************************************************
269 **********************************************************************/
270 static void SyncVideo( hb_work_object_t * w )
272 hb_work_private_t * pv = w->private_data;
273 hb_buffer_t * cur, * next, * sub = NULL;
274 hb_job_t * job = pv->job;
276 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
278 /* We haven't even got a frame yet */
284 /* we got an end-of-stream. Feed it downstream & signal that we're done. */
285 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
290 /* At this point we have a frame to process. Let's check
291 1) if we will be able to push into the fifo ahead
292 2) if the next frame is there already, since we need it to
293 compute the duration of the current frame*/
294 while( !hb_fifo_is_full( job->fifo_sync ) &&
295 ( next = hb_fifo_see( job->fifo_raw ) ) )
297 hb_buffer_t * buf_tmp;
299 if( next->size == 0 )
301 /* we got an end-of-stream. Feed it downstream & signal that
302 * we're done. Note that this means we drop the final frame of
303 * video (we don't know its duration). On DVDs the final frame
304 * is often strange and dropping it seems to be a good idea. */
305 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
309 if( pv->pts_offset == INT64_MIN )
311 /* This is our first frame */
313 if ( cur->start != 0 )
316 * The first pts from a dvd should always be zero but
317 * can be non-zero with a transport or program stream since
318 * we're not guaranteed to start on an IDR frame. If we get
319 * a non-zero initial PTS extend its duration so it behaves
320 * as if it started at zero so that our audio timing will
323 hb_log( "sync: first pts is %lld", cur->start );
329 * since the first frame is always 0 and the upstream reader code
330 * is taking care of adjusting for pts discontinuities, we just have
331 * to deal with the next frame's start being in the past. This can
332 * happen when the PTS is adjusted after data loss but video frame
333 * reordering causes some frames with the old clock to appear after
334 * the clock change. This creates frames that overlap in time which
335 * looks to us like time going backward. The downstream muxing code
336 * can deal with overlaps of up to a frame time but anything larger
337 * we handle by dropping frames here.
339 if ( (int64_t)( next->start - cur->start ) <= 0 )
341 if ( pv->first_drop == 0 )
343 pv->first_drop = next->start;
346 buf_tmp = hb_fifo_get( job->fifo_raw );
347 if ( buf_tmp->new_chap )
349 // don't drop a chapter mark when we drop the buffer
350 pv->chap_mark = buf_tmp->new_chap;
352 hb_buffer_close( &buf_tmp );
355 if ( pv->first_drop )
357 hb_log( "sync: video time didn't advance - dropped %d frames "
358 "(delta %d ms, current %lld, next %lld, dur %d)",
359 pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
360 cur->start, next->start, (int)( next->start - cur->start ) );
366 * Track the video sequence number localy so that we can sync the audio
367 * to it using the sequence number as well as the PTS.
369 pv->video_sequence = cur->sequence;
371 /* Look for a subtitle for this frame */
375 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
377 /* If two subtitles overlap, make the first one stop
378 when the second one starts */
379 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
380 if( sub2 && sub->stop > sub2->start )
381 sub->stop = sub2->start;
383 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
384 // sub, cur->sequence, sub->sequence);
386 if( sub->sequence > cur->sequence )
389 * The video is behind where we are, so wait until
390 * it catches up to the same reader point on the
391 * DVD. Then our PTS should be in the same region
398 if( sub->stop > cur->start ) {
400 * The stop time is in the future, so fall through
401 * and we'll deal with it in the next block of
408 * The subtitle is older than this picture, trash it
410 sub = hb_fifo_get( pv->subtitle->fifo_raw );
411 hb_buffer_close( &sub );
415 * There is a valid subtitle, is it time to display it?
419 if( sub->stop > sub->start)
422 * Normal subtitle which ends after it starts, check to
423 * see that the current video is between the start and end.
425 if( cur->start > sub->start &&
426 cur->start < sub->stop )
429 * We should be playing this, so leave the
432 * fall through to display
434 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
437 * Subtitle is on for less than three seconds, extend
438 * the time that it is displayed to make it easier
439 * to read. Make it 3 seconds or until the next
440 * subtitle is displayed.
442 * This is in response to Indochine which only
443 * displays subs for 1 second - too fast to read.
445 sub->stop = sub->start + ( 3 * 90000 );
447 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
449 if( sub2 && sub->stop > sub2->start )
451 sub->stop = sub2->start;
458 * Defer until the play point is within the subtitle
466 * The end of the subtitle is less than the start, this is a
467 * sign of a PTS discontinuity.
469 if( sub->start > cur->start )
472 * we haven't reached the start time yet, or
473 * we have jumped backwards after having
474 * already started this subtitle.
476 if( cur->start < sub->stop )
479 * We have jumped backwards and so should
480 * continue displaying this subtitle.
482 * fall through to display.
488 * Defer until the play point is within the subtitle
494 * Play this subtitle as the start is greater than our
497 * fall through to display/
505 if ( job->mux & HB_MUX_AVI || job->cfr )
508 * The concept of variable frame rate video was a bit too advanced
509 * for Microsoft so AVI doesn't support it. Since almost all dvd
510 * video is VFR we have to convert it to constant frame rate to
511 * put it in an AVI container. So here we duplicate, drop and
512 * otherwise trash video frames to appease the gods of Redmond.
515 /* mpeg durations are exact when expressed in ticks of the
516 * 27MHz System clock but not in HB's 90KHz PTS clock. To avoid
517 * a truncation bias that will eventually cause the audio to desync
518 * we compute the duration of the next frame using 27MHz ticks
519 * then truncate it to 90KHz. */
520 duration = ( (int64_t)(pv->count_frames + 1 ) * job->vrate_base ) / 300 -
523 /* We don't want the input & output clocks to be exactly in phase
524 * otherwise small variations in the time will cause us to think
525 * we're a full frame off & there will be lots of drops and dups.
526 * We offset the input clock by half the duration so it's maximally
527 * out of phase with the output clock. */
528 if( cur->start < pv->next_start - ( duration >> 1 ) )
530 /* current frame too old - drop it */
533 pv->chap_mark = cur->new_chap;
535 hb_buffer_close( &cur );
536 pv->cur = cur = hb_fifo_get( job->fifo_raw );
537 pv->next_pts = next->start;
542 if( next->start > pv->next_start + duration + ( duration >> 1 ) )
544 /* next frame too far ahead - dup current frame */
545 buf_tmp = hb_buffer_init( cur->size );
546 hb_buffer_copy_settings( buf_tmp, cur );
547 memcpy( buf_tmp->data, cur->data, cur->size );
548 buf_tmp->sequence = cur->sequence;
553 /* this frame in our time window & doesn't need to be duped */
555 pv->cur = cur = hb_fifo_get( job->fifo_raw );
556 pv->next_pts = next->start;
562 * Adjust the pts of the current frame so that it's contiguous
563 * with the previous frame. The start time of the current frame
564 * has to be the end time of the previous frame and the stop
565 * time has to be the start of the next frame. We don't
566 * make any adjustments to the source timestamps other than removing
567 * the clock offsets (which also removes pts discontinuities).
568 * This means we automatically encode at the source's frame rate.
569 * MP2 uses an implicit duration (frames end when the next frame
570 * starts) but more advanced containers like MP4 use an explicit
571 * duration. Since we're looking ahead one frame we set the
572 * explicit stop time from the start time of the next frame.
575 pv->cur = cur = hb_fifo_get( job->fifo_raw );
576 pv->next_pts = cur->start;
577 duration = cur->start - buf_tmp->start;
580 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
581 duration, buf_tmp->start, next->start );
585 buf_tmp->start = pv->next_start;
586 pv->next_start += duration;
587 buf_tmp->stop = pv->next_start;
591 // we have a pending chapter mark from a recent drop - put it on this
592 // buffer (this may make it one frame late but we can't do any better).
593 buf_tmp->new_chap = pv->chap_mark;
597 /* If we have a subtitle for this picture, copy it */
598 /* FIXME: we should avoid this memcpy */
601 buf_tmp->sub = hb_buffer_init( sub->size );
602 buf_tmp->sub->x = sub->x;
603 buf_tmp->sub->y = sub->y;
604 buf_tmp->sub->width = sub->width;
605 buf_tmp->sub->height = sub->height;
606 memcpy( buf_tmp->sub->data, sub->data, sub->size );
609 /* Push the frame to the renderer */
610 hb_fifo_push( job->fifo_sync, buf_tmp );
615 /* Make sure we won't get more frames then expected */
616 if( pv->count_frames >= pv->count_frames_max * 2)
618 hb_log( "sync: got too many frames (%d), exiting early",
621 // Drop an empty buffer into our output to ensure that things
622 // get flushed all the way out.
623 hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
630 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
631 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
633 int64_t start = sync->next_start;
634 int64_t duration = buf->stop - buf->start;
636 sync->next_pts += duration;
638 if( audio->config.in.samplerate == audio->config.out.samplerate ||
639 audio->config.out.codec == HB_ACODEC_AC3 ||
640 audio->config.out.codec == HB_ACODEC_DCA )
643 * If we don't have to do sample rate conversion or this audio is
644 * pass-thru just send the input buffer downstream after adjusting
645 * its timestamps to make the output stream continuous.
650 /* Not pass-thru - do sample rate conversion */
651 int count_in, count_out;
652 hb_buffer_t * buf_raw = buf;
653 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
656 count_in = buf_raw->size / channel_count;
658 * When using stupid rates like 44.1 there will always be some
659 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
660 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
661 * the error will build up over time and eventually the audio will
662 * substantially lag the video. libsamplerate will keep track of the
663 * fractional sample & give it to us when appropriate if we give it
664 * an extra sample of space in the output buffer.
666 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
668 sync->data.input_frames = count_in;
669 sync->data.output_frames = count_out;
670 sync->data.src_ratio = (double)audio->config.out.samplerate /
671 (double)audio->config.in.samplerate;
673 buf = hb_buffer_init( count_out * channel_count );
674 sync->data.data_in = (float *) buf_raw->data;
675 sync->data.data_out = (float *) buf->data;
676 if( src_process( sync->state, &sync->data ) )
678 /* XXX If this happens, we're screwed */
679 hb_log( "sync: audio %d src_process failed", i );
681 hb_buffer_close( &buf_raw );
683 buf->size = sync->data.output_frames_gen * channel_count;
684 duration = ( sync->data.output_frames_gen * 90000 ) /
685 audio->config.out.samplerate;
687 buf->frametype = HB_FRAME_AUDIO;
689 buf->stop = start + duration;
690 sync->next_start = start + duration;
691 hb_fifo_push( fifo, buf );
694 /***********************************************************************
696 ***********************************************************************
698 **********************************************************************/
699 static void SyncAudio( hb_work_object_t * w, int i )
701 hb_work_private_t * pv = w->private_data;
702 hb_job_t * job = pv->job;
703 hb_sync_audio_t * sync = &pv->sync_audio[i];
704 hb_audio_t * audio = sync->audio;
708 if( audio->config.out.codec == HB_ACODEC_AC3 )
710 fifo = audio->priv.fifo_out;
714 fifo = audio->priv.fifo_sync;
717 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
719 /* if the next buffer is an eof send it downstream */
720 if ( buf->size <= 0 )
722 buf = hb_fifo_get( audio->priv.fifo_raw );
723 hb_fifo_push( fifo, buf );
724 pv->busy &=~ (1 << (i + 1) );
727 if ( (int64_t)( buf->start - sync->next_pts ) < 0 )
729 // audio time went backwards.
730 // If our output clock is more than a half frame ahead of the
731 // input clock drop this frame to move closer to sync.
732 // Otherwise drop frames until the input clock matches the output clock.
733 if ( sync->first_drop || sync->next_start - buf->start > 90*15 )
735 // Discard data that's in the past.
736 if ( sync->first_drop == 0 )
738 sync->first_drop = sync->next_pts;
741 buf = hb_fifo_get( audio->priv.fifo_raw );
742 hb_buffer_close( &buf );
745 sync->next_pts = buf->start;
747 if ( sync->first_drop )
749 // we were dropping old data but input buf time is now current
750 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
751 "(next %lld, current %lld)", i,
752 (int)( sync->next_pts - sync->first_drop ) / 90,
753 sync->drop_count, sync->first_drop, sync->next_pts );
754 sync->first_drop = 0;
755 sync->drop_count = 0;
756 sync->next_pts = buf->start;
758 if ( buf->start - sync->next_pts >= (90 * 70) )
760 if ( buf->start - sync->next_pts > (90000LL * 60) )
762 // there's a gap of more than a minute between the last
763 // frame and this. assume we got a corrupted timestamp
764 // and just drop the next buf.
765 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
766 " start %lld, next %lld",
767 (int)((buf->start - sync->next_pts) / (90000*60)),
768 i, buf->start, sync->next_pts );
769 buf = hb_fifo_get( audio->priv.fifo_raw );
770 hb_buffer_close( &buf );
774 * there's a gap of at least 70ms between the last
775 * frame we processed & the next. Fill it with silence.
777 hb_log( "sync: adding %d ms of silence to audio %d"
778 " start %lld, next %lld",
779 (int)((buf->start - sync->next_pts) / 90),
780 i, buf->start, sync->next_pts );
781 InsertSilence( w, i, buf->start - sync->next_pts );
786 * When we get here we've taken care of all the dups and gaps in the
787 * audio stream and are ready to inject the next input frame into
790 buf = hb_fifo_get( audio->priv.fifo_raw );
791 OutputAudioFrame( job, audio, buf, sync, fifo, i );
795 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
797 hb_work_private_t * pv = w->private_data;
798 hb_job_t *job = pv->job;
799 hb_sync_audio_t *sync = &pv->sync_audio[i];
803 // to keep pass-thru and regular audio in sync we generate silence in
804 // AC3 frame-sized units. If the silence duration isn't an integer multiple
805 // of the AC3 frame duration we will truncate or round up depending on
806 // which minimizes the timing error.
807 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
808 sync->audio->config.in.samplerate;
809 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
811 while ( --frame_count >= 0 )
813 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
815 buf = hb_buffer_init( sync->ac3_size );
816 buf->start = sync->next_pts;
817 buf->stop = buf->start + frame_dur;
818 memcpy( buf->data, sync->ac3_buf, buf->size );
819 fifo = sync->audio->priv.fifo_out;
823 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
824 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
825 sync->audio->config.out.mixdown) );
826 buf->start = sync->next_pts;
827 buf->stop = buf->start + frame_dur;
828 memset( buf->data, 0, buf->size );
829 fifo = sync->audio->priv.fifo_sync;
831 OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
835 static void UpdateState( hb_work_object_t * w )
837 hb_work_private_t * pv = w->private_data;
840 if( !pv->count_frames )
842 pv->st_first = hb_get_date();
846 if( hb_get_date() > pv->st_dates[3] + 1000 )
848 memmove( &pv->st_dates[0], &pv->st_dates[1],
849 3 * sizeof( uint64_t ) );
850 memmove( &pv->st_counts[0], &pv->st_counts[1],
851 3 * sizeof( uint64_t ) );
852 pv->st_dates[3] = hb_get_date();
853 pv->st_counts[3] = pv->count_frames;
856 #define p state.param.working
857 state.state = HB_STATE_WORKING;
858 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
859 if( p.progress > 1.0 )
863 p.rate_cur = 1000.0 *
864 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
865 (float) ( pv->st_dates[3] - pv->st_dates[0] );
866 if( hb_get_date() > pv->st_first + 4000 )
869 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
870 (float) ( pv->st_dates[3] - pv->st_first );
871 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
873 p.hours = eta / 3600;
874 p.minutes = ( eta % 3600 ) / 60;
875 p.seconds = eta % 60;
886 hb_set_state( pv->job->h, &state );