1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.m0k.org/>.
5 It may be used under the terms of the GNU General Public License. */
9 #include "samplerate.h"
10 #include "ffmpeg/avcodec.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
23 int64_t next_start; /* start time of next output frame */
24 int64_t next_pts; /* start time of next input frame */
25 int64_t start_silence; /* if we're inserting silence, the time we started */
26 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
27 int drop_count; /* count of 'time went backwards' drops */
28 int inserting_silence;
40 struct hb_work_private_s
46 hb_subtitle_t * subtitle;
48 int64_t next_start; /* start time of next output frame */
49 int64_t next_pts; /* start time of next input frame */
50 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
51 int drop_count; /* count of 'time went backwards' drops */
55 hb_buffer_t * cur; /* The next picture to process */
58 hb_sync_audio_t sync_audio[8];
61 uint64_t st_counts[4];
66 /***********************************************************************
68 **********************************************************************/
69 static void InitAudio( hb_work_object_t * w, int i );
70 static int SyncVideo( hb_work_object_t * w );
71 static void SyncAudio( hb_work_object_t * w, int i );
72 static int NeedSilence( hb_work_object_t * w, hb_audio_t *, int i );
73 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
74 static void UpdateState( hb_work_object_t * w );
76 /***********************************************************************
78 ***********************************************************************
79 * Initialize the work object
80 **********************************************************************/
81 int syncInit( hb_work_object_t * w, hb_job_t * job )
83 hb_title_t * title = job->title;
84 hb_chapter_t * chapter;
87 hb_work_private_t * pv;
89 pv = calloc( 1, sizeof( hb_work_private_t ) );
93 pv->pts_offset = INT64_MIN;
96 /* Calculate how many video frames we are expecting */
98 for( i = job->chapter_start; i <= job->chapter_end; i++ )
100 chapter = hb_list_item( title->list_chapter, i - 1 );
101 duration += chapter->duration;
104 /* 1 second safety so we're sure we won't miss anything */
105 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
107 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
109 /* Initialize libsamplerate for every audio track we have */
110 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
115 /* Get subtitle info, if any */
116 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
118 pv->video_sequence = 0;
123 /***********************************************************************
125 ***********************************************************************
127 **********************************************************************/
128 void syncClose( hb_work_object_t * w )
130 hb_work_private_t * pv = w->private_data;
131 hb_job_t * job = pv->job;
132 hb_title_t * title = job->title;
133 hb_audio_t * audio = NULL;
137 if( pv->cur ) hb_buffer_close( &pv->cur );
139 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
141 if ( pv->sync_audio[i].start_silence )
143 hb_log( "sync: added %d ms of silence to audio %d",
144 (int)((pv->sync_audio[i].next_pts -
145 pv->sync_audio[i].start_silence) / 90), i );
148 audio = hb_list_item( title->list_audio, i );
149 if( audio->config.out.codec == HB_ACODEC_AC3 )
151 free( pv->sync_audio[i].ac3_buf );
155 src_delete( pv->sync_audio[i].state );
160 w->private_data = NULL;
163 /***********************************************************************
165 ***********************************************************************
166 * The root routine of this work abject
168 * The way this works is that we are syncing the audio to the PTS of
169 * the last video that we processed. That's why we skip the audio sync
170 * if we haven't got a valid PTS from the video yet.
172 **********************************************************************/
173 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
174 hb_buffer_t ** unused2 )
176 hb_work_private_t * pv = w->private_data;
179 /* If we ever got a video frame, handle audio now */
180 if( pv->pts_offset != INT64_MIN )
182 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
189 return SyncVideo( w );
192 hb_work_object_t hb_sync =
201 static void InitAudio( hb_work_object_t * w, int i )
203 hb_work_private_t * pv = w->private_data;
204 hb_job_t * job = pv->job;
205 hb_title_t * title = job->title;
206 hb_sync_audio_t * sync;
208 sync = &pv->sync_audio[i];
209 sync->audio = hb_list_item( title->list_audio, i );
211 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
213 /* Have a silent AC-3 frame ready in case we have to fill a
219 codec = avcodec_find_encoder( CODEC_ID_AC3 );
220 c = avcodec_alloc_context();
222 c->bit_rate = sync->audio->config.in.bitrate;
223 c->sample_rate = sync->audio->config.in.samplerate;
224 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
226 if( avcodec_open( c, codec ) < 0 )
228 hb_log( "sync: avcodec_open failed" );
232 zeros = calloc( AC3_SAMPLES_PER_FRAME *
233 sizeof( short ) * c->channels, 1 );
234 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
235 sync->audio->config.in.samplerate / 8;
236 sync->ac3_buf = malloc( sync->ac3_size );
238 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
239 zeros ) != sync->ac3_size )
241 hb_log( "sync: avcodec_encode_audio failed" );
250 /* Initialize libsamplerate */
252 sync->state = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
253 sync->data.end_of_input = 0;
257 /***********************************************************************
259 ***********************************************************************
261 **********************************************************************/
262 static int SyncVideo( hb_work_object_t * w )
264 hb_work_private_t * pv = w->private_data;
265 hb_buffer_t * cur, * next, * sub = NULL;
266 hb_job_t * job = pv->job;
273 if( hb_thread_has_exited( job->reader ) &&
274 !hb_fifo_size( job->fifo_mpeg2 ) &&
275 !hb_fifo_size( job->fifo_raw ) )
277 /* All video data has been processed already, we won't get
279 hb_log( "sync: got %d frames, %d expected",
280 pv->count_frames, pv->count_frames_max );
283 hb_buffer_t * buf_tmp;
285 // Drop an empty buffer into our output to ensure that things
286 // get flushed all the way out.
287 buf_tmp = hb_buffer_init(0); // Empty end buffer
288 hb_fifo_push( job->fifo_sync, buf_tmp );
293 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
295 /* We haven't even got a frame yet */
300 /* At this point we have a frame to process. Let's check
301 1) if we will be able to push into the fifo ahead
302 2) if the next frame is there already, since we need it to
303 compute the duration of the current frame*/
304 while( !hb_fifo_is_full( job->fifo_sync ) &&
305 ( next = hb_fifo_see( job->fifo_raw ) ) )
307 hb_buffer_t * buf_tmp;
309 if( pv->pts_offset == INT64_MIN )
311 /* This is our first frame */
313 if ( cur->start != 0 )
316 * The first pts from a dvd should always be zero but
317 * can be non-zero with a transport or program stream since
318 * we're not guaranteed to start on an IDR frame. If we get
319 * a non-zero initial PTS extend its duration so it behaves
320 * as if it started at zero so that our audio timing will
323 hb_log( "sync: first pts is %lld", cur->start );
329 * since the first frame is always 0 and the upstream reader code
330 * is taking care of adjusting for pts discontinuities, we just have
331 * to deal with the next frame's start being in the past. This can
332 * happen when the PTS is adjusted after data loss but video frame
333 * reordering causes some frames with the old clock to appear after
334 * the clock change. This creates frames that overlap in time which
335 * looks to us like time going backward. The downstream muxing code
336 * can deal with overlaps of up to a frame time but anything larger
337 * we handle by dropping frames here.
339 if ( pv->next_pts - next->start > 1000 )
341 if ( pv->first_drop == 0 )
343 pv->first_drop = next->start;
346 buf_tmp = hb_fifo_get( job->fifo_raw );
347 hb_buffer_close( &buf_tmp );
350 if ( pv->first_drop )
352 hb_log( "sync: video time went backwards %d ms, dropped %d frames "
353 "(frame %lld, expected %lld)",
354 (int)( pv->next_pts - pv->first_drop ) / 90, pv->drop_count,
355 pv->first_drop, pv->next_pts );
361 * Track the video sequence number localy so that we can sync the audio
362 * to it using the sequence number as well as the PTS.
364 pv->video_sequence = cur->sequence;
366 /* Look for a subtitle for this frame */
370 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
372 /* If two subtitles overlap, make the first one stop
373 when the second one starts */
374 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
375 if( sub2 && sub->stop > sub2->start )
376 sub->stop = sub2->start;
378 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
379 // sub, cur->sequence, sub->sequence);
381 if( sub->sequence > cur->sequence )
384 * The video is behind where we are, so wait until
385 * it catches up to the same reader point on the
386 * DVD. Then our PTS should be in the same region
393 if( sub->stop > cur->start ) {
395 * The stop time is in the future, so fall through
396 * and we'll deal with it in the next block of
403 * The subtitle is older than this picture, trash it
405 sub = hb_fifo_get( pv->subtitle->fifo_raw );
406 hb_buffer_close( &sub );
410 * There is a valid subtitle, is it time to display it?
414 if( sub->stop > sub->start)
417 * Normal subtitle which ends after it starts, check to
418 * see that the current video is between the start and end.
420 if( cur->start > sub->start &&
421 cur->start < sub->stop )
424 * We should be playing this, so leave the
427 * fall through to display
429 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
432 * Subtitle is on for less than three seconds, extend
433 * the time that it is displayed to make it easier
434 * to read. Make it 3 seconds or until the next
435 * subtitle is displayed.
437 * This is in response to Indochine which only
438 * displays subs for 1 second - too fast to read.
440 sub->stop = sub->start + ( 3 * 90000 );
442 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
444 if( sub2 && sub->stop > sub2->start )
446 sub->stop = sub2->start;
453 * Defer until the play point is within the subtitle
461 * The end of the subtitle is less than the start, this is a
462 * sign of a PTS discontinuity.
464 if( sub->start > cur->start )
467 * we haven't reached the start time yet, or
468 * we have jumped backwards after having
469 * already started this subtitle.
471 if( cur->start < sub->stop )
474 * We have jumped backwards and so should
475 * continue displaying this subtitle.
477 * fall through to display.
483 * Defer until the play point is within the subtitle
489 * Play this subtitle as the start is greater than our
492 * fall through to display/
500 * Adjust the pts of the current frame so that it's contiguous
501 * with the previous frame. The start time of the current frame
502 * has to be the end time of the previous frame and the stop
503 * time has to be the start of the next frame. We don't
504 * make any adjustments to the source timestamps other than removing
505 * the clock offsets (which also removes pts discontinuities).
506 * This means we automatically encode at the source's frame rate.
507 * MP2 uses an implicit duration (frames end when the next frame
508 * starts) but more advanced containers like MP4 use an explicit
509 * duration. Since we're looking ahead one frame we set the
510 * explicit stop time from the start time of the next frame.
513 pv->cur = cur = hb_fifo_get( job->fifo_raw );
514 pv->next_pts = next->start;
515 int64_t duration = next->start - buf_tmp->start;
516 buf_tmp->start = pv->next_start;
517 pv->next_start += duration;
518 buf_tmp->stop = pv->next_start;
520 /* If we have a subtitle for this picture, copy it */
521 /* FIXME: we should avoid this memcpy */
524 buf_tmp->sub = hb_buffer_init( sub->size );
525 buf_tmp->sub->x = sub->x;
526 buf_tmp->sub->y = sub->y;
527 buf_tmp->sub->width = sub->width;
528 buf_tmp->sub->height = sub->height;
529 memcpy( buf_tmp->sub->data, sub->data, sub->size );
532 /* Push the frame to the renderer */
533 hb_fifo_push( job->fifo_sync, buf_tmp );
538 /* Make sure we won't get more frames then expected */
539 if( pv->count_frames >= pv->count_frames_max * 2)
541 hb_log( "sync: got too many frames (%d), exiting early", pv->count_frames );
544 // Drop an empty buffer into our output to ensure that things
545 // get flushed all the way out.
546 buf_tmp = hb_buffer_init(0); // Empty end buffer
547 hb_fifo_push( job->fifo_sync, buf_tmp );
556 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
557 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
559 int64_t start = sync->next_start;
560 int64_t duration = buf->stop - buf->start;
562 duration > ( 90000 * AC3_SAMPLES_PER_FRAME ) / audio->config.out.samplerate )
564 hb_log("sync: audio %d weird duration %lld, start %lld, stop %lld, next %lld",
565 i, duration, buf->start, buf->stop, sync->next_pts);
568 duration = ( 90000 * AC3_SAMPLES_PER_FRAME ) / audio->config.out.samplerate;
569 buf->stop = buf->start + duration;
572 sync->next_pts += duration;
574 if( /* audio->rate == job->arate || This should work but doesn't */
575 audio->config.out.codec == HB_ACODEC_AC3 ||
576 audio->config.out.codec == HB_ACODEC_DCA )
579 * If we don't have to do sample rate conversion or this audio is AC3
580 * pass-thru just send the input buffer downstream after adjusting
581 * its timestamps to make the output stream continuous.
586 /* Not pass-thru - do sample rate conversion */
587 int count_in, count_out;
588 hb_buffer_t * buf_raw = buf;
589 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
592 count_in = buf_raw->size / channel_count;
593 count_out = ( buf_raw->stop - buf_raw->start ) * audio->config.out.samplerate / 90000;
595 sync->data.input_frames = count_in;
596 sync->data.output_frames = count_out;
597 sync->data.src_ratio = (double)count_out / (double)count_in;
599 buf = hb_buffer_init( count_out * channel_count );
600 sync->data.data_in = (float *) buf_raw->data;
601 sync->data.data_out = (float *) buf->data;
602 if( src_process( sync->state, &sync->data ) )
604 /* XXX If this happens, we're screwed */
605 hb_log( "sync: audio %d src_process failed", i );
607 hb_buffer_close( &buf_raw );
609 buf->size = sync->data.output_frames_gen * channel_count;
612 buf->stop = start + duration;
613 buf->frametype = HB_FRAME_AUDIO;
614 sync->next_start = start + duration;
615 hb_fifo_push( fifo, buf );
618 /***********************************************************************
620 ***********************************************************************
622 **********************************************************************/
623 static void SyncAudio( hb_work_object_t * w, int i )
625 hb_work_private_t * pv = w->private_data;
626 hb_job_t * job = pv->job;
627 hb_sync_audio_t * sync = &pv->sync_audio[i];
628 hb_audio_t * audio = sync->audio;
633 if( audio->config.out.codec == HB_ACODEC_AC3 )
635 fifo = audio->priv.fifo_out;
636 rate = audio->config.in.samplerate;
640 fifo = audio->priv.fifo_sync;
641 rate = audio->config.out.samplerate;
644 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
646 if ( sync->next_pts - buf->start > 500 )
649 * audio time went backwards by more than a frame time (this can
650 * happen when we reset the PTS because of lost data).
651 * Discard data that's in the past.
653 if ( sync->first_drop == 0 )
655 sync->first_drop = buf->start;
658 buf = hb_fifo_get( audio->priv.fifo_raw );
659 hb_buffer_close( &buf );
662 if ( sync->first_drop )
664 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
665 "(frame %lld, expected %lld)", i,
666 (int)( sync->next_pts - sync->first_drop ) / 90,
667 sync->drop_count, sync->first_drop, sync->next_pts );
668 sync->first_drop = 0;
669 sync->drop_count = 0;
672 if ( sync->inserting_silence && buf->start - sync->next_pts > 0 )
675 * if we're within one frame time of the amount of silence
676 * we need, insert just what we need otherwise insert a frame time.
678 int64_t framedur = buf->stop - buf->start;
679 if ( buf->start - sync->next_pts <= framedur )
681 InsertSilence( w, i, buf->start - sync->next_pts );
682 sync->inserting_silence = 0;
686 InsertSilence( w, i, framedur );
690 if ( buf->start - sync->next_pts >= (90 * 100) )
693 * there's a gap of at least 100ms between the last
694 * frame we processed & the next. Fill it with silence.
696 if ( ! sync->inserting_silence )
698 hb_log( "sync: adding %d ms of silence to audio %d"
699 " start %lld, next %lld",
700 (int)((buf->start - sync->next_pts) / 90),
701 i, buf->start, sync->next_pts );
702 sync->inserting_silence = 1;
704 InsertSilence( w, i, buf->stop - buf->start );
709 * When we get here we've taken care of all the dups and gaps in the
710 * audio stream and are ready to inject the next input frame into
713 buf = hb_fifo_get( audio->priv.fifo_raw );
714 OutputAudioFrame( job, audio, buf, sync, fifo, i );
717 if( NeedSilence( w, audio, i ) )
719 InsertSilence( w, i, (90000 * AC3_SAMPLES_PER_FRAME) / sync->audio->config.out.samplerate );
723 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio, int i )
725 hb_work_private_t * pv = w->private_data;
726 hb_job_t * job = pv->job;
727 hb_sync_audio_t * sync = &pv->sync_audio[i];
729 if( hb_fifo_size( audio->priv.fifo_in ) ||
730 hb_fifo_size( audio->priv.fifo_raw ) ||
731 hb_fifo_size( audio->priv.fifo_sync ) ||
732 hb_fifo_size( audio->priv.fifo_out ) )
734 /* We have some audio, we are fine */
738 /* No audio left in fifos */
740 if( hb_thread_has_exited( job->reader ) )
742 /* We might miss some audio to complete encoding and muxing
744 if ( sync->start_silence == 0 )
746 hb_log("sync: reader has exited, adding silence to audio %d", i);
747 sync->start_silence = sync->next_pts;
754 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
756 hb_work_private_t * pv = w->private_data;
757 hb_job_t *job = pv->job;
758 hb_sync_audio_t *sync = &pv->sync_audio[i];
761 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
763 buf = hb_buffer_init( sync->ac3_size );
764 buf->start = sync->next_pts;
765 buf->stop = buf->start + duration;
766 memcpy( buf->data, sync->ac3_buf, buf->size );
767 OutputAudioFrame( job, sync->audio, buf, sync, sync->audio->priv.fifo_out, i );
771 buf = hb_buffer_init( duration * sizeof( float ) *
772 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown) );
773 buf->start = sync->next_pts;
774 buf->stop = buf->start + duration;
775 memset( buf->data, 0, buf->size );
776 OutputAudioFrame( job, sync->audio, buf, sync, sync->audio->priv.fifo_sync, i );
780 static void UpdateState( hb_work_object_t * w )
782 hb_work_private_t * pv = w->private_data;
785 if( !pv->count_frames )
787 pv->st_first = hb_get_date();
791 if( hb_get_date() > pv->st_dates[3] + 1000 )
793 memmove( &pv->st_dates[0], &pv->st_dates[1],
794 3 * sizeof( uint64_t ) );
795 memmove( &pv->st_counts[0], &pv->st_counts[1],
796 3 * sizeof( uint64_t ) );
797 pv->st_dates[3] = hb_get_date();
798 pv->st_counts[3] = pv->count_frames;
801 #define p state.param.working
802 state.state = HB_STATE_WORKING;
803 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
804 if( p.progress > 1.0 )
808 p.rate_cur = 1000.0 *
809 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
810 (float) ( pv->st_dates[3] - pv->st_dates[0] );
811 if( hb_get_date() > pv->st_first + 4000 )
814 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
815 (float) ( pv->st_dates[3] - pv->st_first );
816 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
818 p.hours = eta / 3600;
819 p.minutes = ( eta % 3600 ) / 60;
820 p.seconds = eta % 60;
831 hb_set_state( pv->job->h, &state );