1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.m0k.org/>.
5 It may be used under the terms of the GNU General Public License. */
9 #include "samplerate.h"
10 #include "ffmpeg/avcodec.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
23 int64_t next_start; /* start time of next output frame */
24 int64_t next_pts; /* start time of next input frame */
25 int64_t start_silence; /* if we're inserting silence, the time we started */
26 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
27 int drop_count; /* count of 'time went backwards' drops */
28 int inserting_silence;
40 struct hb_work_private_s
46 hb_subtitle_t * subtitle;
48 int64_t next_start; /* start time of next output frame */
49 int64_t next_pts; /* start time of next input frame */
50 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
51 int drop_count; /* count of 'time went backwards' drops */
55 hb_buffer_t * cur; /* The next picture to process */
58 hb_sync_audio_t sync_audio[8];
61 uint64_t st_counts[4];
66 /***********************************************************************
68 **********************************************************************/
69 static void InitAudio( hb_work_object_t * w, int i );
70 static int SyncVideo( hb_work_object_t * w );
71 static void SyncAudio( hb_work_object_t * w, int i );
72 static int NeedSilence( hb_work_object_t * w, hb_audio_t *, int i );
73 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
74 static void UpdateState( hb_work_object_t * w );
76 /***********************************************************************
78 ***********************************************************************
79 * Initialize the work object
80 **********************************************************************/
81 int syncInit( hb_work_object_t * w, hb_job_t * job )
83 hb_title_t * title = job->title;
84 hb_chapter_t * chapter;
87 hb_work_private_t * pv;
89 pv = calloc( 1, sizeof( hb_work_private_t ) );
93 pv->pts_offset = INT64_MIN;
96 /* Calculate how many video frames we are expecting */
98 for( i = job->chapter_start; i <= job->chapter_end; i++ )
100 chapter = hb_list_item( title->list_chapter, i - 1 );
101 duration += chapter->duration;
104 /* 1 second safety so we're sure we won't miss anything */
105 pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
107 hb_log( "sync: expecting %d video frames", pv->count_frames_max );
109 /* Initialize libsamplerate for every audio track we have */
110 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
115 /* Get subtitle info, if any */
116 pv->subtitle = hb_list_item( title->list_subtitle, 0 );
118 pv->video_sequence = 0;
123 /***********************************************************************
125 ***********************************************************************
127 **********************************************************************/
128 void syncClose( hb_work_object_t * w )
130 hb_work_private_t * pv = w->private_data;
131 hb_job_t * job = pv->job;
132 hb_title_t * title = job->title;
133 hb_audio_t * audio = NULL;
137 if( pv->cur ) hb_buffer_close( &pv->cur );
139 for( i = 0; i < hb_list_count( title->list_audio ); i++ )
141 if ( pv->sync_audio[i].start_silence )
143 hb_log( "sync: added %d ms of silence to audio %d",
144 (int)((pv->sync_audio[i].next_pts -
145 pv->sync_audio[i].start_silence) / 90), i );
148 audio = hb_list_item( title->list_audio, i );
149 if( audio->config.out.codec == HB_ACODEC_AC3 )
151 free( pv->sync_audio[i].ac3_buf );
155 src_delete( pv->sync_audio[i].state );
160 w->private_data = NULL;
163 /***********************************************************************
165 ***********************************************************************
166 * The root routine of this work abject
168 * The way this works is that we are syncing the audio to the PTS of
169 * the last video that we processed. That's why we skip the audio sync
170 * if we haven't got a valid PTS from the video yet.
172 **********************************************************************/
173 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
174 hb_buffer_t ** unused2 )
176 hb_work_private_t * pv = w->private_data;
179 /* If we ever got a video frame, handle audio now */
180 if( pv->pts_offset != INT64_MIN )
182 for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
189 return SyncVideo( w );
192 hb_work_object_t hb_sync =
201 static void InitAudio( hb_work_object_t * w, int i )
203 hb_work_private_t * pv = w->private_data;
204 hb_job_t * job = pv->job;
205 hb_title_t * title = job->title;
206 hb_sync_audio_t * sync;
208 sync = &pv->sync_audio[i];
209 sync->audio = hb_list_item( title->list_audio, i );
211 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
213 /* Have a silent AC-3 frame ready in case we have to fill a
219 codec = avcodec_find_encoder( CODEC_ID_AC3 );
220 c = avcodec_alloc_context();
222 c->bit_rate = sync->audio->config.in.bitrate;
223 c->sample_rate = sync->audio->config.in.samplerate;
224 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
226 if( avcodec_open( c, codec ) < 0 )
228 hb_log( "sync: avcodec_open failed" );
232 zeros = calloc( AC3_SAMPLES_PER_FRAME *
233 sizeof( short ) * c->channels, 1 );
234 sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
235 sync->audio->config.in.samplerate / 8;
236 sync->ac3_buf = malloc( sync->ac3_size );
238 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
239 zeros ) != sync->ac3_size )
241 hb_log( "sync: avcodec_encode_audio failed" );
250 /* Initialize libsamplerate */
252 sync->state = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
253 sync->data.end_of_input = 0;
257 /***********************************************************************
259 ***********************************************************************
261 **********************************************************************/
262 static int SyncVideo( hb_work_object_t * w )
264 hb_work_private_t * pv = w->private_data;
265 hb_buffer_t * cur, * next, * sub = NULL;
266 hb_job_t * job = pv->job;
273 if( hb_thread_has_exited( job->reader ) &&
274 !hb_fifo_size( job->fifo_mpeg2 ) &&
275 !hb_fifo_size( job->fifo_raw ) )
277 /* All video data has been processed already, we won't get
279 hb_log( "sync: got %d frames, %d expected",
280 pv->count_frames, pv->count_frames_max );
283 hb_buffer_t * buf_tmp;
285 // Drop an empty buffer into our output to ensure that things
286 // get flushed all the way out.
287 buf_tmp = hb_buffer_init(0); // Empty end buffer
288 hb_fifo_push( job->fifo_sync, buf_tmp );
293 if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
295 /* We haven't even got a frame yet */
300 /* At this point we have a frame to process. Let's check
301 1) if we will be able to push into the fifo ahead
302 2) if the next frame is there already, since we need it to
303 compute the duration of the current frame*/
304 while( !hb_fifo_is_full( job->fifo_sync ) &&
305 ( next = hb_fifo_see( job->fifo_raw ) ) )
307 hb_buffer_t * buf_tmp;
309 if( pv->pts_offset == INT64_MIN )
311 /* This is our first frame */
313 if ( cur->start != 0 )
316 * The first pts from a dvd should always be zero but
317 * can be non-zero with a transport or program stream since
318 * we're not guaranteed to start on an IDR frame. If we get
319 * a non-zero initial PTS extend its duration so it behaves
320 * as if it started at zero so that our audio timing will
323 hb_log( "sync: first pts is %lld", cur->start );
329 * since the first frame is always 0 and the upstream reader code
330 * is taking care of adjusting for pts discontinuities, we just have
331 * to deal with the next frame's start being in the past. This can
332 * happen when the PTS is adjusted after data loss but video frame
333 * reordering causes some frames with the old clock to appear after
334 * the clock change. This creates frames that overlap in time which
335 * looks to us like time going backward. The downstream muxing code
336 * can deal with overlaps of up to a frame time but anything larger
337 * we handle by dropping frames here.
339 if ( (int64_t)( next->start - pv->next_pts ) <= 0 )
341 if ( pv->first_drop == 0 )
343 pv->first_drop = next->start;
346 buf_tmp = hb_fifo_get( job->fifo_raw );
347 hb_buffer_close( &buf_tmp );
350 if ( pv->first_drop )
352 hb_log( "sync: video time didn't advance - dropped %d frames "
353 "(delta %d ms, current %lld, next %lld)",
354 pv->drop_count, (int)( pv->next_pts - pv->first_drop ) / 90,
355 pv->next_pts, pv->first_drop );
361 * Track the video sequence number localy so that we can sync the audio
362 * to it using the sequence number as well as the PTS.
364 pv->video_sequence = cur->sequence;
366 /* Look for a subtitle for this frame */
370 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
372 /* If two subtitles overlap, make the first one stop
373 when the second one starts */
374 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
375 if( sub2 && sub->stop > sub2->start )
376 sub->stop = sub2->start;
378 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
379 // sub, cur->sequence, sub->sequence);
381 if( sub->sequence > cur->sequence )
384 * The video is behind where we are, so wait until
385 * it catches up to the same reader point on the
386 * DVD. Then our PTS should be in the same region
393 if( sub->stop > cur->start ) {
395 * The stop time is in the future, so fall through
396 * and we'll deal with it in the next block of
403 * The subtitle is older than this picture, trash it
405 sub = hb_fifo_get( pv->subtitle->fifo_raw );
406 hb_buffer_close( &sub );
410 * There is a valid subtitle, is it time to display it?
414 if( sub->stop > sub->start)
417 * Normal subtitle which ends after it starts, check to
418 * see that the current video is between the start and end.
420 if( cur->start > sub->start &&
421 cur->start < sub->stop )
424 * We should be playing this, so leave the
427 * fall through to display
429 if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
432 * Subtitle is on for less than three seconds, extend
433 * the time that it is displayed to make it easier
434 * to read. Make it 3 seconds or until the next
435 * subtitle is displayed.
437 * This is in response to Indochine which only
438 * displays subs for 1 second - too fast to read.
440 sub->stop = sub->start + ( 3 * 90000 );
442 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
444 if( sub2 && sub->stop > sub2->start )
446 sub->stop = sub2->start;
453 * Defer until the play point is within the subtitle
461 * The end of the subtitle is less than the start, this is a
462 * sign of a PTS discontinuity.
464 if( sub->start > cur->start )
467 * we haven't reached the start time yet, or
468 * we have jumped backwards after having
469 * already started this subtitle.
471 if( cur->start < sub->stop )
474 * We have jumped backwards and so should
475 * continue displaying this subtitle.
477 * fall through to display.
483 * Defer until the play point is within the subtitle
489 * Play this subtitle as the start is greater than our
492 * fall through to display/
500 * Adjust the pts of the current frame so that it's contiguous
501 * with the previous frame. The start time of the current frame
502 * has to be the end time of the previous frame and the stop
503 * time has to be the start of the next frame. We don't
504 * make any adjustments to the source timestamps other than removing
505 * the clock offsets (which also removes pts discontinuities).
506 * This means we automatically encode at the source's frame rate.
507 * MP2 uses an implicit duration (frames end when the next frame
508 * starts) but more advanced containers like MP4 use an explicit
509 * duration. Since we're looking ahead one frame we set the
510 * explicit stop time from the start time of the next frame.
513 pv->cur = cur = hb_fifo_get( job->fifo_raw );
514 pv->next_pts = next->start;
515 int64_t duration = next->start - buf_tmp->start;
518 hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
519 duration, buf_tmp->start, next->start );
521 buf_tmp->start = pv->next_start;
522 pv->next_start += duration;
523 buf_tmp->stop = pv->next_start;
525 /* If we have a subtitle for this picture, copy it */
526 /* FIXME: we should avoid this memcpy */
529 buf_tmp->sub = hb_buffer_init( sub->size );
530 buf_tmp->sub->x = sub->x;
531 buf_tmp->sub->y = sub->y;
532 buf_tmp->sub->width = sub->width;
533 buf_tmp->sub->height = sub->height;
534 memcpy( buf_tmp->sub->data, sub->data, sub->size );
537 /* Push the frame to the renderer */
538 hb_fifo_push( job->fifo_sync, buf_tmp );
543 /* Make sure we won't get more frames then expected */
544 if( pv->count_frames >= pv->count_frames_max * 2)
546 hb_log( "sync: got too many frames (%d), exiting early", pv->count_frames );
549 // Drop an empty buffer into our output to ensure that things
550 // get flushed all the way out.
551 buf_tmp = hb_buffer_init(0); // Empty end buffer
552 hb_fifo_push( job->fifo_sync, buf_tmp );
561 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
562 hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
564 int64_t start = sync->next_start;
565 int64_t duration = buf->stop - buf->start;
567 duration > ( 90000 * AC3_SAMPLES_PER_FRAME ) / audio->config.out.samplerate )
569 hb_log("sync: audio %d weird duration %lld, start %lld, stop %lld, next %lld",
570 i, duration, buf->start, buf->stop, sync->next_pts);
573 duration = ( 90000 * AC3_SAMPLES_PER_FRAME ) / audio->config.out.samplerate;
574 buf->stop = buf->start + duration;
577 sync->next_pts += duration;
579 if( /* audio->rate == job->arate || This should work but doesn't */
580 audio->config.out.codec == HB_ACODEC_AC3 ||
581 audio->config.out.codec == HB_ACODEC_DCA )
584 * If we don't have to do sample rate conversion or this audio is AC3
585 * pass-thru just send the input buffer downstream after adjusting
586 * its timestamps to make the output stream continuous.
591 /* Not pass-thru - do sample rate conversion */
592 int count_in, count_out;
593 hb_buffer_t * buf_raw = buf;
594 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
597 count_in = buf_raw->size / channel_count;
598 count_out = ( buf_raw->stop - buf_raw->start ) * audio->config.out.samplerate / 90000;
600 sync->data.input_frames = count_in;
601 sync->data.output_frames = count_out;
602 sync->data.src_ratio = (double)count_out / (double)count_in;
604 buf = hb_buffer_init( count_out * channel_count );
605 sync->data.data_in = (float *) buf_raw->data;
606 sync->data.data_out = (float *) buf->data;
607 if( src_process( sync->state, &sync->data ) )
609 /* XXX If this happens, we're screwed */
610 hb_log( "sync: audio %d src_process failed", i );
612 hb_buffer_close( &buf_raw );
614 buf->size = sync->data.output_frames_gen * channel_count;
617 buf->stop = start + duration;
618 buf->frametype = HB_FRAME_AUDIO;
619 sync->next_start = start + duration;
620 hb_fifo_push( fifo, buf );
623 /***********************************************************************
625 ***********************************************************************
627 **********************************************************************/
628 static void SyncAudio( hb_work_object_t * w, int i )
630 hb_work_private_t * pv = w->private_data;
631 hb_job_t * job = pv->job;
632 hb_sync_audio_t * sync = &pv->sync_audio[i];
633 hb_audio_t * audio = sync->audio;
638 if( audio->config.out.codec == HB_ACODEC_AC3 )
640 fifo = audio->priv.fifo_out;
641 rate = audio->config.in.samplerate;
645 fifo = audio->priv.fifo_sync;
646 rate = audio->config.out.samplerate;
649 while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
651 if ( (int64_t)( buf->start - sync->next_pts ) < 0 )
654 * audio time went backwards by more than a frame time (this can
655 * happen when we reset the PTS because of lost data).
656 * Discard data that's in the past.
658 if ( sync->first_drop == 0 )
660 sync->first_drop = buf->start;
663 buf = hb_fifo_get( audio->priv.fifo_raw );
664 hb_buffer_close( &buf );
667 if ( sync->first_drop )
669 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
670 "(next %lld, current %lld)", i,
671 (int)( sync->next_pts - sync->first_drop ) / 90,
672 sync->drop_count, sync->first_drop, sync->next_pts );
673 sync->first_drop = 0;
674 sync->drop_count = 0;
677 if ( sync->inserting_silence && (int64_t)(buf->start - sync->next_pts) > 0 )
680 * if we're within one frame time of the amount of silence
681 * we need, insert just what we need otherwise insert a frame time.
683 int64_t framedur = buf->stop - buf->start;
684 if ( buf->start - sync->next_pts <= framedur )
686 InsertSilence( w, i, buf->start - sync->next_pts );
687 sync->inserting_silence = 0;
691 InsertSilence( w, i, framedur );
695 if ( buf->start - sync->next_pts >= (90 * 100) )
698 * there's a gap of at least 100ms between the last
699 * frame we processed & the next. Fill it with silence.
701 if ( ! sync->inserting_silence )
703 hb_log( "sync: adding %d ms of silence to audio %d"
704 " start %lld, next %lld",
705 (int)((buf->start - sync->next_pts) / 90),
706 i, buf->start, sync->next_pts );
707 sync->inserting_silence = 1;
709 InsertSilence( w, i, buf->stop - buf->start );
714 * When we get here we've taken care of all the dups and gaps in the
715 * audio stream and are ready to inject the next input frame into
718 buf = hb_fifo_get( audio->priv.fifo_raw );
719 OutputAudioFrame( job, audio, buf, sync, fifo, i );
722 if( NeedSilence( w, audio, i ) )
724 InsertSilence( w, i, (90000 * AC3_SAMPLES_PER_FRAME) / sync->audio->config.out.samplerate );
728 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio, int i )
730 hb_work_private_t * pv = w->private_data;
731 hb_job_t * job = pv->job;
732 hb_sync_audio_t * sync = &pv->sync_audio[i];
734 if( hb_fifo_size( audio->priv.fifo_in ) ||
735 hb_fifo_size( audio->priv.fifo_raw ) ||
736 hb_fifo_size( audio->priv.fifo_sync ) ||
737 hb_fifo_size( audio->priv.fifo_out ) )
739 /* We have some audio, we are fine */
743 /* No audio left in fifos */
745 if( hb_thread_has_exited( job->reader ) )
747 /* We might miss some audio to complete encoding and muxing
749 if ( sync->start_silence == 0 )
751 hb_log("sync: reader has exited, adding silence to audio %d", i);
752 sync->start_silence = sync->next_pts;
759 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
761 hb_work_private_t * pv = w->private_data;
762 hb_job_t *job = pv->job;
763 hb_sync_audio_t *sync = &pv->sync_audio[i];
766 if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
768 buf = hb_buffer_init( sync->ac3_size );
769 buf->start = sync->next_pts;
770 buf->stop = buf->start + duration;
771 memcpy( buf->data, sync->ac3_buf, buf->size );
772 OutputAudioFrame( job, sync->audio, buf, sync, sync->audio->priv.fifo_out, i );
776 buf = hb_buffer_init( duration * sizeof( float ) *
777 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown) );
778 buf->start = sync->next_pts;
779 buf->stop = buf->start + duration;
780 memset( buf->data, 0, buf->size );
781 OutputAudioFrame( job, sync->audio, buf, sync, sync->audio->priv.fifo_sync, i );
785 static void UpdateState( hb_work_object_t * w )
787 hb_work_private_t * pv = w->private_data;
790 if( !pv->count_frames )
792 pv->st_first = hb_get_date();
796 if( hb_get_date() > pv->st_dates[3] + 1000 )
798 memmove( &pv->st_dates[0], &pv->st_dates[1],
799 3 * sizeof( uint64_t ) );
800 memmove( &pv->st_counts[0], &pv->st_counts[1],
801 3 * sizeof( uint64_t ) );
802 pv->st_dates[3] = hb_get_date();
803 pv->st_counts[3] = pv->count_frames;
806 #define p state.param.working
807 state.state = HB_STATE_WORKING;
808 p.progress = (float) pv->count_frames / (float) pv->count_frames_max;
809 if( p.progress > 1.0 )
813 p.rate_cur = 1000.0 *
814 (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
815 (float) ( pv->st_dates[3] - pv->st_dates[0] );
816 if( hb_get_date() > pv->st_first + 4000 )
819 p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
820 (float) ( pv->st_dates[3] - pv->st_first );
821 eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
823 p.hours = eta / 3600;
824 p.minutes = ( eta % 3600 ) / 60;
825 p.seconds = eta % 60;
836 hb_set_state( pv->job->h, &state );