1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
22 int ref; /* Reference count to tell us when it's unused */
24 int64_t audio_passthru_slip;
25 int64_t video_pts_slip;
28 /* Frame based point-to-point support */
29 int64_t audio_pts_thresh;
31 hb_cond_t * next_frame;
39 int64_t next_start; /* start time of next output frame */
40 int64_t next_pts; /* start time of next input frame */
41 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
42 int drop_count; /* count of 'time went backwards' drops */
58 int64_t next_start; /* start time of next output frame */
59 int64_t next_pts; /* start time of next input frame */
60 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
61 int drop_count; /* count of 'time went backwards' drops */
62 int drops; /* frames dropped to make a cbr video stream */
63 int dups; /* frames duplicated to make a cbr video stream */
66 int chap_mark; /* to propagate chapter mark across a drop */
67 hb_buffer_t * cur; /* The next picture to process */
70 uint64_t st_counts[4];
75 struct hb_work_private_s
78 hb_sync_common_t * common;
81 hb_sync_video_t video;
82 hb_sync_audio_t audio;
86 /***********************************************************************
88 **********************************************************************/
89 static void getPtsOffset( hb_work_object_t * w );
90 static int checkPtsOffset( hb_work_object_t * w );
91 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
92 static void InsertSilence( hb_work_object_t * w, int64_t d );
93 static void UpdateState( hb_work_object_t * w );
94 static void UpdateSearchState( hb_work_object_t * w, int64_t start );
95 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
96 hb_sync_audio_t *sync );
98 /***********************************************************************
100 ***********************************************************************
101 * Initialize the work object
102 **********************************************************************/
103 hb_work_object_t * hb_sync_init( hb_job_t * job )
105 hb_title_t * title = job->title;
106 hb_chapter_t * chapter;
109 hb_work_private_t * pv;
110 hb_sync_video_t * sync;
111 hb_work_object_t * w;
112 hb_work_object_t * ret = NULL;
114 pv = calloc( 1, sizeof( hb_work_private_t ) );
115 sync = &pv->type.video;
116 pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
118 pv->common->mutex = hb_lock_init();
119 pv->common->audio_pts_thresh = 0;
120 pv->common->next_frame = hb_cond_init();
121 pv->common->pts_count = 1;
122 if ( job->frame_to_start || job->pts_to_start )
124 pv->common->start_found = 0;
128 pv->common->start_found = 1;
131 ret = w = hb_get_work( WORK_SYNC_VIDEO );
132 w->private_data = pv;
133 w->fifo_in = job->fifo_raw;
134 w->fifo_out = job->fifo_sync;
137 pv->common->pts_offset = INT64_MIN;
138 sync->first_frame = 1;
142 /* We already have an accurate frame count from pass 1 */
143 hb_interjob_t * interjob = hb_interjob_get( job->h );
144 sync->count_frames_max = interjob->frame_count;
148 /* Calculate how many video frames we are expecting */
149 if ( job->pts_to_stop )
151 duration = job->pts_to_stop + 90000;
153 else if( job->frame_to_stop )
155 /* Set the duration to a rough estimate */
156 duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
161 for( i = job->chapter_start; i <= job->chapter_end; i++ )
163 chapter = hb_list_item( title->list_chapter, i - 1 );
164 duration += chapter->duration;
167 /* 1 second safety so we're sure we won't miss anything */
169 sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
172 hb_log( "sync: expecting %d video frames", sync->count_frames_max );
174 /* Initialize libsamplerate for every audio track we have */
175 if ( ! job->indepth_scan )
177 for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
179 InitAudio( job, pv->common, i );
182 pv->common->first_pts = malloc( sizeof(int64_t) * pv->common->pts_count );
183 for ( i = 0; i < pv->common->pts_count; i++ )
184 pv->common->first_pts[i] = INT64_MAX;
189 /***********************************************************************
191 ***********************************************************************
193 **********************************************************************/
194 void syncVideoClose( hb_work_object_t * w )
196 hb_work_private_t * pv = w->private_data;
197 hb_job_t * job = pv->job;
198 hb_sync_video_t * sync = &pv->type.video;
200 // Wake up audio sync if it's still waiting on condition.
201 pv->common->pts_offset = 0;
202 pv->common->start_found = 1;
203 hb_cond_broadcast( pv->common->next_frame );
207 hb_buffer_close( &sync->cur );
210 hb_log( "sync: got %d frames, %d expected",
211 pv->common->count_frames, sync->count_frames_max );
213 /* save data for second pass */
216 /* Preserve frame count for better accuracy in pass 2 */
217 hb_interjob_t * interjob = hb_interjob_get( job->h );
218 interjob->frame_count = pv->common->count_frames;
219 interjob->last_job = job->sequence_id;
220 interjob->total_time = sync->next_start;
223 if (sync->drops || sync->dups )
225 hb_log( "sync: %d frames dropped, %d duplicated",
226 sync->drops, sync->dups );
229 hb_lock( pv->common->mutex );
230 if ( --pv->common->ref == 0 )
232 hb_unlock( pv->common->mutex );
233 hb_lock_close( &pv->common->mutex );
238 hb_unlock( pv->common->mutex );
242 w->private_data = NULL;
245 /***********************************************************************
247 ***********************************************************************
249 **********************************************************************/
250 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
251 hb_buffer_t ** buf_out )
253 hb_buffer_t * cur, * next, * sub = NULL;
254 hb_work_private_t * pv = w->private_data;
255 hb_job_t * job = pv->job;
256 hb_subtitle_t * subtitle;
257 hb_sync_video_t * sync = &pv->type.video;
264 /* Wait for start of point-to-point encoding */
265 if( !pv->common->start_found )
267 hb_sync_video_t * sync = &pv->type.video;
269 if( next->size == 0 )
272 pv->common->start_found = 1;
273 hb_cond_broadcast( pv->common->next_frame );
276 * Push through any subtitle EOFs in case they
277 * were not synced through.
279 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
281 subtitle = hb_list_item( job->list_subtitle, i );
282 if( subtitle->config.dest == PASSTHRUSUB )
284 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
289 if ( pv->common->count_frames < job->frame_to_start ||
290 next->start < job->pts_to_start )
292 // Flush any subtitles that have pts prior to the
294 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
296 subtitle = hb_list_item( job->list_subtitle, i );
297 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
299 if ( sub->start > next->start )
301 sub = hb_fifo_get( subtitle->fifo_raw );
302 hb_buffer_close( &sub );
305 hb_lock( pv->common->mutex );
306 // Tell the audio threads what must be dropped
307 pv->common->audio_pts_thresh = next->start;
308 hb_cond_broadcast( pv->common->next_frame );
309 hb_unlock( pv->common->mutex );
311 UpdateSearchState( w, next->start );
312 hb_buffer_close( &next );
316 hb_lock( pv->common->mutex );
317 pv->common->start_found = 1;
318 pv->common->count_frames = 0;
319 hb_cond_broadcast( pv->common->next_frame );
320 hb_unlock( pv->common->mutex );
324 /* Wait till we can determine the initial pts of all streams */
325 if( pv->common->pts_offset == INT64_MIN )
327 pv->common->first_pts[0] = next->start;
328 hb_lock( pv->common->mutex );
329 while( pv->common->pts_offset == INT64_MIN )
331 // Full fifos will make us wait forever, so get the
332 // pts offset from the available streams if full
333 if ( hb_fifo_is_full( job->fifo_raw ) )
336 hb_cond_broadcast( pv->common->next_frame );
338 else if ( checkPtsOffset( w ) )
339 hb_cond_broadcast( pv->common->next_frame );
341 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
343 hb_unlock( pv->common->mutex );
349 if( sync->cur->size == 0 )
351 /* we got an end-of-stream as our first video packet?
352 * Feed it downstream & signal that we're done.
354 *buf_out = hb_buffer_init( 0 );
356 pv->common->start_found = 1;
357 hb_cond_broadcast( pv->common->next_frame );
360 * Push through any subtitle EOFs in case they
361 * were not synced through.
363 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
365 subtitle = hb_list_item( job->list_subtitle, i );
366 if( subtitle->config.dest == PASSTHRUSUB )
368 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
376 /* At this point we have a frame to process. Let's check
377 1) if we will be able to push into the fifo ahead
378 2) if the next frame is there already, since we need it to
379 compute the duration of the current frame*/
380 if( next->size == 0 )
382 hb_buffer_close( &next );
384 cur->start = sync->next_start;
385 cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
387 /* Push the frame to the renderer */
388 hb_fifo_push( job->fifo_sync, cur );
391 /* we got an end-of-stream. Feed it downstream & signal that
392 * we're done. Note that this means we drop the final frame of
393 * video (we don't know its duration). On DVDs the final frame
394 * is often strange and dropping it seems to be a good idea. */
395 *buf_out = hb_buffer_init( 0 );
398 * Push through any subtitle EOFs in case they were not synced through.
400 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
402 subtitle = hb_list_item( job->list_subtitle, i );
403 if( subtitle->config.dest == PASSTHRUSUB )
405 if( subtitle->source == VOBSUB )
406 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
408 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
411 pv->common->start_found = 1;
412 hb_cond_broadcast( pv->common->next_frame );
416 /* Check for end of point-to-point frame encoding */
417 if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
419 // Drop an empty buffer into our output to ensure that things
420 // get flushed all the way out.
421 hb_buffer_close( &sync->cur );
422 hb_buffer_close( &next );
423 *buf_out = hb_buffer_init( 0 );
424 hb_log( "sync: reached %d frames, exiting early",
425 pv->common->count_frames );
429 /* Check for end of point-to-point pts encoding */
430 if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
432 // Drop an empty buffer into our output to ensure that things
433 // get flushed all the way out.
434 hb_log( "sync: reached pts %"PRId64", exiting early",
436 hb_buffer_close( &sync->cur );
437 hb_buffer_close( &next );
438 *buf_out = hb_buffer_init( 0 );
442 if( sync->first_frame )
444 /* This is our first frame */
445 if ( cur->start > pv->common->pts_offset )
448 * The first pts from a dvd should always be zero but
449 * can be non-zero with a transport or program stream since
450 * we're not guaranteed to start on an IDR frame. If we get
451 * a non-zero initial PTS extend its duration so it behaves
452 * as if it started at zero so that our audio timing will
455 hb_log( "sync: first pts is %"PRId64, cur->start );
456 cur->start = pv->common->pts_offset;
458 sync->first_frame = 0;
462 * since the first frame is always 0 and the upstream reader code
463 * is taking care of adjusting for pts discontinuities, we just have
464 * to deal with the next frame's start being in the past. This can
465 * happen when the PTS is adjusted after data loss but video frame
466 * reordering causes some frames with the old clock to appear after
467 * the clock change. This creates frames that overlap in time which
468 * looks to us like time going backward. The downstream muxing code
469 * can deal with overlaps of up to a frame time but anything larger
470 * we handle by dropping frames here.
472 hb_lock( pv->common->mutex );
473 if ( (int64_t)( next->start - pv->common->video_pts_slip - cur->start ) <= 0 )
475 if ( sync->first_drop == 0 )
477 sync->first_drop = next->start;
480 if (next->start - cur->start > 0)
482 sync->pts_skip += next->start - cur->start;
483 pv->common->video_pts_slip -= next->start - cur->start;
485 hb_unlock( pv->common->mutex );
486 if ( next->new_chap )
488 // don't drop a chapter mark when we drop the buffer
489 sync->chap_mark = next->new_chap;
491 hb_buffer_close( &next );
494 hb_unlock( pv->common->mutex );
495 if ( sync->first_drop )
497 hb_log( "sync: video time didn't advance - dropped %d frames "
498 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
499 sync->drop_count, (int)( cur->start - sync->first_drop ) / 90,
500 cur->start, next->start, (int)( next->start - cur->start ) );
501 sync->first_drop = 0;
502 sync->drop_count = 0;
506 * Track the video sequence number localy so that we can sync the audio
507 * to it using the sequence number as well as the PTS.
509 sync->video_sequence = cur->sequence;
512 * Look for a subtitle for this frame.
514 * If found then it will be tagged onto a video buffer of the correct time and
515 * sent in to the render pipeline. This only needs to be done for VOBSUBs which
516 * get rendered, other types of subtitles can just sit in their raw_queue until
517 * delt with at muxing.
519 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
521 subtitle = hb_list_item( job->list_subtitle, i );
524 * Rewrite timestamps on subtitles that need it (on raw queue).
526 if( subtitle->source == CC608SUB ||
527 subtitle->source == CC708SUB ||
528 subtitle->source == SRTSUB )
531 * Rewrite timestamps on subtitles that came from Closed Captions
532 * since they are using the MPEG2 timestamps.
534 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
537 * Rewrite the timestamps as and when the video
538 * (cur->start) reaches the same timestamp as a
539 * closed caption (sub->start).
541 * What about discontinuity boundaries - not delt
544 * Bypass the sync fifo altogether.
548 sub = hb_fifo_get( subtitle->fifo_raw );
549 hb_fifo_push( subtitle->fifo_out, sub );
554 * Sync the subtitles to the incoming video, and use
555 * the matching converted video timestamp.
557 * Note that it doesn't appear that we need to convert
558 * timestamps, I guess that they were already correct,
559 * so just push them through for rendering.
562 if( sub->start < cur->start )
564 sub = hb_fifo_get( subtitle->fifo_raw );
565 hb_fifo_push( subtitle->fifo_out, sub );
574 if( subtitle->source == VOBSUB )
577 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
582 * EOF, pass it through immediately.
587 /* If two subtitles overlap, make the first one stop
588 when the second one starts */
589 sub2 = hb_fifo_see2( subtitle->fifo_raw );
590 if( sub2 && sub->stop > sub2->start )
592 sub->stop = sub2->start;
595 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
596 // sub, cur->sequence, sub->sequence);
598 if( sub->sequence > cur->sequence )
601 * The video is behind where we are, so wait until
602 * it catches up to the same reader point on the
603 * DVD. Then our PTS should be in the same region
610 if( sub->stop > cur->start ) {
612 * The stop time is in the future, so fall through
613 * and we'll deal with it in the next block of
618 * There is a valid subtitle, is it time to display it?
620 if( sub->stop > sub->start)
623 * Normal subtitle which ends after it starts,
624 * check to see that the current video is between
627 if( cur->start > sub->start &&
628 cur->start < sub->stop )
631 * We should be playing this, so leave the
634 * fall through to display
636 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
639 * Subtitle is on for less than three
640 * seconds, extend the time that it is
641 * displayed to make it easier to read.
642 * Make it 3 seconds or until the next
643 * subtitle is displayed.
645 * This is in response to Indochine which
646 * only displays subs for 1 second -
649 sub->stop = sub->start + ( 2 * 90000 );
651 sub2 = hb_fifo_see2( subtitle->fifo_raw );
653 if( sub2 && sub->stop > sub2->start )
655 sub->stop = sub2->start;
662 * Defer until the play point is within
671 * The end of the subtitle is less than the start,
672 * this is a sign of a PTS discontinuity.
674 if( sub->start > cur->start )
677 * we haven't reached the start time yet, or
678 * we have jumped backwards after having
679 * already started this subtitle.
681 if( cur->start < sub->stop )
684 * We have jumped backwards and so should
685 * continue displaying this subtitle.
687 * fall through to display.
693 * Defer until the play point is
694 * within the subtitle
700 * Play this subtitle as the start is
701 * greater than our video point.
703 * fall through to display/
713 * The subtitle is older than this picture, trash it
715 sub = hb_fifo_get( subtitle->fifo_raw );
716 hb_buffer_close( &sub );
720 /* If we have a subtitle for this picture, copy it */
721 /* FIXME: we should avoid this memcpy */
726 if( subtitle->config.dest == RENDERSUB )
728 if ( cur->sub == NULL )
731 * Tack onto the video buffer for rendering
733 cur->sub = hb_buffer_init( sub->size );
734 cur->sub->x = sub->x;
735 cur->sub->y = sub->y;
736 cur->sub->width = sub->width;
737 cur->sub->height = sub->height;
738 memcpy( cur->sub->data, sub->data, sub->size );
742 * Pass-Through, pop it off of the raw queue,
744 sub = hb_fifo_get( subtitle->fifo_raw );
745 hb_fifo_push( subtitle->fifo_sync, sub );
749 * EOF - consume for rendered, else pass through
751 if( subtitle->config.dest == RENDERSUB )
753 sub = hb_fifo_get( subtitle->fifo_raw );
754 hb_buffer_close( &sub );
756 sub = hb_fifo_get( subtitle->fifo_raw );
757 hb_fifo_push( subtitle->fifo_sync, sub );
765 * Adjust the pts of the current frame so that it's contiguous
766 * with the previous frame. The start time of the current frame
767 * has to be the end time of the previous frame and the stop
768 * time has to be the start of the next frame. We don't
769 * make any adjustments to the source timestamps other than removing
770 * the clock offsets (which also removes pts discontinuities).
771 * This means we automatically encode at the source's frame rate.
772 * MP2 uses an implicit duration (frames end when the next frame
773 * starts) but more advanced containers like MP4 use an explicit
774 * duration. Since we're looking ahead one frame we set the
775 * explicit stop time from the start time of the next frame.
778 sync->cur = cur = next;
780 sync->next_pts = cur->start;
781 int64_t duration = cur->start - sync->pts_skip - (*buf_out)->start;
785 hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
786 duration, (*buf_out)->start, next->start );
789 (*buf_out)->start = sync->next_start;
790 sync->next_start += duration;
791 (*buf_out)->stop = sync->next_start;
793 if ( sync->chap_mark )
795 // we have a pending chapter mark from a recent drop - put it on this
796 // buffer (this may make it one frame late but we can't do any better).
797 (*buf_out)->new_chap = sync->chap_mark;
807 // sync*Init does nothing because sync has a special initializer
808 // that takes care of initializing video and all audio tracks
809 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
814 hb_work_object_t hb_sync_video =
817 "Video Synchronization",
823 /***********************************************************************
825 ***********************************************************************
827 **********************************************************************/
828 void syncAudioClose( hb_work_object_t * w )
830 hb_work_private_t * pv = w->private_data;
831 hb_sync_audio_t * sync = &pv->type.audio;
833 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
835 free( sync->ac3_buf );
839 src_delete( sync->state );
842 hb_lock( pv->common->mutex );
843 if ( --pv->common->ref == 0 )
845 hb_unlock( pv->common->mutex );
846 hb_lock_close( &pv->common->mutex );
851 hb_unlock( pv->common->mutex );
855 w->private_data = NULL;
858 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
863 /***********************************************************************
865 ***********************************************************************
867 **********************************************************************/
868 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
869 hb_buffer_t ** buf_out )
871 hb_work_private_t * pv = w->private_data;
872 hb_job_t * job = pv->job;
873 hb_sync_audio_t * sync = &pv->type.audio;
880 /* if the next buffer is an eof send it downstream */
881 if ( buf->size <= 0 )
883 hb_buffer_close( &buf );
884 *buf_out = hb_buffer_init( 0 );
888 /* Wait for start frame if doing point-to-point */
889 hb_lock( pv->common->mutex );
890 while ( !pv->common->start_found )
892 if ( buf->start < pv->common->audio_pts_thresh )
894 hb_buffer_close( &buf );
895 hb_unlock( pv->common->mutex );
898 while ( !pv->common->start_found &&
899 buf->start >= pv->common->audio_pts_thresh )
901 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
904 if ( buf->start < pv->common->audio_pts_thresh )
906 hb_buffer_close( &buf );
907 hb_unlock( pv->common->mutex );
910 hb_unlock( pv->common->mutex );
912 /* Wait till we can determine the initial pts of all streams */
913 if( pv->common->pts_offset == INT64_MIN )
915 pv->common->first_pts[sync->index+1] = buf->start;
916 hb_lock( pv->common->mutex );
917 while( pv->common->pts_offset == INT64_MIN )
919 // Full fifos will make us wait forever, so get the
920 // pts offset from the available streams if full
921 if (hb_fifo_is_full(w->fifo_in))
924 hb_cond_broadcast( pv->common->next_frame );
926 else if ( checkPtsOffset( w ) )
927 hb_cond_broadcast( pv->common->next_frame );
929 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
931 hb_unlock( pv->common->mutex );
934 if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
936 hb_buffer_close( &buf );
937 *buf_out = hb_buffer_init( 0 );
941 if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
943 hb_buffer_close( &buf );
944 *buf_out = hb_buffer_init( 0 );
948 hb_lock( pv->common->mutex );
949 start = buf->start - pv->common->audio_passthru_slip;
950 hb_unlock( pv->common->mutex );
951 if ( (int64_t)( start - sync->next_pts ) < 0 )
953 // audio time went backwards.
954 // If our output clock is more than a half frame ahead of the
955 // input clock drop this frame to move closer to sync.
956 // Otherwise drop frames until the input clock matches the output clock.
957 if ( sync->first_drop || sync->next_start - start > 90*15 )
959 // Discard data that's in the past.
960 if ( sync->first_drop == 0 )
962 sync->first_drop = sync->next_pts;
965 hb_buffer_close( &buf );
968 sync->next_pts = start;
970 if ( sync->first_drop )
972 // we were dropping old data but input buf time is now current
973 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
974 "(next %"PRId64", current %"PRId64")", w->audio->id,
975 (int)( sync->next_pts - sync->first_drop ) / 90,
976 sync->drop_count, sync->first_drop, sync->next_pts );
977 sync->first_drop = 0;
978 sync->drop_count = 0;
979 sync->next_pts = start;
981 if ( start - sync->next_pts >= (90 * 70) )
983 if ( start - sync->next_pts > (90000LL * 60) )
985 // there's a gap of more than a minute between the last
986 // frame and this. assume we got a corrupted timestamp
987 // and just drop the next buf.
988 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
989 " start %"PRId64", next %"PRId64,
990 (int)((start - sync->next_pts) / (90000*60)),
991 w->audio->id, start, sync->next_pts );
992 hb_buffer_close( &buf );
996 * there's a gap of at least 70ms between the last
997 * frame we processed & the next. Fill it with silence.
998 * Or in the case of DCA, skip some frames from the
1001 if( w->audio->config.out.codec == HB_ACODEC_DCA )
1003 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
1004 " start %"PRId64", next %"PRId64,
1005 (int)((start - sync->next_pts) / 90),
1006 w->audio->id, start, sync->next_pts );
1007 hb_lock( pv->common->mutex );
1008 pv->common->audio_passthru_slip += (start - sync->next_pts);
1009 pv->common->video_pts_slip += (start - sync->next_pts);
1010 hb_unlock( pv->common->mutex );
1014 hb_log( "sync: adding %d ms of silence to audio %d"
1015 " start %"PRId64", next %"PRId64,
1016 (int)((start - sync->next_pts) / 90),
1017 w->audio->id, start, sync->next_pts );
1018 InsertSilence( w, start - sync->next_pts );
1022 * When we get here we've taken care of all the dups and gaps in the
1023 * audio stream and are ready to inject the next input frame into
1024 * the output stream.
1026 *buf_out = OutputAudioFrame( w->audio, buf, sync );
1030 hb_work_object_t hb_sync_audio =
1033 "AudioSynchronization",
1039 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
1041 hb_work_object_t * w;
1042 hb_work_private_t * pv;
1043 hb_title_t * title = job->title;
1044 hb_sync_audio_t * sync;
1046 pv = calloc( 1, sizeof( hb_work_private_t ) );
1047 sync = &pv->type.audio;
1050 pv->common = common;
1052 pv->common->pts_count++;
1054 w = hb_get_work( WORK_SYNC_AUDIO );
1055 w->private_data = pv;
1056 w->audio = hb_list_item( title->list_audio, i );
1057 w->fifo_in = w->audio->priv.fifo_raw;
1059 if( w->audio->config.out.codec == HB_ACODEC_AC3 ||
1060 w->audio->config.out.codec == HB_ACODEC_DCA )
1062 w->fifo_out = w->audio->priv.fifo_out;
1066 w->fifo_out = w->audio->priv.fifo_sync;
1069 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1071 /* Have a silent AC-3 frame ready in case we have to fill a
1077 codec = avcodec_find_encoder( CODEC_ID_AC3 );
1078 c = avcodec_alloc_context();
1080 c->bit_rate = w->audio->config.in.bitrate;
1081 c->sample_rate = w->audio->config.in.samplerate;
1082 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
1084 if( hb_avcodec_open( c, codec ) < 0 )
1086 hb_log( "sync: avcodec_open failed" );
1090 zeros = calloc( AC3_SAMPLES_PER_FRAME *
1091 sizeof( short ) * c->channels, 1 );
1092 sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
1093 w->audio->config.in.samplerate / 8;
1094 sync->ac3_buf = malloc( sync->ac3_size );
1096 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
1097 zeros ) != sync->ac3_size )
1099 hb_log( "sync: avcodec_encode_audio failed" );
1103 hb_avcodec_close( c );
1108 /* Initialize libsamplerate */
1110 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY,
1111 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1112 w->audio->config.out.mixdown), &error );
1113 sync->data.end_of_input = 0;
1115 hb_list_add( job->list_work, w );
1118 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
1119 hb_sync_audio_t *sync )
1121 int64_t start = sync->next_start;
1122 int64_t duration = buf->stop - buf->start;
1124 sync->next_pts += duration;
1126 if( audio->config.in.samplerate == audio->config.out.samplerate ||
1127 audio->config.out.codec == HB_ACODEC_AC3 ||
1128 audio->config.out.codec == HB_ACODEC_DCA )
1131 * If we don't have to do sample rate conversion or this audio is
1132 * pass-thru just send the input buffer downstream after adjusting
1133 * its timestamps to make the output stream continuous.
1138 /* Not pass-thru - do sample rate conversion */
1139 int count_in, count_out;
1140 hb_buffer_t * buf_raw = buf;
1141 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
1144 count_in = buf_raw->size / channel_count;
1146 * When using stupid rates like 44.1 there will always be some
1147 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
1148 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
1149 * the error will build up over time and eventually the audio will
1150 * substantially lag the video. libsamplerate will keep track of the
1151 * fractional sample & give it to us when appropriate if we give it
1152 * an extra sample of space in the output buffer.
1154 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
1156 sync->data.input_frames = count_in;
1157 sync->data.output_frames = count_out;
1158 sync->data.src_ratio = (double)audio->config.out.samplerate /
1159 (double)audio->config.in.samplerate;
1161 buf = hb_buffer_init( count_out * channel_count );
1162 sync->data.data_in = (float *) buf_raw->data;
1163 sync->data.data_out = (float *) buf->data;
1164 if( src_process( sync->state, &sync->data ) )
1166 /* XXX If this happens, we're screwed */
1167 hb_log( "sync: audio %d src_process failed", audio->id );
1169 hb_buffer_close( &buf_raw );
1171 buf->size = sync->data.output_frames_gen * channel_count;
1172 duration = ( sync->data.output_frames_gen * 90000 ) /
1173 audio->config.out.samplerate;
1175 buf->frametype = HB_FRAME_AUDIO;
1177 buf->stop = start + duration;
1178 sync->next_start = start + duration;
1182 static void InsertSilence( hb_work_object_t * w, int64_t duration )
1184 hb_work_private_t * pv = w->private_data;
1185 hb_sync_audio_t *sync = &pv->type.audio;
1189 // to keep pass-thru and regular audio in sync we generate silence in
1190 // AC3 frame-sized units. If the silence duration isn't an integer multiple
1191 // of the AC3 frame duration we will truncate or round up depending on
1192 // which minimizes the timing error.
1193 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1194 w->audio->config.in.samplerate;
1195 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1197 while ( --frame_count >= 0 )
1199 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1201 buf = hb_buffer_init( sync->ac3_size );
1202 buf->start = sync->next_pts;
1203 buf->stop = buf->start + frame_dur;
1204 memcpy( buf->data, sync->ac3_buf, buf->size );
1205 fifo = w->audio->priv.fifo_out;
1209 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1210 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1211 w->audio->config.out.mixdown) );
1212 buf->start = sync->next_pts;
1213 buf->stop = buf->start + frame_dur;
1214 memset( buf->data, 0, buf->size );
1215 fifo = w->audio->priv.fifo_sync;
1217 buf = OutputAudioFrame( w->audio, buf, sync );
1218 hb_fifo_push( fifo, buf );
1222 static void UpdateState( hb_work_object_t * w )
1224 hb_work_private_t * pv = w->private_data;
1225 hb_sync_video_t * sync = &pv->type.video;
1228 if( !pv->common->count_frames )
1230 sync->st_first = hb_get_date();
1231 pv->job->st_pause_date = -1;
1232 pv->job->st_paused = 0;
1234 pv->common->count_frames++;
1236 if( hb_get_date() > sync->st_dates[3] + 1000 )
1238 memmove( &sync->st_dates[0], &sync->st_dates[1],
1239 3 * sizeof( uint64_t ) );
1240 memmove( &sync->st_counts[0], &sync->st_counts[1],
1241 3 * sizeof( uint64_t ) );
1242 sync->st_dates[3] = hb_get_date();
1243 sync->st_counts[3] = pv->common->count_frames;
1246 #define p state.param.working
1247 state.state = HB_STATE_WORKING;
1248 p.progress = (float) pv->common->count_frames / (float) sync->count_frames_max;
1249 if( p.progress > 1.0 )
1253 p.rate_cur = 1000.0 *
1254 (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1255 (float) ( sync->st_dates[3] - sync->st_dates[0] );
1256 if( hb_get_date() > sync->st_first + 4000 )
1259 p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1260 (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1261 eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1263 p.hours = eta / 3600;
1264 p.minutes = ( eta % 3600 ) / 60;
1265 p.seconds = eta % 60;
1276 hb_set_state( pv->job->h, &state );
1279 static void UpdateSearchState( hb_work_object_t * w, int64_t start )
1281 hb_work_private_t * pv = w->private_data;
1282 hb_sync_video_t * sync = &pv->type.video;
1287 now = hb_get_date();
1288 if( !pv->common->count_frames )
1290 sync->st_first = now;
1291 pv->job->st_pause_date = -1;
1292 pv->job->st_paused = 0;
1294 pv->common->count_frames++;
1296 #define p state.param.working
1297 state.state = HB_STATE_SEARCHING;
1298 if ( pv->job->frame_to_start )
1299 p.progress = (float) pv->common->count_frames /
1300 (float) pv->job->frame_to_start;
1301 else if ( pv->job->pts_to_start )
1302 p.progress = (float) start / (float) pv->job->pts_to_start;
1305 if( p.progress > 1.0 )
1309 if (now > sync->st_first)
1313 if ( pv->job->frame_to_start )
1315 avg = 1000.0 * (double)pv->common->count_frames / (now - sync->st_first);
1316 eta = ( pv->job->frame_to_start - pv->common->count_frames ) / avg;
1318 else if ( pv->job->pts_to_start )
1320 avg = 1000.0 * (double)start / (now - sync->st_first);
1321 eta = ( pv->job->pts_to_start - start ) / avg;
1323 p.hours = eta / 3600;
1324 p.minutes = ( eta % 3600 ) / 60;
1325 p.seconds = eta % 60;
1336 hb_set_state( pv->job->h, &state );
1339 static void getPtsOffset( hb_work_object_t * w )
1341 hb_work_private_t * pv = w->private_data;
1343 int64_t first_pts = INT64_MAX;
1345 for( i = 0; i < pv->common->pts_count; i++ )
1347 if ( pv->common->first_pts[i] < first_pts )
1348 first_pts = pv->common->first_pts[i];
1350 pv->common->audio_passthru_slip = pv->common->pts_offset = first_pts;
1354 static int checkPtsOffset( hb_work_object_t * w )
1356 hb_work_private_t * pv = w->private_data;
1359 for( i = 0; i < pv->common->pts_count; i++ )
1361 if ( pv->common->first_pts[i] == INT64_MAX )