1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
22 int ref; /* Reference count to tell us when it's unused */
24 int64_t audio_passthru_slip;
25 int64_t video_pts_slip;
28 /* Frame based point-to-point support */
29 int64_t audio_pts_thresh;
31 hb_cond_t * next_frame;
39 int64_t next_start; /* start time of next output frame */
40 int64_t next_pts; /* start time of next input frame */
41 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
42 int drop_count; /* count of 'time went backwards' drops */
58 int64_t next_start; /* start time of next output frame */
59 int64_t next_pts; /* start time of next input frame */
60 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
61 int drop_count; /* count of 'time went backwards' drops */
62 int drops; /* frames dropped to make a cbr video stream */
63 int dups; /* frames duplicated to make a cbr video stream */
66 int chap_mark; /* to propagate chapter mark across a drop */
67 hb_buffer_t * cur; /* The next picture to process */
70 uint64_t st_counts[4];
75 struct hb_work_private_s
78 hb_sync_common_t * common;
81 hb_sync_video_t video;
82 hb_sync_audio_t audio;
86 /***********************************************************************
88 **********************************************************************/
89 static void getPtsOffset( hb_work_object_t * w );
90 static int checkPtsOffset( hb_work_object_t * w );
91 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
92 static void InsertSilence( hb_work_object_t * w, int64_t d );
93 static void UpdateState( hb_work_object_t * w );
94 static void UpdateSearchState( hb_work_object_t * w, int64_t start );
95 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
96 hb_sync_audio_t *sync );
98 /***********************************************************************
100 ***********************************************************************
101 * Initialize the work object
102 **********************************************************************/
103 hb_work_object_t * hb_sync_init( hb_job_t * job )
105 hb_title_t * title = job->title;
106 hb_chapter_t * chapter;
109 hb_work_private_t * pv;
110 hb_sync_video_t * sync;
111 hb_work_object_t * w;
112 hb_work_object_t * ret = NULL;
114 pv = calloc( 1, sizeof( hb_work_private_t ) );
115 sync = &pv->type.video;
116 pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
118 pv->common->mutex = hb_lock_init();
119 pv->common->audio_pts_thresh = -1;
120 pv->common->next_frame = hb_cond_init();
121 pv->common->pts_count = 1;
122 if ( job->frame_to_start || job->pts_to_start )
124 pv->common->start_found = 0;
128 pv->common->start_found = 1;
131 ret = w = hb_get_work( WORK_SYNC_VIDEO );
132 w->private_data = pv;
133 w->fifo_in = job->fifo_raw;
134 w->fifo_out = job->fifo_sync;
137 pv->common->pts_offset = INT64_MIN;
138 sync->first_frame = 1;
142 /* We already have an accurate frame count from pass 1 */
143 hb_interjob_t * interjob = hb_interjob_get( job->h );
144 sync->count_frames_max = interjob->frame_count;
148 /* Calculate how many video frames we are expecting */
149 if ( job->pts_to_stop )
151 duration = job->pts_to_stop + 90000;
153 else if( job->frame_to_stop )
155 /* Set the duration to a rough estimate */
156 duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
161 for( i = job->chapter_start; i <= job->chapter_end; i++ )
163 chapter = hb_list_item( title->list_chapter, i - 1 );
164 duration += chapter->duration;
167 /* 1 second safety so we're sure we won't miss anything */
169 sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
172 hb_log( "sync: expecting %d video frames", sync->count_frames_max );
174 /* Initialize libsamplerate for every audio track we have */
175 if ( ! job->indepth_scan )
177 for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
179 InitAudio( job, pv->common, i );
182 pv->common->first_pts = malloc( sizeof(int64_t) * pv->common->pts_count );
183 for ( i = 0; i < pv->common->pts_count; i++ )
184 pv->common->first_pts[i] = INT64_MAX;
189 /***********************************************************************
191 ***********************************************************************
193 **********************************************************************/
194 void syncVideoClose( hb_work_object_t * w )
196 hb_work_private_t * pv = w->private_data;
197 hb_job_t * job = pv->job;
198 hb_sync_video_t * sync = &pv->type.video;
200 // Wake up audio sync if it's still waiting on condition.
201 pv->common->pts_offset = 0;
202 pv->common->start_found = 1;
203 hb_cond_broadcast( pv->common->next_frame );
207 hb_buffer_close( &sync->cur );
210 hb_log( "sync: got %d frames, %d expected",
211 pv->common->count_frames, sync->count_frames_max );
213 /* save data for second pass */
216 /* Preserve frame count for better accuracy in pass 2 */
217 hb_interjob_t * interjob = hb_interjob_get( job->h );
218 interjob->frame_count = pv->common->count_frames;
219 interjob->last_job = job->sequence_id;
220 interjob->total_time = sync->next_start;
223 if (sync->drops || sync->dups )
225 hb_log( "sync: %d frames dropped, %d duplicated",
226 sync->drops, sync->dups );
229 hb_lock( pv->common->mutex );
230 if ( --pv->common->ref == 0 )
232 hb_unlock( pv->common->mutex );
233 hb_lock_close( &pv->common->mutex );
238 hb_unlock( pv->common->mutex );
242 w->private_data = NULL;
245 /***********************************************************************
247 ***********************************************************************
249 **********************************************************************/
250 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
251 hb_buffer_t ** buf_out )
253 hb_buffer_t * cur, * next, * sub = NULL;
254 hb_work_private_t * pv = w->private_data;
255 hb_job_t * job = pv->job;
256 hb_subtitle_t * subtitle;
257 hb_sync_video_t * sync = &pv->type.video;
264 /* Wait for start of point-to-point encoding */
265 if( !pv->common->start_found )
267 hb_sync_video_t * sync = &pv->type.video;
269 if( next->size == 0 )
272 pv->common->start_found = 1;
273 hb_cond_broadcast( pv->common->next_frame );
276 * Push through any subtitle EOFs in case they
277 * were not synced through.
279 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
281 subtitle = hb_list_item( job->list_subtitle, i );
282 if( subtitle->config.dest == PASSTHRUSUB )
284 if( subtitle->source == VOBSUB )
285 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
287 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
292 if ( pv->common->count_frames < job->frame_to_start ||
293 next->start < job->pts_to_start )
295 // Flush any subtitles that have pts prior to the
297 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
299 subtitle = hb_list_item( job->list_subtitle, i );
300 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
302 if ( sub->start > next->start )
304 sub = hb_fifo_get( subtitle->fifo_raw );
305 hb_buffer_close( &sub );
308 hb_lock( pv->common->mutex );
309 // Tell the audio threads what must be dropped
310 pv->common->audio_pts_thresh = next->start;
311 hb_cond_broadcast( pv->common->next_frame );
312 hb_unlock( pv->common->mutex );
314 UpdateSearchState( w, next->start );
315 hb_buffer_close( &next );
319 hb_lock( pv->common->mutex );
320 pv->common->start_found = 1;
321 pv->common->count_frames = 0;
322 hb_cond_broadcast( pv->common->next_frame );
323 hb_unlock( pv->common->mutex );
327 /* Wait till we can determine the initial pts of all streams */
328 if( pv->common->pts_offset == INT64_MIN )
330 pv->common->first_pts[0] = next->start;
331 hb_lock( pv->common->mutex );
332 while( pv->common->pts_offset == INT64_MIN )
334 // Full fifos will make us wait forever, so get the
335 // pts offset from the available streams if full
336 if ( hb_fifo_is_full( job->fifo_raw ) )
339 hb_cond_broadcast( pv->common->next_frame );
341 else if ( checkPtsOffset( w ) )
342 hb_cond_broadcast( pv->common->next_frame );
344 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
346 hb_unlock( pv->common->mutex );
352 if( sync->cur->size == 0 )
354 /* we got an end-of-stream as our first video packet?
355 * Feed it downstream & signal that we're done.
357 *buf_out = hb_buffer_init( 0 );
359 pv->common->start_found = 1;
360 hb_cond_broadcast( pv->common->next_frame );
363 * Push through any subtitle EOFs in case they
364 * were not synced through.
366 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
368 subtitle = hb_list_item( job->list_subtitle, i );
369 if( subtitle->config.dest == PASSTHRUSUB )
371 if( subtitle->source == VOBSUB )
372 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
374 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
382 /* At this point we have a frame to process. Let's check
383 1) if we will be able to push into the fifo ahead
384 2) if the next frame is there already, since we need it to
385 compute the duration of the current frame*/
386 if( next->size == 0 )
388 hb_buffer_close( &next );
390 cur->start = sync->next_start;
391 cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
393 /* Make sure last frame is reflected in frame count */
394 pv->common->count_frames++;
396 /* Push the frame to the renderer */
397 hb_fifo_push( job->fifo_sync, cur );
400 /* we got an end-of-stream. Feed it downstream & signal that
401 * we're done. Note that this means we drop the final frame of
402 * video (we don't know its duration). On DVDs the final frame
403 * is often strange and dropping it seems to be a good idea. */
404 *buf_out = hb_buffer_init( 0 );
407 * Push through any subtitle EOFs in case they were not synced through.
409 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
411 subtitle = hb_list_item( job->list_subtitle, i );
412 if( subtitle->config.dest == PASSTHRUSUB )
414 if( subtitle->source == VOBSUB )
415 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
417 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
420 pv->common->start_found = 1;
421 hb_cond_broadcast( pv->common->next_frame );
425 /* Check for end of point-to-point frame encoding */
426 if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
428 // Drop an empty buffer into our output to ensure that things
429 // get flushed all the way out.
430 hb_buffer_close( &sync->cur );
431 hb_buffer_close( &next );
432 *buf_out = hb_buffer_init( 0 );
433 hb_log( "sync: reached %d frames, exiting early",
434 pv->common->count_frames );
437 * Push through any subtitle EOFs in case they were not synced through.
439 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
441 subtitle = hb_list_item( job->list_subtitle, i );
442 if( subtitle->config.dest == PASSTHRUSUB )
444 if( subtitle->source == VOBSUB )
445 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
447 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
453 /* Check for end of point-to-point pts encoding */
454 if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
456 // Drop an empty buffer into our output to ensure that things
457 // get flushed all the way out.
458 hb_log( "sync: reached pts %"PRId64", exiting early",
460 hb_buffer_close( &sync->cur );
461 hb_buffer_close( &next );
462 *buf_out = hb_buffer_init( 0 );
465 * Push through any subtitle EOFs in case they were not synced through.
467 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
469 subtitle = hb_list_item( job->list_subtitle, i );
470 if( subtitle->config.dest == PASSTHRUSUB )
472 if( subtitle->source == VOBSUB )
473 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
475 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
481 if( sync->first_frame )
483 /* This is our first frame */
484 if ( cur->start > pv->common->pts_offset )
487 * The first pts from a dvd should always be zero but
488 * can be non-zero with a transport or program stream since
489 * we're not guaranteed to start on an IDR frame. If we get
490 * a non-zero initial PTS extend its duration so it behaves
491 * as if it started at zero so that our audio timing will
494 hb_log( "sync: first pts is %"PRId64, cur->start );
495 cur->start = pv->common->pts_offset;
497 sync->first_frame = 0;
501 * since the first frame is always 0 and the upstream reader code
502 * is taking care of adjusting for pts discontinuities, we just have
503 * to deal with the next frame's start being in the past. This can
504 * happen when the PTS is adjusted after data loss but video frame
505 * reordering causes some frames with the old clock to appear after
506 * the clock change. This creates frames that overlap in time which
507 * looks to us like time going backward. The downstream muxing code
508 * can deal with overlaps of up to a frame time but anything larger
509 * we handle by dropping frames here.
511 hb_lock( pv->common->mutex );
512 if ( (int64_t)( next->start - pv->common->video_pts_slip - cur->start ) <= 0 )
514 if ( sync->first_drop == 0 )
516 sync->first_drop = next->start;
519 if (next->start - cur->start > 0)
521 sync->pts_skip += next->start - cur->start;
522 pv->common->video_pts_slip -= next->start - cur->start;
524 hb_unlock( pv->common->mutex );
525 if ( next->new_chap )
527 // don't drop a chapter mark when we drop the buffer
528 sync->chap_mark = next->new_chap;
530 hb_buffer_close( &next );
533 hb_unlock( pv->common->mutex );
534 if ( sync->first_drop )
536 hb_log( "sync: video time didn't advance - dropped %d frames "
537 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
538 sync->drop_count, (int)( cur->start - sync->first_drop ) / 90,
539 cur->start, next->start, (int)( next->start - cur->start ) );
540 sync->first_drop = 0;
541 sync->drop_count = 0;
545 * Track the video sequence number localy so that we can sync the audio
546 * to it using the sequence number as well as the PTS.
548 sync->video_sequence = cur->sequence;
551 * Look for a subtitle for this frame.
553 * If found then it will be tagged onto a video buffer of the correct time and
554 * sent in to the render pipeline. This only needs to be done for VOBSUBs which
555 * get rendered, other types of subtitles can just sit in their raw_queue until
556 * delt with at muxing.
558 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
560 subtitle = hb_list_item( job->list_subtitle, i );
563 * Rewrite timestamps on subtitles that need it (on raw queue).
565 if( subtitle->source == CC608SUB ||
566 subtitle->source == CC708SUB ||
567 subtitle->source == SRTSUB ||
568 subtitle->source == UTF8SUB ||
569 subtitle->source == TX3GSUB ||
570 subtitle->source == SSASUB)
573 * Rewrite timestamps on subtitles that came from Closed Captions
574 * since they are using the MPEG2 timestamps.
576 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
579 * Rewrite the timestamps as and when the video
580 * (cur->start) reaches the same timestamp as a
581 * closed caption (sub->start).
583 * What about discontinuity boundaries - not delt
586 * Bypass the sync fifo altogether.
590 sub = hb_fifo_get( subtitle->fifo_raw );
591 hb_fifo_push( subtitle->fifo_out, sub );
596 * Sync the subtitles to the incoming video, and use
597 * the matching converted video timestamp.
599 * Note that it doesn't appear that we need to convert
600 * timestamps, I guess that they were already correct,
601 * so just push them through for rendering.
604 if( sub->start < cur->start )
606 sub = hb_fifo_get( subtitle->fifo_raw );
607 hb_fifo_push( subtitle->fifo_out, sub );
616 if( subtitle->source == VOBSUB )
619 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
624 * EOF, pass it through immediately.
629 /* If two subtitles overlap, make the first one stop
630 when the second one starts */
631 sub2 = hb_fifo_see2( subtitle->fifo_raw );
632 if( sub2 && sub->stop > sub2->start )
634 sub->stop = sub2->start;
637 // hb_log("0x%x: video seq: %"PRId64" subtitle sequence: %"PRId64,
638 // sub, cur->sequence, sub->sequence);
640 if( sub->sequence > cur->sequence )
643 * The video is behind where we are, so wait until
644 * it catches up to the same reader point on the
645 * DVD. Then our PTS should be in the same region
652 if( sub->stop > cur->start ) {
654 * The stop time is in the future, so fall through
655 * and we'll deal with it in the next block of
660 * There is a valid subtitle, is it time to display it?
662 if( sub->stop > sub->start)
665 * Normal subtitle which ends after it starts,
666 * check to see that the current video is between
669 if( cur->start > sub->start &&
670 cur->start < sub->stop )
673 * We should be playing this, so leave the
676 * fall through to display
678 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
681 * Subtitle is on for less than three
682 * seconds, extend the time that it is
683 * displayed to make it easier to read.
684 * Make it 3 seconds or until the next
685 * subtitle is displayed.
687 * This is in response to Indochine which
688 * only displays subs for 1 second -
691 sub->stop = sub->start + ( 2 * 90000 );
693 sub2 = hb_fifo_see2( subtitle->fifo_raw );
695 if( sub2 && sub->stop > sub2->start )
697 sub->stop = sub2->start;
704 * Defer until the play point is within
713 * The end of the subtitle is less than the start,
714 * this is a sign of a PTS discontinuity.
716 if( sub->start > cur->start )
719 * we haven't reached the start time yet, or
720 * we have jumped backwards after having
721 * already started this subtitle.
723 if( cur->start < sub->stop )
726 * We have jumped backwards and so should
727 * continue displaying this subtitle.
729 * fall through to display.
735 * Defer until the play point is
736 * within the subtitle
742 * Play this subtitle as the start is
743 * greater than our video point.
745 * fall through to display/
755 * The subtitle is older than this picture, trash it
757 sub = hb_fifo_get( subtitle->fifo_raw );
758 hb_buffer_close( &sub );
762 /* If we have a subtitle for this picture, copy it */
763 /* FIXME: we should avoid this memcpy */
768 if( subtitle->config.dest == RENDERSUB )
770 if ( cur->sub == NULL )
773 * Tack onto the video buffer for rendering
775 cur->sub = hb_buffer_init( sub->size );
776 cur->sub->x = sub->x;
777 cur->sub->y = sub->y;
778 cur->sub->width = sub->width;
779 cur->sub->height = sub->height;
780 memcpy( cur->sub->data, sub->data, sub->size );
784 * Pass-Through, pop it off of the raw queue,
786 sub = hb_fifo_get( subtitle->fifo_raw );
787 hb_fifo_push( subtitle->fifo_sync, sub );
791 * EOF - consume for rendered, else pass through
793 if( subtitle->config.dest == RENDERSUB )
795 sub = hb_fifo_get( subtitle->fifo_raw );
796 hb_buffer_close( &sub );
798 sub = hb_fifo_get( subtitle->fifo_raw );
799 hb_fifo_push( subtitle->fifo_sync, sub );
807 * Adjust the pts of the current frame so that it's contiguous
808 * with the previous frame. The start time of the current frame
809 * has to be the end time of the previous frame and the stop
810 * time has to be the start of the next frame. We don't
811 * make any adjustments to the source timestamps other than removing
812 * the clock offsets (which also removes pts discontinuities).
813 * This means we automatically encode at the source's frame rate.
814 * MP2 uses an implicit duration (frames end when the next frame
815 * starts) but more advanced containers like MP4 use an explicit
816 * duration. Since we're looking ahead one frame we set the
817 * explicit stop time from the start time of the next frame.
820 sync->cur = cur = next;
822 sync->next_pts = cur->start;
823 int64_t duration = cur->start - sync->pts_skip - (*buf_out)->start;
827 hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
828 duration, (*buf_out)->start, next->start );
831 (*buf_out)->start = sync->next_start;
832 sync->next_start += duration;
833 (*buf_out)->stop = sync->next_start;
835 if ( sync->chap_mark )
837 // we have a pending chapter mark from a recent drop - put it on this
838 // buffer (this may make it one frame late but we can't do any better).
839 (*buf_out)->new_chap = sync->chap_mark;
849 // sync*Init does nothing because sync has a special initializer
850 // that takes care of initializing video and all audio tracks
851 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
856 hb_work_object_t hb_sync_video =
859 "Video Synchronization",
865 /***********************************************************************
867 ***********************************************************************
869 **********************************************************************/
870 void syncAudioClose( hb_work_object_t * w )
872 hb_work_private_t * pv = w->private_data;
873 hb_sync_audio_t * sync = &pv->type.audio;
875 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
877 free( sync->ac3_buf );
881 src_delete( sync->state );
884 hb_lock( pv->common->mutex );
885 if ( --pv->common->ref == 0 )
887 hb_unlock( pv->common->mutex );
888 hb_lock_close( &pv->common->mutex );
893 hb_unlock( pv->common->mutex );
897 w->private_data = NULL;
900 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
905 /***********************************************************************
907 ***********************************************************************
909 **********************************************************************/
910 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
911 hb_buffer_t ** buf_out )
913 hb_work_private_t * pv = w->private_data;
914 hb_job_t * job = pv->job;
915 hb_sync_audio_t * sync = &pv->type.audio;
922 /* if the next buffer is an eof send it downstream */
923 if ( buf->size <= 0 )
925 hb_buffer_close( &buf );
926 *buf_out = hb_buffer_init( 0 );
930 /* Wait for start frame if doing point-to-point */
931 hb_lock( pv->common->mutex );
932 while ( !pv->common->start_found )
934 if ( pv->common->audio_pts_thresh < 0 )
936 // I would initialize this in hb_sync_init, but
937 // job->pts_to_start can be modified by reader
938 // after hb_sync_init is called.
939 pv->common->audio_pts_thresh = job->pts_to_start;
941 if ( buf->start < pv->common->audio_pts_thresh )
943 hb_buffer_close( &buf );
944 hb_unlock( pv->common->mutex );
947 while ( !pv->common->start_found &&
948 buf->start >= pv->common->audio_pts_thresh )
950 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
953 if ( buf->start < pv->common->audio_pts_thresh )
955 hb_buffer_close( &buf );
956 hb_unlock( pv->common->mutex );
959 hb_unlock( pv->common->mutex );
961 /* Wait till we can determine the initial pts of all streams */
962 if( pv->common->pts_offset == INT64_MIN )
964 pv->common->first_pts[sync->index+1] = buf->start;
965 hb_lock( pv->common->mutex );
966 while( pv->common->pts_offset == INT64_MIN )
968 // Full fifos will make us wait forever, so get the
969 // pts offset from the available streams if full
970 if (hb_fifo_is_full(w->fifo_in))
973 hb_cond_broadcast( pv->common->next_frame );
975 else if ( checkPtsOffset( w ) )
976 hb_cond_broadcast( pv->common->next_frame );
978 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
980 hb_unlock( pv->common->mutex );
983 if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
985 hb_buffer_close( &buf );
986 *buf_out = hb_buffer_init( 0 );
990 if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
992 hb_buffer_close( &buf );
993 *buf_out = hb_buffer_init( 0 );
997 hb_lock( pv->common->mutex );
998 start = buf->start - pv->common->audio_passthru_slip;
999 hb_unlock( pv->common->mutex );
1000 if ( (int64_t)( start - sync->next_pts ) < 0 )
1002 // audio time went backwards.
1003 // If our output clock is more than a half frame ahead of the
1004 // input clock drop this frame to move closer to sync.
1005 // Otherwise drop frames until the input clock matches the output clock.
1006 if ( sync->first_drop || sync->next_start - start > 90*15 )
1008 // Discard data that's in the past.
1009 if ( sync->first_drop == 0 )
1011 sync->first_drop = sync->next_pts;
1014 hb_buffer_close( &buf );
1017 sync->next_pts = start;
1019 if ( sync->first_drop )
1021 // we were dropping old data but input buf time is now current
1022 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
1023 "(next %"PRId64", current %"PRId64")", w->audio->id,
1024 (int)( sync->next_pts - sync->first_drop ) / 90,
1025 sync->drop_count, sync->first_drop, sync->next_pts );
1026 sync->first_drop = 0;
1027 sync->drop_count = 0;
1028 sync->next_pts = start;
1030 if ( start - sync->next_pts >= (90 * 70) )
1032 if ( start - sync->next_pts > (90000LL * 60) )
1034 // there's a gap of more than a minute between the last
1035 // frame and this. assume we got a corrupted timestamp
1036 // and just drop the next buf.
1037 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
1038 " start %"PRId64", next %"PRId64,
1039 (int)((start - sync->next_pts) / (90000*60)),
1040 w->audio->id, start, sync->next_pts );
1041 hb_buffer_close( &buf );
1045 * there's a gap of at least 70ms between the last
1046 * frame we processed & the next. Fill it with silence.
1047 * Or in the case of DCA, skip some frames from the
1050 if( w->audio->config.out.codec == HB_ACODEC_DCA )
1052 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
1053 " start %"PRId64", next %"PRId64,
1054 (int)((start - sync->next_pts) / 90),
1055 w->audio->id, start, sync->next_pts );
1056 hb_lock( pv->common->mutex );
1057 pv->common->audio_passthru_slip += (start - sync->next_pts);
1058 pv->common->video_pts_slip += (start - sync->next_pts);
1059 hb_unlock( pv->common->mutex );
1063 hb_log( "sync: adding %d ms of silence to audio %d"
1064 " start %"PRId64", next %"PRId64,
1065 (int)((start - sync->next_pts) / 90),
1066 w->audio->id, start, sync->next_pts );
1067 InsertSilence( w, start - sync->next_pts );
1071 * When we get here we've taken care of all the dups and gaps in the
1072 * audio stream and are ready to inject the next input frame into
1073 * the output stream.
1075 *buf_out = OutputAudioFrame( w->audio, buf, sync );
1079 hb_work_object_t hb_sync_audio =
1082 "AudioSynchronization",
1088 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
1090 hb_work_object_t * w;
1091 hb_work_private_t * pv;
1092 hb_title_t * title = job->title;
1093 hb_sync_audio_t * sync;
1095 pv = calloc( 1, sizeof( hb_work_private_t ) );
1096 sync = &pv->type.audio;
1099 pv->common = common;
1101 pv->common->pts_count++;
1103 w = hb_get_work( WORK_SYNC_AUDIO );
1104 w->private_data = pv;
1105 w->audio = hb_list_item( title->list_audio, i );
1106 w->fifo_in = w->audio->priv.fifo_raw;
1108 if( w->audio->config.out.codec == HB_ACODEC_AC3 ||
1109 w->audio->config.out.codec == HB_ACODEC_DCA )
1111 w->fifo_out = w->audio->priv.fifo_out;
1115 w->fifo_out = w->audio->priv.fifo_sync;
1118 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1120 /* Have a silent AC-3 frame ready in case we have to fill a
1126 codec = avcodec_find_encoder( CODEC_ID_AC3 );
1127 c = avcodec_alloc_context();
1129 c->bit_rate = w->audio->config.in.bitrate;
1130 c->sample_rate = w->audio->config.in.samplerate;
1131 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
1133 if( hb_avcodec_open( c, codec ) < 0 )
1135 hb_log( "sync: avcodec_open failed" );
1139 zeros = calloc( AC3_SAMPLES_PER_FRAME *
1140 sizeof( short ) * c->channels, 1 );
1141 sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
1142 w->audio->config.in.samplerate / 8;
1143 sync->ac3_buf = malloc( sync->ac3_size );
1145 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
1146 zeros ) != sync->ac3_size )
1148 hb_log( "sync: avcodec_encode_audio failed" );
1152 hb_avcodec_close( c );
1157 /* Initialize libsamplerate */
1159 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY,
1160 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1161 w->audio->config.out.mixdown), &error );
1162 sync->data.end_of_input = 0;
1164 hb_list_add( job->list_work, w );
1167 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
1168 hb_sync_audio_t *sync )
1170 int64_t start = sync->next_start;
1171 int64_t duration = buf->stop - buf->start;
1173 sync->next_pts += duration;
1175 if( audio->config.in.samplerate == audio->config.out.samplerate ||
1176 audio->config.out.codec == HB_ACODEC_AC3 ||
1177 audio->config.out.codec == HB_ACODEC_DCA )
1180 * If we don't have to do sample rate conversion or this audio is
1181 * pass-thru just send the input buffer downstream after adjusting
1182 * its timestamps to make the output stream continuous.
1187 /* Not pass-thru - do sample rate conversion */
1188 int count_in, count_out;
1189 hb_buffer_t * buf_raw = buf;
1190 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
1193 count_in = buf_raw->size / channel_count;
1195 * When using stupid rates like 44.1 there will always be some
1196 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
1197 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
1198 * the error will build up over time and eventually the audio will
1199 * substantially lag the video. libsamplerate will keep track of the
1200 * fractional sample & give it to us when appropriate if we give it
1201 * an extra sample of space in the output buffer.
1203 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
1205 sync->data.input_frames = count_in;
1206 sync->data.output_frames = count_out;
1207 sync->data.src_ratio = (double)audio->config.out.samplerate /
1208 (double)audio->config.in.samplerate;
1210 buf = hb_buffer_init( count_out * channel_count );
1211 sync->data.data_in = (float *) buf_raw->data;
1212 sync->data.data_out = (float *) buf->data;
1213 if( src_process( sync->state, &sync->data ) )
1215 /* XXX If this happens, we're screwed */
1216 hb_log( "sync: audio %d src_process failed", audio->id );
1218 hb_buffer_close( &buf_raw );
1220 buf->size = sync->data.output_frames_gen * channel_count;
1221 duration = ( sync->data.output_frames_gen * 90000 ) /
1222 audio->config.out.samplerate;
1224 buf->frametype = HB_FRAME_AUDIO;
1226 buf->stop = start + duration;
1227 sync->next_start = start + duration;
1231 static void InsertSilence( hb_work_object_t * w, int64_t duration )
1233 hb_work_private_t * pv = w->private_data;
1234 hb_sync_audio_t *sync = &pv->type.audio;
1238 // to keep pass-thru and regular audio in sync we generate silence in
1239 // AC3 frame-sized units. If the silence duration isn't an integer multiple
1240 // of the AC3 frame duration we will truncate or round up depending on
1241 // which minimizes the timing error.
1242 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1243 w->audio->config.in.samplerate;
1244 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1246 while ( --frame_count >= 0 )
1248 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1250 buf = hb_buffer_init( sync->ac3_size );
1251 buf->start = sync->next_pts;
1252 buf->stop = buf->start + frame_dur;
1253 memcpy( buf->data, sync->ac3_buf, buf->size );
1254 fifo = w->audio->priv.fifo_out;
1258 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1259 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1260 w->audio->config.out.mixdown) );
1261 buf->start = sync->next_pts;
1262 buf->stop = buf->start + frame_dur;
1263 memset( buf->data, 0, buf->size );
1264 fifo = w->audio->priv.fifo_sync;
1266 buf = OutputAudioFrame( w->audio, buf, sync );
1267 hb_fifo_push( fifo, buf );
1271 static void UpdateState( hb_work_object_t * w )
1273 hb_work_private_t * pv = w->private_data;
1274 hb_sync_video_t * sync = &pv->type.video;
1277 if( !pv->common->count_frames )
1279 sync->st_first = hb_get_date();
1280 pv->job->st_pause_date = -1;
1281 pv->job->st_paused = 0;
1283 pv->common->count_frames++;
1285 if( hb_get_date() > sync->st_dates[3] + 1000 )
1287 memmove( &sync->st_dates[0], &sync->st_dates[1],
1288 3 * sizeof( uint64_t ) );
1289 memmove( &sync->st_counts[0], &sync->st_counts[1],
1290 3 * sizeof( uint64_t ) );
1291 sync->st_dates[3] = hb_get_date();
1292 sync->st_counts[3] = pv->common->count_frames;
1295 #define p state.param.working
1296 state.state = HB_STATE_WORKING;
1297 p.progress = (float) pv->common->count_frames / (float) sync->count_frames_max;
1298 if( p.progress > 1.0 )
1302 p.rate_cur = 1000.0 *
1303 (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1304 (float) ( sync->st_dates[3] - sync->st_dates[0] );
1305 if( hb_get_date() > sync->st_first + 4000 )
1308 p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1309 (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1310 eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1312 p.hours = eta / 3600;
1313 p.minutes = ( eta % 3600 ) / 60;
1314 p.seconds = eta % 60;
1325 hb_set_state( pv->job->h, &state );
1328 static void UpdateSearchState( hb_work_object_t * w, int64_t start )
1330 hb_work_private_t * pv = w->private_data;
1331 hb_sync_video_t * sync = &pv->type.video;
1336 now = hb_get_date();
1337 if( !pv->common->count_frames )
1339 sync->st_first = now;
1340 pv->job->st_pause_date = -1;
1341 pv->job->st_paused = 0;
1343 pv->common->count_frames++;
1345 #define p state.param.working
1346 state.state = HB_STATE_SEARCHING;
1347 if ( pv->job->frame_to_start )
1348 p.progress = (float) pv->common->count_frames /
1349 (float) pv->job->frame_to_start;
1350 else if ( pv->job->pts_to_start )
1351 p.progress = (float) start / (float) pv->job->pts_to_start;
1354 if( p.progress > 1.0 )
1358 if (now > sync->st_first)
1362 if ( pv->job->frame_to_start )
1364 avg = 1000.0 * (double)pv->common->count_frames / (now - sync->st_first);
1365 eta = ( pv->job->frame_to_start - pv->common->count_frames ) / avg;
1367 else if ( pv->job->pts_to_start )
1369 avg = 1000.0 * (double)start / (now - sync->st_first);
1370 eta = ( pv->job->pts_to_start - start ) / avg;
1372 p.hours = eta / 3600;
1373 p.minutes = ( eta % 3600 ) / 60;
1374 p.seconds = eta % 60;
1385 hb_set_state( pv->job->h, &state );
1388 static void getPtsOffset( hb_work_object_t * w )
1390 hb_work_private_t * pv = w->private_data;
1392 int64_t first_pts = INT64_MAX;
1394 for( i = 0; i < pv->common->pts_count; i++ )
1396 if ( pv->common->first_pts[i] < first_pts )
1397 first_pts = pv->common->first_pts[i];
1399 pv->common->audio_passthru_slip = pv->common->pts_offset = first_pts;
1403 static int checkPtsOffset( hb_work_object_t * w )
1405 hb_work_private_t * pv = w->private_data;
1408 for( i = 0; i < pv->common->pts_count; i++ )
1410 if ( pv->common->first_pts[i] == INT64_MAX )