1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.fr/>.
5 It may be used under the terms of the GNU General Public License. */
10 #include "samplerate.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
22 int ref; /* Reference count to tell us when it's unused */
24 int64_t audio_passthru_slip;
25 int64_t video_pts_slip;
28 /* Frame based point-to-point support */
29 int64_t audio_pts_thresh;
31 hb_cond_t * next_frame;
39 int64_t next_start; /* start time of next output frame */
40 int64_t next_pts; /* start time of next input frame */
41 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
42 int drop_count; /* count of 'time went backwards' drops */
58 int64_t next_start; /* start time of next output frame */
59 int64_t next_pts; /* start time of next input frame */
60 int64_t first_drop; /* PTS of first 'went backwards' frame dropped */
61 int drop_count; /* count of 'time went backwards' drops */
62 int drops; /* frames dropped to make a cbr video stream */
63 int dups; /* frames duplicated to make a cbr video stream */
66 int chap_mark; /* to propagate chapter mark across a drop */
67 hb_buffer_t * cur; /* The next picture to process */
70 uint64_t st_counts[4];
75 struct hb_work_private_s
78 hb_sync_common_t * common;
81 hb_sync_video_t video;
82 hb_sync_audio_t audio;
86 /***********************************************************************
88 **********************************************************************/
89 static void getPtsOffset( hb_work_object_t * w );
90 static int checkPtsOffset( hb_work_object_t * w );
91 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
92 static void InsertSilence( hb_work_object_t * w, int64_t d );
93 static void UpdateState( hb_work_object_t * w );
94 static void UpdateSearchState( hb_work_object_t * w, int64_t start );
95 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
96 hb_sync_audio_t *sync );
98 /***********************************************************************
100 ***********************************************************************
101 * Initialize the work object
102 **********************************************************************/
103 int hb_sync_init( hb_job_t * job )
105 hb_title_t * title = job->title;
106 hb_chapter_t * chapter;
109 hb_work_private_t * pv;
110 hb_sync_video_t * sync;
111 hb_work_object_t * w;
113 pv = calloc( 1, sizeof( hb_work_private_t ) );
114 sync = &pv->type.video;
115 pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
117 pv->common->mutex = hb_lock_init();
118 pv->common->audio_pts_thresh = 0;
119 pv->common->next_frame = hb_cond_init();
120 pv->common->pts_count = 1;
121 if ( job->frame_to_start || job->pts_to_start )
123 pv->common->start_found = 0;
127 pv->common->start_found = 1;
130 w = hb_get_work( WORK_SYNC_VIDEO );
131 w->private_data = pv;
132 w->fifo_in = job->fifo_raw;
133 w->fifo_out = job->fifo_sync;
136 pv->common->pts_offset = INT64_MIN;
137 sync->first_frame = 1;
141 /* We already have an accurate frame count from pass 1 */
142 hb_interjob_t * interjob = hb_interjob_get( job->h );
143 sync->count_frames_max = interjob->frame_count;
147 /* Calculate how many video frames we are expecting */
148 if ( job->pts_to_stop )
150 duration = job->pts_to_stop + 90000;
152 else if( job->frame_to_stop )
154 /* Set the duration to a rough estimate */
155 duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
160 for( i = job->chapter_start; i <= job->chapter_end; i++ )
162 chapter = hb_list_item( title->list_chapter, i - 1 );
163 duration += chapter->duration;
166 /* 1 second safety so we're sure we won't miss anything */
168 sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
170 hb_list_add( job->list_work, w );
172 hb_log( "sync: expecting %d video frames", sync->count_frames_max );
174 /* Initialize libsamplerate for every audio track we have */
175 if ( ! job->indepth_scan )
177 for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
179 InitAudio( job, pv->common, i );
182 pv->common->first_pts = malloc( sizeof(int64_t) * pv->common->pts_count );
183 for ( i = 0; i < pv->common->pts_count; i++ )
184 pv->common->first_pts[i] = INT64_MAX;
189 /***********************************************************************
191 ***********************************************************************
193 **********************************************************************/
194 void syncVideoClose( hb_work_object_t * w )
196 hb_work_private_t * pv = w->private_data;
197 hb_job_t * job = pv->job;
198 hb_sync_video_t * sync = &pv->type.video;
202 hb_buffer_close( &sync->cur );
205 hb_log( "sync: got %d frames, %d expected",
206 pv->common->count_frames, sync->count_frames_max );
208 /* save data for second pass */
211 /* Preserve frame count for better accuracy in pass 2 */
212 hb_interjob_t * interjob = hb_interjob_get( job->h );
213 interjob->frame_count = pv->common->count_frames;
214 interjob->last_job = job->sequence_id;
215 interjob->total_time = sync->next_start;
218 if (sync->drops || sync->dups )
220 hb_log( "sync: %d frames dropped, %d duplicated",
221 sync->drops, sync->dups );
224 hb_lock( pv->common->mutex );
225 if ( --pv->common->ref == 0 )
227 hb_unlock( pv->common->mutex );
228 hb_lock_close( &pv->common->mutex );
233 hb_unlock( pv->common->mutex );
237 w->private_data = NULL;
240 /***********************************************************************
242 ***********************************************************************
244 **********************************************************************/
245 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
246 hb_buffer_t ** buf_out )
248 hb_buffer_t * cur, * next, * sub = NULL;
249 hb_work_private_t * pv = w->private_data;
250 hb_job_t * job = pv->job;
251 hb_subtitle_t * subtitle;
252 hb_sync_video_t * sync = &pv->type.video;
259 /* Wait for start of point-to-point encoding */
260 if( !pv->common->start_found )
262 hb_sync_video_t * sync = &pv->type.video;
264 if( next->size == 0 )
267 pv->common->start_found = 1;
268 hb_cond_broadcast( pv->common->next_frame );
271 * Push through any subtitle EOFs in case they
272 * were not synced through.
274 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
276 subtitle = hb_list_item( job->list_subtitle, i );
277 if( subtitle->config.dest == PASSTHRUSUB )
279 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
284 if ( pv->common->count_frames < job->frame_to_start ||
285 next->start < job->pts_to_start )
287 // Flush any subtitles that have pts prior to the
289 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
291 subtitle = hb_list_item( job->list_subtitle, i );
292 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
294 if ( sub->start > next->start )
296 sub = hb_fifo_get( subtitle->fifo_raw );
297 hb_buffer_close( &sub );
300 hb_lock( pv->common->mutex );
301 // Tell the audio threads what must be dropped
302 pv->common->audio_pts_thresh = next->start;
303 hb_cond_broadcast( pv->common->next_frame );
304 hb_unlock( pv->common->mutex );
306 UpdateSearchState( w, next->start );
307 hb_buffer_close( &next );
311 hb_lock( pv->common->mutex );
312 pv->common->start_found = 1;
313 pv->common->count_frames = 0;
314 hb_cond_broadcast( pv->common->next_frame );
315 hb_unlock( pv->common->mutex );
319 /* Wait till we can determine the initial pts of all streams */
320 if( pv->common->pts_offset == INT64_MIN )
322 pv->common->first_pts[0] = next->start;
323 hb_lock( pv->common->mutex );
324 while( pv->common->pts_offset == INT64_MIN )
326 // Full fifos will make us wait forever, so get the
327 // pts offset from the available streams if full
328 if ( hb_fifo_is_full( job->fifo_raw ) )
331 hb_cond_broadcast( pv->common->next_frame );
333 else if ( checkPtsOffset( w ) )
334 hb_cond_broadcast( pv->common->next_frame );
336 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
338 hb_unlock( pv->common->mutex );
344 if( sync->cur->size == 0 )
346 /* we got an end-of-stream as our first video packet?
347 * Feed it downstream & signal that we're done.
349 *buf_out = hb_buffer_init( 0 );
351 pv->common->start_found = 1;
352 hb_cond_broadcast( pv->common->next_frame );
355 * Push through any subtitle EOFs in case they
356 * were not synced through.
358 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
360 subtitle = hb_list_item( job->list_subtitle, i );
361 if( subtitle->config.dest == PASSTHRUSUB )
363 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
371 /* At this point we have a frame to process. Let's check
372 1) if we will be able to push into the fifo ahead
373 2) if the next frame is there already, since we need it to
374 compute the duration of the current frame*/
375 if( next->size == 0 )
377 hb_buffer_close( &next );
379 cur->start = sync->next_start;
380 cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
382 /* Push the frame to the renderer */
383 hb_fifo_push( job->fifo_sync, cur );
386 /* we got an end-of-stream. Feed it downstream & signal that
387 * we're done. Note that this means we drop the final frame of
388 * video (we don't know its duration). On DVDs the final frame
389 * is often strange and dropping it seems to be a good idea. */
390 *buf_out = hb_buffer_init( 0 );
393 * Push through any subtitle EOFs in case they were not synced through.
395 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
397 subtitle = hb_list_item( job->list_subtitle, i );
398 if( subtitle->config.dest == PASSTHRUSUB )
400 if( subtitle->source == VOBSUB )
401 hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
403 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
406 pv->common->start_found = 1;
407 hb_cond_broadcast( pv->common->next_frame );
411 /* Check for end of point-to-point frame encoding */
412 if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
414 // Drop an empty buffer into our output to ensure that things
415 // get flushed all the way out.
416 hb_buffer_close( &sync->cur );
417 hb_buffer_close( &next );
418 *buf_out = hb_buffer_init( 0 );
419 hb_log( "sync: reached %d frames, exiting early",
420 pv->common->count_frames );
424 /* Check for end of point-to-point pts encoding */
425 if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
427 // Drop an empty buffer into our output to ensure that things
428 // get flushed all the way out.
429 hb_log( "sync: reached pts %"PRId64", exiting early",
431 hb_buffer_close( &sync->cur );
432 hb_buffer_close( &next );
433 *buf_out = hb_buffer_init( 0 );
437 if( sync->first_frame )
439 /* This is our first frame */
440 if ( cur->start > pv->common->pts_offset )
443 * The first pts from a dvd should always be zero but
444 * can be non-zero with a transport or program stream since
445 * we're not guaranteed to start on an IDR frame. If we get
446 * a non-zero initial PTS extend its duration so it behaves
447 * as if it started at zero so that our audio timing will
450 hb_log( "sync: first pts is %"PRId64, cur->start );
451 cur->start = pv->common->pts_offset;
453 sync->first_frame = 0;
457 * since the first frame is always 0 and the upstream reader code
458 * is taking care of adjusting for pts discontinuities, we just have
459 * to deal with the next frame's start being in the past. This can
460 * happen when the PTS is adjusted after data loss but video frame
461 * reordering causes some frames with the old clock to appear after
462 * the clock change. This creates frames that overlap in time which
463 * looks to us like time going backward. The downstream muxing code
464 * can deal with overlaps of up to a frame time but anything larger
465 * we handle by dropping frames here.
467 hb_lock( pv->common->mutex );
468 if ( (int64_t)( next->start - pv->common->video_pts_slip - cur->start ) <= 0 )
470 if ( sync->first_drop == 0 )
472 sync->first_drop = next->start;
475 if (next->start - cur->start > 0)
477 sync->pts_skip += next->start - cur->start;
478 pv->common->video_pts_slip -= next->start - cur->start;
480 hb_unlock( pv->common->mutex );
481 if ( next->new_chap )
483 // don't drop a chapter mark when we drop the buffer
484 sync->chap_mark = next->new_chap;
486 hb_buffer_close( &next );
489 hb_unlock( pv->common->mutex );
490 if ( sync->first_drop )
492 hb_log( "sync: video time didn't advance - dropped %d frames "
493 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
494 sync->drop_count, (int)( cur->start - sync->first_drop ) / 90,
495 cur->start, next->start, (int)( next->start - cur->start ) );
496 sync->first_drop = 0;
497 sync->drop_count = 0;
501 * Track the video sequence number localy so that we can sync the audio
502 * to it using the sequence number as well as the PTS.
504 sync->video_sequence = cur->sequence;
507 * Look for a subtitle for this frame.
509 * If found then it will be tagged onto a video buffer of the correct time and
510 * sent in to the render pipeline. This only needs to be done for VOBSUBs which
511 * get rendered, other types of subtitles can just sit in their raw_queue until
512 * delt with at muxing.
514 for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
516 subtitle = hb_list_item( job->list_subtitle, i );
519 * Rewrite timestamps on subtitles that need it (on raw queue).
521 if( subtitle->source == CC608SUB ||
522 subtitle->source == CC708SUB ||
523 subtitle->source == SRTSUB )
526 * Rewrite timestamps on subtitles that came from Closed Captions
527 * since they are using the MPEG2 timestamps.
529 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
532 * Rewrite the timestamps as and when the video
533 * (cur->start) reaches the same timestamp as a
534 * closed caption (sub->start).
536 * What about discontinuity boundaries - not delt
539 * Bypass the sync fifo altogether.
543 sub = hb_fifo_get( subtitle->fifo_raw );
544 hb_fifo_push( subtitle->fifo_out, sub );
549 * Sync the subtitles to the incoming video, and use
550 * the matching converted video timestamp.
552 * Note that it doesn't appear that we need to convert
553 * timestamps, I guess that they were already correct,
554 * so just push them through for rendering.
557 if( sub->start < cur->start )
559 sub = hb_fifo_get( subtitle->fifo_raw );
560 hb_fifo_push( subtitle->fifo_out, sub );
569 if( subtitle->source == VOBSUB )
572 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
577 * EOF, pass it through immediately.
582 /* If two subtitles overlap, make the first one stop
583 when the second one starts */
584 sub2 = hb_fifo_see2( subtitle->fifo_raw );
585 if( sub2 && sub->stop > sub2->start )
587 sub->stop = sub2->start;
590 // hb_log("0x%x: video seq: %lld subtitle sequence: %lld",
591 // sub, cur->sequence, sub->sequence);
593 if( sub->sequence > cur->sequence )
596 * The video is behind where we are, so wait until
597 * it catches up to the same reader point on the
598 * DVD. Then our PTS should be in the same region
605 if( sub->stop > cur->start ) {
607 * The stop time is in the future, so fall through
608 * and we'll deal with it in the next block of
613 * There is a valid subtitle, is it time to display it?
615 if( sub->stop > sub->start)
618 * Normal subtitle which ends after it starts,
619 * check to see that the current video is between
622 if( cur->start > sub->start &&
623 cur->start < sub->stop )
626 * We should be playing this, so leave the
629 * fall through to display
631 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
634 * Subtitle is on for less than three
635 * seconds, extend the time that it is
636 * displayed to make it easier to read.
637 * Make it 3 seconds or until the next
638 * subtitle is displayed.
640 * This is in response to Indochine which
641 * only displays subs for 1 second -
644 sub->stop = sub->start + ( 2 * 90000 );
646 sub2 = hb_fifo_see2( subtitle->fifo_raw );
648 if( sub2 && sub->stop > sub2->start )
650 sub->stop = sub2->start;
657 * Defer until the play point is within
666 * The end of the subtitle is less than the start,
667 * this is a sign of a PTS discontinuity.
669 if( sub->start > cur->start )
672 * we haven't reached the start time yet, or
673 * we have jumped backwards after having
674 * already started this subtitle.
676 if( cur->start < sub->stop )
679 * We have jumped backwards and so should
680 * continue displaying this subtitle.
682 * fall through to display.
688 * Defer until the play point is
689 * within the subtitle
695 * Play this subtitle as the start is
696 * greater than our video point.
698 * fall through to display/
708 * The subtitle is older than this picture, trash it
710 sub = hb_fifo_get( subtitle->fifo_raw );
711 hb_buffer_close( &sub );
715 /* If we have a subtitle for this picture, copy it */
716 /* FIXME: we should avoid this memcpy */
721 if( subtitle->config.dest == RENDERSUB )
723 if ( cur->sub == NULL )
726 * Tack onto the video buffer for rendering
728 cur->sub = hb_buffer_init( sub->size );
729 cur->sub->x = sub->x;
730 cur->sub->y = sub->y;
731 cur->sub->width = sub->width;
732 cur->sub->height = sub->height;
733 memcpy( cur->sub->data, sub->data, sub->size );
737 * Pass-Through, pop it off of the raw queue,
739 sub = hb_fifo_get( subtitle->fifo_raw );
740 hb_fifo_push( subtitle->fifo_sync, sub );
744 * EOF - consume for rendered, else pass through
746 if( subtitle->config.dest == RENDERSUB )
748 sub = hb_fifo_get( subtitle->fifo_raw );
749 hb_buffer_close( &sub );
751 sub = hb_fifo_get( subtitle->fifo_raw );
752 hb_fifo_push( subtitle->fifo_sync, sub );
760 * Adjust the pts of the current frame so that it's contiguous
761 * with the previous frame. The start time of the current frame
762 * has to be the end time of the previous frame and the stop
763 * time has to be the start of the next frame. We don't
764 * make any adjustments to the source timestamps other than removing
765 * the clock offsets (which also removes pts discontinuities).
766 * This means we automatically encode at the source's frame rate.
767 * MP2 uses an implicit duration (frames end when the next frame
768 * starts) but more advanced containers like MP4 use an explicit
769 * duration. Since we're looking ahead one frame we set the
770 * explicit stop time from the start time of the next frame.
773 sync->cur = cur = next;
775 sync->next_pts = cur->start;
776 int64_t duration = cur->start - sync->pts_skip - (*buf_out)->start;
780 hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
781 duration, (*buf_out)->start, next->start );
784 (*buf_out)->start = sync->next_start;
785 sync->next_start += duration;
786 (*buf_out)->stop = sync->next_start;
788 if ( sync->chap_mark )
790 // we have a pending chapter mark from a recent drop - put it on this
791 // buffer (this may make it one frame late but we can't do any better).
792 (*buf_out)->new_chap = sync->chap_mark;
802 // sync*Init does nothing because sync has a special initializer
803 // that takes care of initializing video and all audio tracks
804 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
809 hb_work_object_t hb_sync_video =
812 "Video Synchronization",
818 /***********************************************************************
820 ***********************************************************************
822 **********************************************************************/
823 void syncAudioClose( hb_work_object_t * w )
825 hb_work_private_t * pv = w->private_data;
826 hb_sync_audio_t * sync = &pv->type.audio;
828 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
830 free( sync->ac3_buf );
834 src_delete( sync->state );
837 hb_lock( pv->common->mutex );
838 if ( --pv->common->ref == 0 )
840 hb_unlock( pv->common->mutex );
841 hb_lock_close( &pv->common->mutex );
846 hb_unlock( pv->common->mutex );
850 w->private_data = NULL;
853 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
858 /***********************************************************************
860 ***********************************************************************
862 **********************************************************************/
863 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
864 hb_buffer_t ** buf_out )
866 hb_work_private_t * pv = w->private_data;
867 hb_job_t * job = pv->job;
868 hb_sync_audio_t * sync = &pv->type.audio;
876 /* if the next buffer is an eof send it downstream */
877 if ( buf->size <= 0 )
879 hb_buffer_close( &buf );
880 *buf_out = hb_buffer_init( 0 );
884 /* Wait for start frame if doing point-to-point */
885 hb_lock( pv->common->mutex );
886 while ( !pv->common->start_found )
888 if ( buf->start < pv->common->audio_pts_thresh )
890 hb_buffer_close( &buf );
891 hb_unlock( pv->common->mutex );
894 while ( !pv->common->start_found &&
895 buf->start >= pv->common->audio_pts_thresh )
897 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
900 if ( buf->start < pv->common->audio_pts_thresh )
902 hb_buffer_close( &buf );
903 hb_unlock( pv->common->mutex );
906 hb_unlock( pv->common->mutex );
908 /* Wait till we can determine the initial pts of all streams */
909 if( pv->common->pts_offset == INT64_MIN )
911 pv->common->first_pts[sync->index+1] = buf->start;
912 hb_lock( pv->common->mutex );
913 while( pv->common->pts_offset == INT64_MIN )
915 // Full fifos will make us wait forever, so get the
916 // pts offset from the available streams if full
917 if (hb_fifo_is_full(w->fifo_in))
920 hb_cond_broadcast( pv->common->next_frame );
922 else if ( checkPtsOffset( w ) )
923 hb_cond_broadcast( pv->common->next_frame );
925 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
927 hb_unlock( pv->common->mutex );
930 if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
932 hb_buffer_close( &buf );
933 *buf_out = hb_buffer_init( 0 );
937 if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
939 hb_buffer_close( &buf );
940 *buf_out = hb_buffer_init( 0 );
944 hb_lock( pv->common->mutex );
945 start = buf->start - pv->common->audio_passthru_slip;
946 hb_unlock( pv->common->mutex );
947 if ( (int64_t)( start - sync->next_pts ) < 0 )
949 // audio time went backwards.
950 // If our output clock is more than a half frame ahead of the
951 // input clock drop this frame to move closer to sync.
952 // Otherwise drop frames until the input clock matches the output clock.
953 if ( sync->first_drop || sync->next_start - start > 90*15 )
955 // Discard data that's in the past.
956 if ( sync->first_drop == 0 )
958 sync->first_drop = sync->next_pts;
961 hb_buffer_close( &buf );
964 sync->next_pts = start;
966 if ( sync->first_drop )
968 // we were dropping old data but input buf time is now current
969 hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
970 "(next %"PRId64", current %"PRId64")", w->audio->id,
971 (int)( sync->next_pts - sync->first_drop ) / 90,
972 sync->drop_count, sync->first_drop, sync->next_pts );
973 sync->first_drop = 0;
974 sync->drop_count = 0;
975 sync->next_pts = start;
977 if ( start - sync->next_pts >= (90 * 70) )
979 if ( start - sync->next_pts > (90000LL * 60) )
981 // there's a gap of more than a minute between the last
982 // frame and this. assume we got a corrupted timestamp
983 // and just drop the next buf.
984 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
985 " start %"PRId64", next %"PRId64,
986 (int)((start - sync->next_pts) / (90000*60)),
987 w->audio->id, start, sync->next_pts );
988 hb_buffer_close( &buf );
992 * there's a gap of at least 70ms between the last
993 * frame we processed & the next. Fill it with silence.
994 * Or in the case of DCA, skip some frames from the
997 if( w->audio->config.out.codec == HB_ACODEC_DCA )
999 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
1000 " start %"PRId64", next %"PRId64,
1001 (int)((start - sync->next_pts) / 90),
1002 w->audio->id, start, sync->next_pts );
1003 hb_lock( pv->common->mutex );
1004 pv->common->audio_passthru_slip += (start - sync->next_pts);
1005 pv->common->video_pts_slip += (start - sync->next_pts);
1006 hb_unlock( pv->common->mutex );
1010 hb_log( "sync: adding %d ms of silence to audio %d"
1011 " start %"PRId64", next %"PRId64,
1012 (int)((start - sync->next_pts) / 90),
1013 w->audio->id, start, sync->next_pts );
1014 InsertSilence( w, start - sync->next_pts );
1020 * When we get here we've taken care of all the dups and gaps in the
1021 * audio stream and are ready to inject the next input frame into
1022 * the output stream.
1024 *buf_out = OutputAudioFrame( w->audio, buf, sync );
1028 hb_work_object_t hb_sync_audio =
1031 "AudioSynchronization",
1037 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
1039 hb_work_object_t * w;
1040 hb_work_private_t * pv;
1041 hb_title_t * title = job->title;
1042 hb_sync_audio_t * sync;
1044 pv = calloc( 1, sizeof( hb_work_private_t ) );
1045 sync = &pv->type.audio;
1048 pv->common = common;
1050 pv->common->pts_count++;
1052 w = hb_get_work( WORK_SYNC_AUDIO );
1053 w->private_data = pv;
1054 w->audio = hb_list_item( title->list_audio, i );
1055 w->fifo_in = w->audio->priv.fifo_raw;
1057 if( w->audio->config.out.codec == HB_ACODEC_AC3 ||
1058 w->audio->config.out.codec == HB_ACODEC_DCA )
1060 w->fifo_out = w->audio->priv.fifo_out;
1064 w->fifo_out = w->audio->priv.fifo_sync;
1067 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1069 /* Have a silent AC-3 frame ready in case we have to fill a
1075 codec = avcodec_find_encoder( CODEC_ID_AC3 );
1076 c = avcodec_alloc_context();
1078 c->bit_rate = w->audio->config.in.bitrate;
1079 c->sample_rate = w->audio->config.in.samplerate;
1080 c->channels = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
1082 if( hb_avcodec_open( c, codec ) < 0 )
1084 hb_log( "sync: avcodec_open failed" );
1088 zeros = calloc( AC3_SAMPLES_PER_FRAME *
1089 sizeof( short ) * c->channels, 1 );
1090 sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
1091 w->audio->config.in.samplerate / 8;
1092 sync->ac3_buf = malloc( sync->ac3_size );
1094 if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
1095 zeros ) != sync->ac3_size )
1097 hb_log( "sync: avcodec_encode_audio failed" );
1101 hb_avcodec_close( c );
1106 /* Initialize libsamplerate */
1108 sync->state = src_new( SRC_SINC_MEDIUM_QUALITY,
1109 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1110 w->audio->config.out.mixdown), &error );
1111 sync->data.end_of_input = 0;
1113 hb_list_add( job->list_work, w );
1116 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
1117 hb_sync_audio_t *sync )
1119 int64_t start = sync->next_start;
1120 int64_t duration = buf->stop - buf->start;
1122 sync->next_pts += duration;
1124 if( audio->config.in.samplerate == audio->config.out.samplerate ||
1125 audio->config.out.codec == HB_ACODEC_AC3 ||
1126 audio->config.out.codec == HB_ACODEC_DCA )
1129 * If we don't have to do sample rate conversion or this audio is
1130 * pass-thru just send the input buffer downstream after adjusting
1131 * its timestamps to make the output stream continuous.
1136 /* Not pass-thru - do sample rate conversion */
1137 int count_in, count_out;
1138 hb_buffer_t * buf_raw = buf;
1139 int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
1142 count_in = buf_raw->size / channel_count;
1144 * When using stupid rates like 44.1 there will always be some
1145 * truncation error. E.g., a 1536 sample AC3 frame will turn into a
1146 * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
1147 * the error will build up over time and eventually the audio will
1148 * substantially lag the video. libsamplerate will keep track of the
1149 * fractional sample & give it to us when appropriate if we give it
1150 * an extra sample of space in the output buffer.
1152 count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
1154 sync->data.input_frames = count_in;
1155 sync->data.output_frames = count_out;
1156 sync->data.src_ratio = (double)audio->config.out.samplerate /
1157 (double)audio->config.in.samplerate;
1159 buf = hb_buffer_init( count_out * channel_count );
1160 sync->data.data_in = (float *) buf_raw->data;
1161 sync->data.data_out = (float *) buf->data;
1162 if( src_process( sync->state, &sync->data ) )
1164 /* XXX If this happens, we're screwed */
1165 hb_log( "sync: audio %d src_process failed", audio->id );
1167 hb_buffer_close( &buf_raw );
1169 buf->size = sync->data.output_frames_gen * channel_count;
1170 duration = ( sync->data.output_frames_gen * 90000 ) /
1171 audio->config.out.samplerate;
1173 buf->frametype = HB_FRAME_AUDIO;
1175 buf->stop = start + duration;
1176 sync->next_start = start + duration;
1180 static void InsertSilence( hb_work_object_t * w, int64_t duration )
1182 hb_work_private_t * pv = w->private_data;
1183 hb_sync_audio_t *sync = &pv->type.audio;
1187 // to keep pass-thru and regular audio in sync we generate silence in
1188 // AC3 frame-sized units. If the silence duration isn't an integer multiple
1189 // of the AC3 frame duration we will truncate or round up depending on
1190 // which minimizes the timing error.
1191 const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1192 w->audio->config.in.samplerate;
1193 int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1195 while ( --frame_count >= 0 )
1197 if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1199 buf = hb_buffer_init( sync->ac3_size );
1200 buf->start = sync->next_pts;
1201 buf->stop = buf->start + frame_dur;
1202 memcpy( buf->data, sync->ac3_buf, buf->size );
1203 fifo = w->audio->priv.fifo_out;
1207 buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1208 HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1209 w->audio->config.out.mixdown) );
1210 buf->start = sync->next_pts;
1211 buf->stop = buf->start + frame_dur;
1212 memset( buf->data, 0, buf->size );
1213 fifo = w->audio->priv.fifo_sync;
1215 buf = OutputAudioFrame( w->audio, buf, sync );
1216 hb_fifo_push( fifo, buf );
1220 static void UpdateState( hb_work_object_t * w )
1222 hb_work_private_t * pv = w->private_data;
1223 hb_sync_video_t * sync = &pv->type.video;
1226 if( !pv->common->count_frames )
1228 sync->st_first = hb_get_date();
1229 pv->job->st_pause_date = -1;
1230 pv->job->st_paused = 0;
1232 pv->common->count_frames++;
1234 if( hb_get_date() > sync->st_dates[3] + 1000 )
1236 memmove( &sync->st_dates[0], &sync->st_dates[1],
1237 3 * sizeof( uint64_t ) );
1238 memmove( &sync->st_counts[0], &sync->st_counts[1],
1239 3 * sizeof( uint64_t ) );
1240 sync->st_dates[3] = hb_get_date();
1241 sync->st_counts[3] = pv->common->count_frames;
1244 #define p state.param.working
1245 state.state = HB_STATE_WORKING;
1246 p.progress = (float) pv->common->count_frames / (float) sync->count_frames_max;
1247 if( p.progress > 1.0 )
1251 p.rate_cur = 1000.0 *
1252 (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1253 (float) ( sync->st_dates[3] - sync->st_dates[0] );
1254 if( hb_get_date() > sync->st_first + 4000 )
1257 p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1258 (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1259 eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1261 p.hours = eta / 3600;
1262 p.minutes = ( eta % 3600 ) / 60;
1263 p.seconds = eta % 60;
1274 hb_set_state( pv->job->h, &state );
1277 static void UpdateSearchState( hb_work_object_t * w, int64_t start )
1279 hb_work_private_t * pv = w->private_data;
1280 hb_sync_video_t * sync = &pv->type.video;
1285 now = hb_get_date();
1286 if( !pv->common->count_frames )
1288 sync->st_first = now;
1289 pv->job->st_pause_date = -1;
1290 pv->job->st_paused = 0;
1292 pv->common->count_frames++;
1294 #define p state.param.working
1295 state.state = HB_STATE_SEARCHING;
1296 if ( pv->job->frame_to_start )
1297 p.progress = (float) pv->common->count_frames /
1298 (float) pv->job->frame_to_start;
1299 else if ( pv->job->pts_to_start )
1300 p.progress = (float) start / (float) pv->job->pts_to_start;
1303 if( p.progress > 1.0 )
1307 if (now > sync->st_first)
1311 if ( pv->job->frame_to_start )
1313 avg = 1000.0 * (double)pv->common->count_frames / (now - sync->st_first);
1314 eta = ( pv->job->frame_to_start - pv->common->count_frames ) / avg;
1316 else if ( pv->job->pts_to_start )
1318 avg = 1000.0 * (double)start / (now - sync->st_first);
1319 eta = ( pv->job->pts_to_start - start ) / avg;
1321 p.hours = eta / 3600;
1322 p.minutes = ( eta % 3600 ) / 60;
1323 p.seconds = eta % 60;
1334 hb_set_state( pv->job->h, &state );
1337 static void getPtsOffset( hb_work_object_t * w )
1339 hb_work_private_t * pv = w->private_data;
1341 int64_t first_pts = INT64_MAX;
1343 for( i = 0; i < pv->common->pts_count; i++ )
1345 if ( pv->common->first_pts[i] < first_pts )
1346 first_pts = pv->common->first_pts[i];
1348 pv->common->audio_passthru_slip = pv->common->pts_offset = first_pts;
1352 static int checkPtsOffset( hb_work_object_t * w )
1354 hb_work_private_t * pv = w->private_data;
1357 for( i = 0; i < pv->common->pts_count; i++ )
1359 if ( pv->common->first_pts[i] == INT64_MAX )