OSDN Git Service

fix silence insertion problem in audio sync and pipeline stall problem
[handbrake-jp/handbrake-jp-git.git] / libhb / sync.c
1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
2
3    This file is part of the HandBrake source code.
4    Homepage: <http://handbrake.fr/>.
5    It may be used under the terms of the GNU General Public License. */
6
7 #include "hb.h"
8 #include "hbffmpeg.h"
9 #include <stdio.h>
10 #include "samplerate.h"
11
12 #ifdef INT64_MIN
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
14 #endif
15 #define INT64_MIN (-9223372036854775807LL-1)
16
17 #define AC3_SAMPLES_PER_FRAME 1536
18
19 typedef struct
20 {
21     hb_lock_t * mutex;
22     int         ref;        /* Reference count to tell us when it's unused */
23     int         count_frames;
24     int64_t     audio_passthru_slip;
25     int64_t     video_pts_slip;
26     int64_t     pts_offset;
27
28     /* Frame based point-to-point support */
29     int64_t     audio_pts_thresh;
30     int         start_found;
31     hb_cond_t * next_frame;
32     int         pts_count;
33     int64_t   * first_pts;
34 } hb_sync_common_t;
35
36 typedef struct
37 {
38     int          index;
39     int64_t      next_start;    /* start time of next output frame */
40     int64_t      next_pts;     /* start time of next input frame */
41     int64_t      first_drop;   /* PTS of first 'went backwards' frame dropped */
42     int          drop_count;   /* count of 'time went backwards' drops */
43
44     /* Raw */
45     SRC_STATE  * state;
46     SRC_DATA     data;
47
48     /* AC-3 */
49     int          ac3_size;
50     uint8_t    * ac3_buf;
51 } hb_sync_audio_t;
52
53 typedef struct
54 {
55     /* Video */
56     int        first_frame;
57     int64_t    pts_skip;
58     int64_t    next_start;    /* start time of next output frame */
59     int64_t    next_pts;      /* start time of next input frame */
60     int64_t    first_drop;    /* PTS of first 'went backwards' frame dropped */
61     int        drop_count;    /* count of 'time went backwards' drops */
62     int        drops;         /* frames dropped to make a cbr video stream */
63     int        dups;          /* frames duplicated to make a cbr video stream */
64     int        video_sequence;
65     int        count_frames_max;
66     int        chap_mark;     /* to propagate chapter mark across a drop */
67     hb_buffer_t * cur;        /* The next picture to process */
68
69     /* Statistics */
70     uint64_t   st_counts[4];
71     uint64_t   st_dates[4];
72     uint64_t   st_first;
73 } hb_sync_video_t;
74
75 struct hb_work_private_s
76 {
77     hb_job_t * job;
78     hb_sync_common_t * common;
79     union
80     {
81         hb_sync_video_t video;
82         hb_sync_audio_t audio;
83     } type;
84 };
85
86 /***********************************************************************
87  * Local prototypes
88  **********************************************************************/
89 static void getPtsOffset( hb_work_object_t * w );
90 static int  checkPtsOffset( hb_work_object_t * w );
91 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
92 static void InsertSilence( hb_work_object_t * w, int64_t d );
93 static void UpdateState( hb_work_object_t * w );
94 static void UpdateSearchState( hb_work_object_t * w, int64_t start );
95 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
96                                        hb_sync_audio_t *sync );
97
98 /***********************************************************************
99  * hb_work_sync_init
100  ***********************************************************************
101  * Initialize the work object
102  **********************************************************************/
103 hb_work_object_t * hb_sync_init( hb_job_t * job )
104 {
105     hb_title_t        * title = job->title;
106     hb_chapter_t      * chapter;
107     int                 i;
108     uint64_t            duration;
109     hb_work_private_t * pv;
110     hb_sync_video_t   * sync;
111     hb_work_object_t  * w;
112     hb_work_object_t  * ret = NULL;
113
114     pv = calloc( 1, sizeof( hb_work_private_t ) );
115     sync = &pv->type.video;
116     pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
117     pv->common->ref++;
118     pv->common->mutex = hb_lock_init();
119     pv->common->audio_pts_thresh = 0;
120     pv->common->next_frame = hb_cond_init();
121     pv->common->pts_count = 1;
122     if ( job->frame_to_start || job->pts_to_start )
123     {
124         pv->common->start_found = 0;
125     }
126     else
127     {
128         pv->common->start_found = 1;
129     }
130
131     ret = w = hb_get_work( WORK_SYNC_VIDEO );
132     w->private_data = pv;
133     w->fifo_in = job->fifo_raw;
134     w->fifo_out = job->fifo_sync;
135
136     pv->job            = job;
137     pv->common->pts_offset   = INT64_MIN;
138     sync->first_frame = 1;
139
140     if( job->pass == 2 )
141     {
142         /* We already have an accurate frame count from pass 1 */
143         hb_interjob_t * interjob = hb_interjob_get( job->h );
144         sync->count_frames_max = interjob->frame_count;
145     }
146     else
147     {
148         /* Calculate how many video frames we are expecting */
149         if ( job->pts_to_stop )
150         {
151             duration = job->pts_to_stop + 90000;
152         }
153         else if( job->frame_to_stop )
154         {
155             /* Set the duration to a rough estimate */
156             duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
157         }
158         else
159         {
160             duration = 0;
161             for( i = job->chapter_start; i <= job->chapter_end; i++ )
162             {
163                 chapter   = hb_list_item( title->list_chapter, i - 1 );
164                 duration += chapter->duration;
165             }
166             duration += 90000;
167             /* 1 second safety so we're sure we won't miss anything */
168         }
169         sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
170     }
171
172     hb_log( "sync: expecting %d video frames", sync->count_frames_max );
173
174     /* Initialize libsamplerate for every audio track we have */
175     if ( ! job->indepth_scan )
176     {
177         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
178         {
179             InitAudio( job, pv->common, i );
180         }
181     }
182     pv->common->first_pts = malloc( sizeof(int64_t) * pv->common->pts_count );
183     for ( i = 0; i < pv->common->pts_count; i++ )
184         pv->common->first_pts[i] = INT64_MAX;
185
186     return ret;
187 }
188
189 /***********************************************************************
190  * Close Video
191  ***********************************************************************
192  *
193  **********************************************************************/
194 void syncVideoClose( hb_work_object_t * w )
195 {
196     hb_work_private_t * pv = w->private_data;
197     hb_job_t          * job   = pv->job;
198     hb_sync_video_t   * sync = &pv->type.video;
199
200     // Wake up audio sync if it's still waiting on condition.
201     pv->common->pts_offset = 0;
202     pv->common->start_found = 1;
203     hb_cond_broadcast( pv->common->next_frame );
204
205     if( sync->cur )
206     {
207         hb_buffer_close( &sync->cur );
208     }
209
210     hb_log( "sync: got %d frames, %d expected",
211             pv->common->count_frames, sync->count_frames_max );
212
213     /* save data for second pass */
214     if( job->pass == 1 )
215     {
216         /* Preserve frame count for better accuracy in pass 2 */
217         hb_interjob_t * interjob = hb_interjob_get( job->h );
218         interjob->frame_count = pv->common->count_frames;
219         interjob->last_job = job->sequence_id;
220         interjob->total_time = sync->next_start;
221     }
222
223     if (sync->drops || sync->dups )
224     {
225         hb_log( "sync: %d frames dropped, %d duplicated", 
226                 sync->drops, sync->dups );
227     }
228
229     hb_lock( pv->common->mutex );
230     if ( --pv->common->ref == 0 )
231     {
232         hb_unlock( pv->common->mutex );
233         hb_lock_close( &pv->common->mutex );
234         free( pv->common );
235     }
236     else
237     {
238         hb_unlock( pv->common->mutex );
239     }
240
241     free( pv );
242     w->private_data = NULL;
243 }
244
245 /***********************************************************************
246  * syncVideoWork
247  ***********************************************************************
248  *
249  **********************************************************************/
250 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
251               hb_buffer_t ** buf_out )
252 {
253     hb_buffer_t * cur, * next, * sub = NULL;
254     hb_work_private_t * pv = w->private_data;
255     hb_job_t          * job = pv->job;
256     hb_subtitle_t     * subtitle;
257     hb_sync_video_t   * sync = &pv->type.video;
258     int i;
259
260     *buf_out = NULL;
261     next = *buf_in;
262     *buf_in = NULL;
263
264     /* Wait for start of point-to-point encoding */
265     if( !pv->common->start_found )
266     {
267         hb_sync_video_t   * sync = &pv->type.video;
268
269         if( next->size == 0 )
270         {
271             *buf_out = next;
272             pv->common->start_found = 1;
273             hb_cond_broadcast( pv->common->next_frame );
274
275             /*
276              * Push through any subtitle EOFs in case they 
277              * were not synced through.
278              */
279             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
280             {
281                 subtitle = hb_list_item( job->list_subtitle, i );
282                 if( subtitle->config.dest == PASSTHRUSUB )
283                 {
284                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
285                 }
286             }
287             return HB_WORK_DONE;
288         }
289         if ( pv->common->count_frames < job->frame_to_start ||
290              next->start < job->pts_to_start )
291         {
292             // Flush any subtitles that have pts prior to the
293             // current frame
294             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
295             {
296                 subtitle = hb_list_item( job->list_subtitle, i );
297                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
298                 {
299                     if ( sub->start > next->start )
300                         break;
301                     sub = hb_fifo_get( subtitle->fifo_raw );
302                     hb_buffer_close( &sub );
303                 }
304             }
305             hb_lock( pv->common->mutex );
306             // Tell the audio threads what must be dropped
307             pv->common->audio_pts_thresh = next->start;
308             hb_cond_broadcast( pv->common->next_frame );
309             hb_unlock( pv->common->mutex );
310
311             UpdateSearchState( w, next->start );
312             hb_buffer_close( &next );
313
314             return HB_WORK_OK;
315         }
316         hb_lock( pv->common->mutex );
317         pv->common->start_found = 1;
318         pv->common->count_frames = 0;
319         hb_cond_broadcast( pv->common->next_frame );
320         hb_unlock( pv->common->mutex );
321         sync->st_first = 0;
322     }
323
324     /* Wait till we can determine the initial pts of all streams */
325     if( pv->common->pts_offset == INT64_MIN )
326     {
327         pv->common->first_pts[0] = next->start;
328         hb_lock( pv->common->mutex );
329         while( pv->common->pts_offset == INT64_MIN )
330         {
331             // Full fifos will make us wait forever, so get the
332             // pts offset from the available streams if full
333             if ( hb_fifo_is_full( job->fifo_raw ) )
334             {
335                 getPtsOffset( w );
336                 hb_cond_broadcast( pv->common->next_frame );
337             }
338             else if ( checkPtsOffset( w ) )
339                 hb_cond_broadcast( pv->common->next_frame );
340             else
341                 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
342         }
343         hb_unlock( pv->common->mutex );
344     }
345
346     if( !sync->cur )
347     {
348         sync->cur = next;
349         if( sync->cur->size == 0 )
350         {
351             /* we got an end-of-stream as our first video packet? 
352              * Feed it downstream & signal that we're done. 
353              */
354             *buf_out = hb_buffer_init( 0 );
355
356             pv->common->start_found = 1;
357             hb_cond_broadcast( pv->common->next_frame );
358
359             /*
360              * Push through any subtitle EOFs in case they 
361              * were not synced through.
362              */
363             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
364             {
365                 subtitle = hb_list_item( job->list_subtitle, i );
366                 if( subtitle->config.dest == PASSTHRUSUB )
367                 {
368                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
369                 }
370             }
371             return HB_WORK_DONE;
372         }
373         return HB_WORK_OK;
374     }
375     cur = sync->cur;
376     /* At this point we have a frame to process. Let's check
377         1) if we will be able to push into the fifo ahead
378         2) if the next frame is there already, since we need it to
379            compute the duration of the current frame*/
380     if( next->size == 0 )
381     {
382         hb_buffer_close( &next );
383
384         cur->start = sync->next_start;
385         cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
386
387         /* Push the frame to the renderer */
388         hb_fifo_push( job->fifo_sync, cur );
389         sync->cur = NULL;
390
391         /* we got an end-of-stream. Feed it downstream & signal that
392          * we're done. Note that this means we drop the final frame of
393          * video (we don't know its duration). On DVDs the final frame
394          * is often strange and dropping it seems to be a good idea. */
395         *buf_out = hb_buffer_init( 0 );
396
397         /*
398          * Push through any subtitle EOFs in case they were not synced through.
399          */
400         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
401         {
402             subtitle = hb_list_item( job->list_subtitle, i );
403             if( subtitle->config.dest == PASSTHRUSUB )
404             {
405                 if( subtitle->source == VOBSUB ) 
406                     hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
407                 else
408                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
409             }
410         }
411         pv->common->start_found = 1;
412         hb_cond_broadcast( pv->common->next_frame );
413         return HB_WORK_DONE;
414     }
415
416     /* Check for end of point-to-point frame encoding */
417     if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
418     {
419         // Drop an empty buffer into our output to ensure that things
420         // get flushed all the way out.
421         hb_buffer_close( &sync->cur );
422         hb_buffer_close( &next );
423         *buf_out = hb_buffer_init( 0 );
424         hb_log( "sync: reached %d frames, exiting early",
425                 pv->common->count_frames );
426         return HB_WORK_DONE;
427     }
428
429     /* Check for end of point-to-point pts encoding */
430     if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
431     {
432         // Drop an empty buffer into our output to ensure that things
433         // get flushed all the way out.
434         hb_log( "sync: reached pts %"PRId64", exiting early",
435                 sync->cur->start );
436         hb_buffer_close( &sync->cur );
437         hb_buffer_close( &next );
438         *buf_out = hb_buffer_init( 0 );
439         return HB_WORK_DONE;
440     }
441
442     if( sync->first_frame )
443     {
444         /* This is our first frame */
445         if ( cur->start > pv->common->pts_offset )
446         {
447             /*
448              * The first pts from a dvd should always be zero but
449              * can be non-zero with a transport or program stream since
450              * we're not guaranteed to start on an IDR frame. If we get
451              * a non-zero initial PTS extend its duration so it behaves
452              * as if it started at zero so that our audio timing will
453              * be in sync.
454              */
455             hb_log( "sync: first pts is %"PRId64, cur->start );
456             cur->start = pv->common->pts_offset;
457         }
458         sync->first_frame = 0;
459     }
460
461     /*
462      * since the first frame is always 0 and the upstream reader code
463      * is taking care of adjusting for pts discontinuities, we just have
464      * to deal with the next frame's start being in the past. This can
465      * happen when the PTS is adjusted after data loss but video frame
466      * reordering causes some frames with the old clock to appear after
467      * the clock change. This creates frames that overlap in time which
468      * looks to us like time going backward. The downstream muxing code
469      * can deal with overlaps of up to a frame time but anything larger
470      * we handle by dropping frames here.
471      */
472     hb_lock( pv->common->mutex );
473     if ( (int64_t)( next->start - pv->common->video_pts_slip - cur->start ) <= 0 )
474     {
475         if ( sync->first_drop == 0 )
476         {
477             sync->first_drop = next->start;
478         }
479         ++sync->drop_count;
480         if (next->start - cur->start > 0)
481         {
482             sync->pts_skip += next->start - cur->start;
483             pv->common->video_pts_slip -= next->start - cur->start;
484         }
485         hb_unlock( pv->common->mutex );
486         if ( next->new_chap )
487         {
488             // don't drop a chapter mark when we drop the buffer
489             sync->chap_mark = next->new_chap;
490         }
491         hb_buffer_close( &next );
492         return HB_WORK_OK;
493     }
494     hb_unlock( pv->common->mutex );
495     if ( sync->first_drop )
496     {
497         hb_log( "sync: video time didn't advance - dropped %d frames "
498                 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
499                 sync->drop_count, (int)( cur->start - sync->first_drop ) / 90,
500                 cur->start, next->start, (int)( next->start - cur->start ) );
501         sync->first_drop = 0;
502         sync->drop_count = 0;
503     }
504
505     /*
506      * Track the video sequence number localy so that we can sync the audio
507      * to it using the sequence number as well as the PTS.
508      */
509     sync->video_sequence = cur->sequence;
510
511     /*
512      * Look for a subtitle for this frame.
513      *
514      * If found then it will be tagged onto a video buffer of the correct time and 
515      * sent in to the render pipeline. This only needs to be done for VOBSUBs which
516      * get rendered, other types of subtitles can just sit in their raw_queue until
517      * delt with at muxing.
518      */
519     for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
520     {
521         subtitle = hb_list_item( job->list_subtitle, i );
522
523         /*
524          * Rewrite timestamps on subtitles that need it (on raw queue).
525          */
526         if( subtitle->source == CC608SUB ||
527             subtitle->source == CC708SUB ||
528             subtitle->source == SRTSUB )
529         {
530             /*
531              * Rewrite timestamps on subtitles that came from Closed Captions
532              * since they are using the MPEG2 timestamps.
533              */
534             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
535             {
536                 /*
537                  * Rewrite the timestamps as and when the video
538                  * (cur->start) reaches the same timestamp as a
539                  * closed caption (sub->start).
540                  *
541                  * What about discontinuity boundaries - not delt
542                  * with here - Van?
543                  *
544                  * Bypass the sync fifo altogether.
545                  */
546                 if( sub->size <= 0 )
547                 {
548                     sub = hb_fifo_get( subtitle->fifo_raw );
549                     hb_fifo_push( subtitle->fifo_out, sub );
550                     sub = NULL;
551                     break;
552                 } else {
553                     /*
554                      * Sync the subtitles to the incoming video, and use
555                      * the matching converted video timestamp.
556                      *
557                      * Note that it doesn't appear that we need to convert 
558                      * timestamps, I guess that they were already correct,
559                      * so just push them through for rendering.
560                      *
561                      */
562                     if( sub->start < cur->start )
563                     {
564                         sub = hb_fifo_get( subtitle->fifo_raw );
565                         hb_fifo_push( subtitle->fifo_out, sub );
566                     } else {
567                         sub = NULL;
568                         break;
569                     }
570                 }
571             }
572         }
573
574         if( subtitle->source == VOBSUB ) 
575         {
576             hb_buffer_t * sub2;
577             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
578             {
579                 if( sub->size == 0 )
580                 {
581                     /*
582                      * EOF, pass it through immediately.
583                      */
584                     break;
585                 }
586
587                 /* If two subtitles overlap, make the first one stop
588                    when the second one starts */
589                 sub2 = hb_fifo_see2( subtitle->fifo_raw );
590                 if( sub2 && sub->stop > sub2->start )
591                 {
592                     sub->stop = sub2->start;
593                 }
594                 
595                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
596                 //       sub, cur->sequence, sub->sequence);
597                 
598                 if( sub->sequence > cur->sequence )
599                 {
600                     /*
601                      * The video is behind where we are, so wait until
602                      * it catches up to the same reader point on the
603                      * DVD. Then our PTS should be in the same region
604                      * as the video.
605                      */
606                     sub = NULL;
607                     break;
608                 }
609                 
610                 if( sub->stop > cur->start ) {
611                     /*
612                      * The stop time is in the future, so fall through
613                      * and we'll deal with it in the next block of
614                      * code.
615                      */
616
617                     /*
618                      * There is a valid subtitle, is it time to display it?
619                      */
620                     if( sub->stop > sub->start)
621                     {
622                         /*
623                          * Normal subtitle which ends after it starts, 
624                          * check to see that the current video is between 
625                          * the start and end.
626                          */
627                         if( cur->start > sub->start &&
628                             cur->start < sub->stop )
629                         {
630                             /*
631                             * We should be playing this, so leave the
632                             * subtitle in place.
633                             *
634                             * fall through to display
635                             */
636                             if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
637                             {
638                                 /*
639                                  * Subtitle is on for less than three 
640                                  * seconds, extend the time that it is 
641                                  * displayed to make it easier to read. 
642                                  * Make it 3 seconds or until the next
643                                  * subtitle is displayed.
644                                  *
645                                  * This is in response to Indochine which 
646                                  * only displays subs for 1 second - 
647                                  * too fast to read.
648                                  */
649                                 sub->stop = sub->start + ( 2 * 90000 );
650                             
651                                 sub2 = hb_fifo_see2( subtitle->fifo_raw );
652                             
653                                 if( sub2 && sub->stop > sub2->start )
654                                 {
655                                     sub->stop = sub2->start;
656                                 }
657                             }
658                         }
659                         else
660                         {
661                             /*
662                              * Defer until the play point is within 
663                              * the subtitle
664                              */
665                             sub = NULL;
666                         }
667                     }
668                     else
669                     {
670                         /*
671                          * The end of the subtitle is less than the start, 
672                          * this is a sign of a PTS discontinuity.
673                          */
674                         if( sub->start > cur->start )
675                         {
676                             /*
677                              * we haven't reached the start time yet, or
678                              * we have jumped backwards after having
679                              * already started this subtitle.
680                              */
681                             if( cur->start < sub->stop )
682                             {
683                                 /*
684                                  * We have jumped backwards and so should
685                                  * continue displaying this subtitle.
686                                  *
687                                  * fall through to display.
688                                  */
689                             }
690                             else
691                             {
692                                 /*
693                                  * Defer until the play point is 
694                                  * within the subtitle
695                                  */
696                                 sub = NULL;
697                             }
698                         } else {
699                             /*
700                             * Play this subtitle as the start is 
701                             * greater than our video point.
702                             *
703                             * fall through to display/
704                             */
705                         }
706                     }
707                         break;
708                 }
709                 else
710                 {
711                     
712                     /*
713                      * The subtitle is older than this picture, trash it
714                      */
715                     sub = hb_fifo_get( subtitle->fifo_raw );
716                     hb_buffer_close( &sub );
717                 }
718             }
719             
720             /* If we have a subtitle for this picture, copy it */
721             /* FIXME: we should avoid this memcpy */
722             if( sub )
723             {
724                 if( sub->size > 0 )
725                 {
726                     if( subtitle->config.dest == RENDERSUB )
727                     {
728                         if ( cur->sub == NULL )
729                         {
730                             /*
731                              * Tack onto the video buffer for rendering
732                              */
733                             cur->sub         = hb_buffer_init( sub->size );
734                             cur->sub->x      = sub->x;
735                             cur->sub->y      = sub->y;
736                             cur->sub->width  = sub->width;
737                             cur->sub->height = sub->height;
738                             memcpy( cur->sub->data, sub->data, sub->size ); 
739                         }
740                     } else {
741                         /*
742                          * Pass-Through, pop it off of the raw queue, 
743                          */
744                         sub = hb_fifo_get( subtitle->fifo_raw );
745                         hb_fifo_push( subtitle->fifo_sync, sub );
746                     }
747                 } else {
748                     /*
749                     * EOF - consume for rendered, else pass through
750                     */
751                     if( subtitle->config.dest == RENDERSUB )
752                     {
753                         sub = hb_fifo_get( subtitle->fifo_raw );
754                         hb_buffer_close( &sub );
755                     } else {
756                         sub = hb_fifo_get( subtitle->fifo_raw );
757                         hb_fifo_push( subtitle->fifo_sync, sub );
758                     }
759                 }
760             }
761         }
762     } // end subtitles
763
764     /*
765      * Adjust the pts of the current frame so that it's contiguous
766      * with the previous frame. The start time of the current frame
767      * has to be the end time of the previous frame and the stop
768      * time has to be the start of the next frame.  We don't
769      * make any adjustments to the source timestamps other than removing
770      * the clock offsets (which also removes pts discontinuities).
771      * This means we automatically encode at the source's frame rate.
772      * MP2 uses an implicit duration (frames end when the next frame
773      * starts) but more advanced containers like MP4 use an explicit
774      * duration. Since we're looking ahead one frame we set the
775      * explicit stop time from the start time of the next frame.
776      */
777     *buf_out = cur;
778     sync->cur = cur = next;
779     cur->sub = NULL;
780     sync->next_pts = cur->start;
781     int64_t duration = cur->start - sync->pts_skip - (*buf_out)->start;
782     sync->pts_skip = 0;
783     if ( duration <= 0 )
784     {
785         hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
786                 duration, (*buf_out)->start, next->start );
787     }
788
789     (*buf_out)->start = sync->next_start;
790     sync->next_start += duration;
791     (*buf_out)->stop = sync->next_start;
792
793     if ( sync->chap_mark )
794     {
795         // we have a pending chapter mark from a recent drop - put it on this
796         // buffer (this may make it one frame late but we can't do any better).
797         (*buf_out)->new_chap = sync->chap_mark;
798         sync->chap_mark = 0;
799     }
800
801     /* Update UI */
802     UpdateState( w );
803
804     return HB_WORK_OK;
805 }
806
807 // sync*Init does nothing because sync has a special initializer
808 // that takes care of initializing video and all audio tracks
809 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
810 {
811     return 0;
812 }
813
814 hb_work_object_t hb_sync_video =
815 {
816     WORK_SYNC_VIDEO,
817     "Video Synchronization",
818     syncVideoInit,
819     syncVideoWork,
820     syncVideoClose
821 };
822
823 /***********************************************************************
824  * Close Audio
825  ***********************************************************************
826  *
827  **********************************************************************/
828 void syncAudioClose( hb_work_object_t * w )
829 {
830     hb_work_private_t * pv    = w->private_data;
831     hb_sync_audio_t   * sync  = &pv->type.audio;
832
833     if( w->audio->config.out.codec == HB_ACODEC_AC3 )
834     {
835         free( sync->ac3_buf );
836     }
837     else
838     {
839         src_delete( sync->state );
840     }
841
842     hb_lock( pv->common->mutex );
843     if ( --pv->common->ref == 0 )
844     {
845         hb_unlock( pv->common->mutex );
846         hb_lock_close( &pv->common->mutex );
847         free( pv->common );
848     }
849     else
850     {
851         hb_unlock( pv->common->mutex );
852     }
853
854     free( pv );
855     w->private_data = NULL;
856 }
857
858 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
859 {
860     return 0;
861 }
862
863 /***********************************************************************
864  * SyncAudio
865  ***********************************************************************
866  *
867  **********************************************************************/
868 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
869                        hb_buffer_t ** buf_out )
870 {
871     hb_work_private_t * pv = w->private_data;
872     hb_job_t        * job = pv->job;
873     hb_sync_audio_t * sync = &pv->type.audio;
874     hb_buffer_t     * buf;
875     int64_t start;
876
877     *buf_out = NULL;
878     buf = *buf_in;
879     *buf_in = NULL;
880     /* if the next buffer is an eof send it downstream */
881     if ( buf->size <= 0 )
882     {
883         hb_buffer_close( &buf );
884         *buf_out = hb_buffer_init( 0 );
885         return HB_WORK_DONE;
886     }
887
888     /* Wait for start frame if doing point-to-point */
889     hb_lock( pv->common->mutex );
890     while ( !pv->common->start_found )
891     {
892         if ( buf->start < pv->common->audio_pts_thresh )
893         {
894             hb_buffer_close( &buf );
895             hb_unlock( pv->common->mutex );
896             return HB_WORK_OK;
897         }
898         while ( !pv->common->start_found && 
899                 buf->start >= pv->common->audio_pts_thresh )
900         {
901             hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
902         }
903     }
904     if ( buf->start < pv->common->audio_pts_thresh )
905     {
906         hb_buffer_close( &buf );
907         hb_unlock( pv->common->mutex );
908         return HB_WORK_OK;
909     }
910     hb_unlock( pv->common->mutex );
911
912     /* Wait till we can determine the initial pts of all streams */
913     if( pv->common->pts_offset == INT64_MIN )
914     {
915         pv->common->first_pts[sync->index+1] = buf->start;
916         hb_lock( pv->common->mutex );
917         while( pv->common->pts_offset == INT64_MIN )
918         {
919             // Full fifos will make us wait forever, so get the
920             // pts offset from the available streams if full
921             if (hb_fifo_is_full(w->fifo_in))
922             {
923                 getPtsOffset( w );
924                 hb_cond_broadcast( pv->common->next_frame );
925             }
926             else if ( checkPtsOffset( w ) )
927                 hb_cond_broadcast( pv->common->next_frame );
928             else
929                 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
930         }
931         hb_unlock( pv->common->mutex );
932     }
933
934     if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
935     {
936         hb_buffer_close( &buf );
937         *buf_out = hb_buffer_init( 0 );
938         return HB_WORK_DONE;
939     }
940
941     if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
942     {
943         hb_buffer_close( &buf );
944         *buf_out = hb_buffer_init( 0 );
945         return HB_WORK_DONE;
946     }
947
948     hb_lock( pv->common->mutex );
949     start = buf->start - pv->common->audio_passthru_slip;
950     hb_unlock( pv->common->mutex );
951     if ( (int64_t)( start - sync->next_pts ) < 0 )
952     {
953         // audio time went backwards.
954         // If our output clock is more than a half frame ahead of the
955         // input clock drop this frame to move closer to sync.
956         // Otherwise drop frames until the input clock matches the output clock.
957         if ( sync->first_drop || sync->next_start - start > 90*15 )
958         {
959             // Discard data that's in the past.
960             if ( sync->first_drop == 0 )
961             {
962                 sync->first_drop = sync->next_pts;
963             }
964             ++sync->drop_count;
965             hb_buffer_close( &buf );
966             return HB_WORK_OK;
967         }
968         sync->next_pts = start;
969     }
970     if ( sync->first_drop )
971     {
972         // we were dropping old data but input buf time is now current
973         hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
974                 "(next %"PRId64", current %"PRId64")", w->audio->id,
975                 (int)( sync->next_pts - sync->first_drop ) / 90,
976                 sync->drop_count, sync->first_drop, sync->next_pts );
977         sync->first_drop = 0;
978         sync->drop_count = 0;
979         sync->next_pts = start;
980     }
981     if ( start - sync->next_pts >= (90 * 70) )
982     {
983         if ( start - sync->next_pts > (90000LL * 60) )
984         {
985             // there's a gap of more than a minute between the last
986             // frame and this. assume we got a corrupted timestamp
987             // and just drop the next buf.
988             hb_log( "sync: %d minute time gap in audio %d - dropping buf"
989                     "  start %"PRId64", next %"PRId64,
990                     (int)((start - sync->next_pts) / (90000*60)),
991                     w->audio->id, start, sync->next_pts );
992             hb_buffer_close( &buf );
993             return HB_WORK_OK;
994         }
995         /*
996          * there's a gap of at least 70ms between the last
997          * frame we processed & the next. Fill it with silence.
998          * Or in the case of DCA, skip some frames from the
999          * other streams.
1000          */
1001         if( w->audio->config.out.codec == HB_ACODEC_DCA )
1002         {
1003             hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
1004                     "  start %"PRId64", next %"PRId64,
1005                     (int)((start - sync->next_pts) / 90),
1006                     w->audio->id, start, sync->next_pts );
1007             hb_lock( pv->common->mutex );
1008             pv->common->audio_passthru_slip += (start - sync->next_pts);
1009             pv->common->video_pts_slip += (start - sync->next_pts);
1010             hb_unlock( pv->common->mutex );
1011             *buf_out = buf;
1012             return HB_WORK_OK;
1013         }
1014         hb_log( "sync: adding %d ms of silence to audio %d"
1015                 "  start %"PRId64", next %"PRId64,
1016                 (int)((start - sync->next_pts) / 90),
1017                 w->audio->id, start, sync->next_pts );
1018         InsertSilence( w, start - sync->next_pts );
1019     }
1020
1021     /*
1022      * When we get here we've taken care of all the dups and gaps in the
1023      * audio stream and are ready to inject the next input frame into
1024      * the output stream.
1025      */
1026     *buf_out = OutputAudioFrame( w->audio, buf, sync );
1027     return HB_WORK_OK;
1028 }
1029
1030 hb_work_object_t hb_sync_audio =
1031 {
1032     WORK_SYNC_AUDIO,
1033     "AudioSynchronization",
1034     syncAudioInit,
1035     syncAudioWork,
1036     syncAudioClose
1037 };
1038
1039 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
1040 {
1041     hb_work_object_t  * w;
1042     hb_work_private_t * pv;
1043     hb_title_t        * title = job->title;
1044     hb_sync_audio_t   * sync;
1045
1046     pv = calloc( 1, sizeof( hb_work_private_t ) );
1047     sync = &pv->type.audio;
1048     sync->index = i;
1049     pv->job    = job;
1050     pv->common = common;
1051     pv->common->ref++;
1052     pv->common->pts_count++;
1053
1054     w = hb_get_work( WORK_SYNC_AUDIO );
1055     w->private_data = pv;
1056     w->audio = hb_list_item( title->list_audio, i );
1057     w->fifo_in = w->audio->priv.fifo_raw;
1058
1059     if( w->audio->config.out.codec == HB_ACODEC_AC3 ||
1060         w->audio->config.out.codec == HB_ACODEC_DCA )
1061     {
1062         w->fifo_out = w->audio->priv.fifo_out;
1063     }
1064     else
1065     {
1066         w->fifo_out = w->audio->priv.fifo_sync;
1067     }
1068
1069     if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1070     {
1071         /* Have a silent AC-3 frame ready in case we have to fill a
1072            gap */
1073         AVCodec        * codec;
1074         AVCodecContext * c;
1075         short          * zeros;
1076
1077         codec = avcodec_find_encoder( CODEC_ID_AC3 );
1078         c     = avcodec_alloc_context();
1079
1080         c->bit_rate    = w->audio->config.in.bitrate;
1081         c->sample_rate = w->audio->config.in.samplerate;
1082         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
1083
1084         if( hb_avcodec_open( c, codec ) < 0 )
1085         {
1086             hb_log( "sync: avcodec_open failed" );
1087             return;
1088         }
1089
1090         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
1091                                  sizeof( short ) * c->channels, 1 );
1092         sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
1093                              w->audio->config.in.samplerate / 8;
1094         sync->ac3_buf  = malloc( sync->ac3_size );
1095
1096         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
1097                                   zeros ) != sync->ac3_size )
1098         {
1099             hb_log( "sync: avcodec_encode_audio failed" );
1100         }
1101
1102         free( zeros );
1103         hb_avcodec_close( c );
1104         av_free( c );
1105     }
1106     else
1107     {
1108         /* Initialize libsamplerate */
1109         int error;
1110         sync->state = src_new( SRC_SINC_MEDIUM_QUALITY, 
1111             HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1112                 w->audio->config.out.mixdown), &error );
1113         sync->data.end_of_input = 0;
1114     }
1115     hb_list_add( job->list_work, w );
1116 }
1117
1118 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
1119                                        hb_sync_audio_t *sync )
1120 {
1121     int64_t start = sync->next_start;
1122     int64_t duration = buf->stop - buf->start;
1123
1124     sync->next_pts += duration;
1125
1126     if( audio->config.in.samplerate == audio->config.out.samplerate ||
1127         audio->config.out.codec == HB_ACODEC_AC3 ||
1128         audio->config.out.codec == HB_ACODEC_DCA )
1129     {
1130         /*
1131          * If we don't have to do sample rate conversion or this audio is 
1132          * pass-thru just send the input buffer downstream after adjusting
1133          * its timestamps to make the output stream continuous.
1134          */
1135     }
1136     else
1137     {
1138         /* Not pass-thru - do sample rate conversion */
1139         int count_in, count_out;
1140         hb_buffer_t * buf_raw = buf;
1141         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
1142                             sizeof( float );
1143
1144         count_in  = buf_raw->size / channel_count;
1145         /*
1146          * When using stupid rates like 44.1 there will always be some
1147          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
1148          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
1149          * the error will build up over time and eventually the audio will
1150          * substantially lag the video. libsamplerate will keep track of the
1151          * fractional sample & give it to us when appropriate if we give it
1152          * an extra sample of space in the output buffer.
1153          */
1154         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
1155
1156         sync->data.input_frames = count_in;
1157         sync->data.output_frames = count_out;
1158         sync->data.src_ratio = (double)audio->config.out.samplerate /
1159                                (double)audio->config.in.samplerate;
1160
1161         buf = hb_buffer_init( count_out * channel_count );
1162         sync->data.data_in  = (float *) buf_raw->data;
1163         sync->data.data_out = (float *) buf->data;
1164         if( src_process( sync->state, &sync->data ) )
1165         {
1166             /* XXX If this happens, we're screwed */
1167             hb_log( "sync: audio %d src_process failed", audio->id );
1168         }
1169         hb_buffer_close( &buf_raw );
1170
1171         buf->size = sync->data.output_frames_gen * channel_count;
1172         duration = ( sync->data.output_frames_gen * 90000 ) /
1173                    audio->config.out.samplerate;
1174     }
1175     buf->frametype = HB_FRAME_AUDIO;
1176     buf->start = start;
1177     buf->stop  = start + duration;
1178     sync->next_start = start + duration;
1179     return buf;
1180 }
1181
1182 static void InsertSilence( hb_work_object_t * w, int64_t duration )
1183 {
1184     hb_work_private_t * pv = w->private_data;
1185     hb_sync_audio_t *sync = &pv->type.audio;
1186     hb_buffer_t     *buf;
1187     hb_fifo_t       *fifo;
1188
1189     // to keep pass-thru and regular audio in sync we generate silence in
1190     // AC3 frame-sized units. If the silence duration isn't an integer multiple
1191     // of the AC3 frame duration we will truncate or round up depending on
1192     // which minimizes the timing error.
1193     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1194                           w->audio->config.in.samplerate;
1195     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1196
1197     while ( --frame_count >= 0 )
1198     {
1199         if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1200         {
1201             buf        = hb_buffer_init( sync->ac3_size );
1202             buf->start = sync->next_pts;
1203             buf->stop  = buf->start + frame_dur;
1204             memcpy( buf->data, sync->ac3_buf, buf->size );
1205             fifo = w->audio->priv.fifo_out;
1206         }
1207         else
1208         {
1209             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1210                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1211                                          w->audio->config.out.mixdown) );
1212             buf->start = sync->next_pts;
1213             buf->stop  = buf->start + frame_dur;
1214             memset( buf->data, 0, buf->size );
1215             fifo = w->audio->priv.fifo_sync;
1216         }
1217         buf = OutputAudioFrame( w->audio, buf, sync );
1218         hb_fifo_push( fifo, buf );
1219     }
1220 }
1221
1222 static void UpdateState( hb_work_object_t * w )
1223 {
1224     hb_work_private_t * pv = w->private_data;
1225     hb_sync_video_t   * sync = &pv->type.video;
1226     hb_state_t state;
1227
1228     if( !pv->common->count_frames )
1229     {
1230         sync->st_first = hb_get_date();
1231         pv->job->st_pause_date = -1;
1232         pv->job->st_paused = 0;
1233     }
1234     pv->common->count_frames++;
1235
1236     if( hb_get_date() > sync->st_dates[3] + 1000 )
1237     {
1238         memmove( &sync->st_dates[0], &sync->st_dates[1],
1239                  3 * sizeof( uint64_t ) );
1240         memmove( &sync->st_counts[0], &sync->st_counts[1],
1241                  3 * sizeof( uint64_t ) );
1242         sync->st_dates[3]  = hb_get_date();
1243         sync->st_counts[3] = pv->common->count_frames;
1244     }
1245
1246 #define p state.param.working
1247     state.state = HB_STATE_WORKING;
1248     p.progress  = (float) pv->common->count_frames / (float) sync->count_frames_max;
1249     if( p.progress > 1.0 )
1250     {
1251         p.progress = 1.0;
1252     }
1253     p.rate_cur   = 1000.0 *
1254         (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1255         (float) ( sync->st_dates[3] - sync->st_dates[0] );
1256     if( hb_get_date() > sync->st_first + 4000 )
1257     {
1258         int eta;
1259         p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1260             (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1261         eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1262             p.rate_avg;
1263         p.hours   = eta / 3600;
1264         p.minutes = ( eta % 3600 ) / 60;
1265         p.seconds = eta % 60;
1266     }
1267     else
1268     {
1269         p.rate_avg = 0.0;
1270         p.hours    = -1;
1271         p.minutes  = -1;
1272         p.seconds  = -1;
1273     }
1274 #undef p
1275
1276     hb_set_state( pv->job->h, &state );
1277 }
1278
1279 static void UpdateSearchState( hb_work_object_t * w, int64_t start )
1280 {
1281     hb_work_private_t * pv = w->private_data;
1282     hb_sync_video_t   * sync = &pv->type.video;
1283     hb_state_t state;
1284     uint64_t now;
1285     double avg;
1286
1287     now = hb_get_date();
1288     if( !pv->common->count_frames )
1289     {
1290         sync->st_first = now;
1291         pv->job->st_pause_date = -1;
1292         pv->job->st_paused = 0;
1293     }
1294     pv->common->count_frames++;
1295
1296 #define p state.param.working
1297     state.state = HB_STATE_SEARCHING;
1298     if ( pv->job->frame_to_start )
1299         p.progress  = (float) pv->common->count_frames / 
1300                       (float) pv->job->frame_to_start;
1301     else if ( pv->job->pts_to_start )
1302         p.progress  = (float) start / (float) pv->job->pts_to_start;
1303     else
1304         p.progress = 0;
1305     if( p.progress > 1.0 )
1306     {
1307         p.progress = 1.0;
1308     }
1309     if (now > sync->st_first)
1310     {
1311         int eta;
1312
1313         if ( pv->job->frame_to_start )
1314         {
1315             avg = 1000.0 * (double)pv->common->count_frames / (now - sync->st_first);
1316             eta = ( pv->job->frame_to_start - pv->common->count_frames ) / avg;
1317         }
1318         else if ( pv->job->pts_to_start )
1319         {
1320             avg = 1000.0 * (double)start / (now - sync->st_first);
1321             eta = ( pv->job->pts_to_start - start ) / avg;
1322         }
1323         p.hours   = eta / 3600;
1324         p.minutes = ( eta % 3600 ) / 60;
1325         p.seconds = eta % 60;
1326     }
1327     else
1328     {
1329         p.rate_avg = 0.0;
1330         p.hours    = -1;
1331         p.minutes  = -1;
1332         p.seconds  = -1;
1333     }
1334 #undef p
1335
1336     hb_set_state( pv->job->h, &state );
1337 }
1338
1339 static void getPtsOffset( hb_work_object_t * w )
1340 {
1341     hb_work_private_t * pv = w->private_data;
1342     int           i ;
1343     int64_t       first_pts = INT64_MAX;
1344
1345     for( i = 0; i < pv->common->pts_count; i++ )
1346     {
1347         if ( pv->common->first_pts[i] < first_pts )
1348             first_pts = pv->common->first_pts[i];
1349     }
1350     pv->common->audio_passthru_slip = pv->common->pts_offset = first_pts;
1351     return;
1352 }
1353
1354 static int checkPtsOffset( hb_work_object_t * w )
1355 {
1356     hb_work_private_t * pv = w->private_data;
1357     int           i ;
1358
1359     for( i = 0; i < pv->common->pts_count; i++ )
1360     {
1361         if ( pv->common->first_pts[i] == INT64_MAX )
1362             return 0;
1363     }
1364     getPtsOffset( w );
1365     return 1;
1366 }