OSDN Git Service

cli: add vbv settings to appletv preset
[handbrake-jp/handbrake-jp-git.git] / libhb / sync.c
1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
2
3    This file is part of the HandBrake source code.
4    Homepage: <http://handbrake.fr/>.
5    It may be used under the terms of the GNU General Public License. */
6
7 #include "hb.h"
8 #include "hbffmpeg.h"
9 #include <stdio.h>
10 #include "samplerate.h"
11
12 #ifdef INT64_MIN
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
14 #endif
15 #define INT64_MIN (-9223372036854775807LL-1)
16
17 #define AC3_SAMPLES_PER_FRAME 1536
18
19 typedef struct
20 {
21     hb_lock_t * mutex;
22     int         ref;        /* Reference count to tell us when it's unused */
23     int         count_frames;
24     int64_t     audio_passthru_slip;
25     int64_t     video_pts_slip;
26     int64_t     pts_offset;
27
28     /* Frame based point-to-point support */
29     int64_t     audio_pts_thresh;
30     int         start_found;
31     hb_cond_t * next_frame;
32     int         pts_count;
33     int64_t   * first_pts;
34 } hb_sync_common_t;
35
36 typedef struct
37 {
38     int          index;
39     int64_t      next_start;    /* start time of next output frame */
40     int64_t      next_pts;     /* start time of next input frame */
41     int64_t      first_drop;   /* PTS of first 'went backwards' frame dropped */
42     int          drop_count;   /* count of 'time went backwards' drops */
43
44     /* Raw */
45     SRC_STATE  * state;
46     SRC_DATA     data;
47
48     /* AC-3 */
49     int          ac3_size;
50     uint8_t    * ac3_buf;
51 } hb_sync_audio_t;
52
53 typedef struct
54 {
55     /* Video */
56     int        first_frame;
57     int64_t    pts_skip;
58     int64_t    next_start;    /* start time of next output frame */
59     int64_t    next_pts;      /* start time of next input frame */
60     int64_t    first_drop;    /* PTS of first 'went backwards' frame dropped */
61     int        drop_count;    /* count of 'time went backwards' drops */
62     int        drops;         /* frames dropped to make a cbr video stream */
63     int        dups;          /* frames duplicated to make a cbr video stream */
64     int        video_sequence;
65     int        count_frames_max;
66     int        chap_mark;     /* to propagate chapter mark across a drop */
67     hb_buffer_t * cur;        /* The next picture to process */
68
69     /* Statistics */
70     uint64_t   st_counts[4];
71     uint64_t   st_dates[4];
72     uint64_t   st_first;
73 } hb_sync_video_t;
74
75 struct hb_work_private_s
76 {
77     hb_job_t * job;
78     hb_sync_common_t * common;
79     union
80     {
81         hb_sync_video_t video;
82         hb_sync_audio_t audio;
83     } type;
84 };
85
86 /***********************************************************************
87  * Local prototypes
88  **********************************************************************/
89 static void getPtsOffset( hb_work_object_t * w );
90 static int  checkPtsOffset( hb_work_object_t * w );
91 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
92 static void InsertSilence( hb_work_object_t * w, int64_t d );
93 static void UpdateState( hb_work_object_t * w );
94 static void UpdateSearchState( hb_work_object_t * w, int64_t start );
95 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
96                                        hb_sync_audio_t *sync );
97
98 /***********************************************************************
99  * hb_work_sync_init
100  ***********************************************************************
101  * Initialize the work object
102  **********************************************************************/
103 int hb_sync_init( hb_job_t * job )
104 {
105     hb_title_t        * title = job->title;
106     hb_chapter_t      * chapter;
107     int                 i;
108     uint64_t            duration;
109     hb_work_private_t * pv;
110     hb_sync_video_t   * sync;
111     hb_work_object_t  * w;
112
113     pv = calloc( 1, sizeof( hb_work_private_t ) );
114     sync = &pv->type.video;
115     pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
116     pv->common->ref++;
117     pv->common->mutex = hb_lock_init();
118     pv->common->audio_pts_thresh = 0;
119     pv->common->next_frame = hb_cond_init();
120     pv->common->pts_count = 1;
121     if ( job->frame_to_start || job->pts_to_start )
122     {
123         pv->common->start_found = 0;
124     }
125     else
126     {
127         pv->common->start_found = 1;
128     }
129
130     w = hb_get_work( WORK_SYNC_VIDEO );
131     w->private_data = pv;
132     w->fifo_in = job->fifo_raw;
133     w->fifo_out = job->fifo_sync;
134
135     pv->job            = job;
136     pv->common->pts_offset   = INT64_MIN;
137     sync->first_frame = 1;
138
139     if( job->pass == 2 )
140     {
141         /* We already have an accurate frame count from pass 1 */
142         hb_interjob_t * interjob = hb_interjob_get( job->h );
143         sync->count_frames_max = interjob->frame_count;
144     }
145     else
146     {
147         /* Calculate how many video frames we are expecting */
148         if ( job->pts_to_stop )
149         {
150             duration = job->pts_to_stop + 90000;
151         }
152         else if( job->frame_to_stop )
153         {
154             /* Set the duration to a rough estimate */
155             duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
156         }
157         else
158         {
159             duration = 0;
160             for( i = job->chapter_start; i <= job->chapter_end; i++ )
161             {
162                 chapter   = hb_list_item( title->list_chapter, i - 1 );
163                 duration += chapter->duration;
164             }
165             duration += 90000;
166             /* 1 second safety so we're sure we won't miss anything */
167         }
168         sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
169     }
170     hb_list_add( job->list_work, w );
171
172     hb_log( "sync: expecting %d video frames", sync->count_frames_max );
173
174     /* Initialize libsamplerate for every audio track we have */
175     if ( ! job->indepth_scan )
176     {
177         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
178         {
179             InitAudio( job, pv->common, i );
180         }
181     }
182     pv->common->first_pts = malloc( sizeof(int64_t) * pv->common->pts_count );
183     for ( i = 0; i < pv->common->pts_count; i++ )
184         pv->common->first_pts[i] = INT64_MAX;
185
186     return 0;
187 }
188
189 /***********************************************************************
190  * Close Video
191  ***********************************************************************
192  *
193  **********************************************************************/
194 void syncVideoClose( hb_work_object_t * w )
195 {
196     hb_work_private_t * pv = w->private_data;
197     hb_job_t          * job   = pv->job;
198     hb_sync_video_t   * sync = &pv->type.video;
199
200     // Wake up audio sync if it's still waiting on condition.
201     pv->common->pts_offset = 0;
202     pv->common->start_found = 1;
203     hb_cond_broadcast( pv->common->next_frame );
204
205     if( sync->cur )
206     {
207         hb_buffer_close( &sync->cur );
208     }
209
210     hb_log( "sync: got %d frames, %d expected",
211             pv->common->count_frames, sync->count_frames_max );
212
213     /* save data for second pass */
214     if( job->pass == 1 )
215     {
216         /* Preserve frame count for better accuracy in pass 2 */
217         hb_interjob_t * interjob = hb_interjob_get( job->h );
218         interjob->frame_count = pv->common->count_frames;
219         interjob->last_job = job->sequence_id;
220         interjob->total_time = sync->next_start;
221     }
222
223     if (sync->drops || sync->dups )
224     {
225         hb_log( "sync: %d frames dropped, %d duplicated", 
226                 sync->drops, sync->dups );
227     }
228
229     hb_lock( pv->common->mutex );
230     if ( --pv->common->ref == 0 )
231     {
232         hb_unlock( pv->common->mutex );
233         hb_lock_close( &pv->common->mutex );
234         free( pv->common );
235     }
236     else
237     {
238         hb_unlock( pv->common->mutex );
239     }
240
241     free( pv );
242     w->private_data = NULL;
243 }
244
245 /***********************************************************************
246  * syncVideoWork
247  ***********************************************************************
248  *
249  **********************************************************************/
250 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
251               hb_buffer_t ** buf_out )
252 {
253     hb_buffer_t * cur, * next, * sub = NULL;
254     hb_work_private_t * pv = w->private_data;
255     hb_job_t          * job = pv->job;
256     hb_subtitle_t     * subtitle;
257     hb_sync_video_t   * sync = &pv->type.video;
258     int i;
259
260     *buf_out = NULL;
261     next = *buf_in;
262     *buf_in = NULL;
263
264     /* Wait for start of point-to-point encoding */
265     if( !pv->common->start_found )
266     {
267         hb_sync_video_t   * sync = &pv->type.video;
268
269         if( next->size == 0 )
270         {
271             *buf_out = next;
272             pv->common->start_found = 1;
273             hb_cond_broadcast( pv->common->next_frame );
274
275             /*
276              * Push through any subtitle EOFs in case they 
277              * were not synced through.
278              */
279             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
280             {
281                 subtitle = hb_list_item( job->list_subtitle, i );
282                 if( subtitle->config.dest == PASSTHRUSUB )
283                 {
284                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
285                 }
286             }
287             return HB_WORK_DONE;
288         }
289         if ( pv->common->count_frames < job->frame_to_start ||
290              next->start < job->pts_to_start )
291         {
292             // Flush any subtitles that have pts prior to the
293             // current frame
294             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
295             {
296                 subtitle = hb_list_item( job->list_subtitle, i );
297                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
298                 {
299                     if ( sub->start > next->start )
300                         break;
301                     sub = hb_fifo_get( subtitle->fifo_raw );
302                     hb_buffer_close( &sub );
303                 }
304             }
305             hb_lock( pv->common->mutex );
306             // Tell the audio threads what must be dropped
307             pv->common->audio_pts_thresh = next->start;
308             hb_cond_broadcast( pv->common->next_frame );
309             hb_unlock( pv->common->mutex );
310
311             UpdateSearchState( w, next->start );
312             hb_buffer_close( &next );
313
314             return HB_WORK_OK;
315         }
316         hb_lock( pv->common->mutex );
317         pv->common->start_found = 1;
318         pv->common->count_frames = 0;
319         hb_cond_broadcast( pv->common->next_frame );
320         hb_unlock( pv->common->mutex );
321         sync->st_first = 0;
322     }
323
324     /* Wait till we can determine the initial pts of all streams */
325     if( pv->common->pts_offset == INT64_MIN )
326     {
327         pv->common->first_pts[0] = next->start;
328         hb_lock( pv->common->mutex );
329         while( pv->common->pts_offset == INT64_MIN )
330         {
331             // Full fifos will make us wait forever, so get the
332             // pts offset from the available streams if full
333             if ( hb_fifo_is_full( job->fifo_raw ) )
334             {
335                 getPtsOffset( w );
336                 hb_cond_broadcast( pv->common->next_frame );
337             }
338             else if ( checkPtsOffset( w ) )
339                 hb_cond_broadcast( pv->common->next_frame );
340             else
341                 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
342         }
343         hb_unlock( pv->common->mutex );
344     }
345
346     if( !sync->cur )
347     {
348         sync->cur = next;
349         if( sync->cur->size == 0 )
350         {
351             /* we got an end-of-stream as our first video packet? 
352              * Feed it downstream & signal that we're done. 
353              */
354             *buf_out = hb_buffer_init( 0 );
355
356             pv->common->start_found = 1;
357             hb_cond_broadcast( pv->common->next_frame );
358
359             /*
360              * Push through any subtitle EOFs in case they 
361              * were not synced through.
362              */
363             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
364             {
365                 subtitle = hb_list_item( job->list_subtitle, i );
366                 if( subtitle->config.dest == PASSTHRUSUB )
367                 {
368                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
369                 }
370             }
371             return HB_WORK_DONE;
372         }
373         return HB_WORK_OK;
374     }
375     cur = sync->cur;
376     /* At this point we have a frame to process. Let's check
377         1) if we will be able to push into the fifo ahead
378         2) if the next frame is there already, since we need it to
379            compute the duration of the current frame*/
380     if( next->size == 0 )
381     {
382         hb_buffer_close( &next );
383
384         cur->start = sync->next_start;
385         cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
386
387         /* Push the frame to the renderer */
388         hb_fifo_push( job->fifo_sync, cur );
389         sync->cur = NULL;
390
391         /* we got an end-of-stream. Feed it downstream & signal that
392          * we're done. Note that this means we drop the final frame of
393          * video (we don't know its duration). On DVDs the final frame
394          * is often strange and dropping it seems to be a good idea. */
395         *buf_out = hb_buffer_init( 0 );
396
397         /*
398          * Push through any subtitle EOFs in case they were not synced through.
399          */
400         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
401         {
402             subtitle = hb_list_item( job->list_subtitle, i );
403             if( subtitle->config.dest == PASSTHRUSUB )
404             {
405                 if( subtitle->source == VOBSUB ) 
406                     hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
407                 else
408                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
409             }
410         }
411         pv->common->start_found = 1;
412         hb_cond_broadcast( pv->common->next_frame );
413         return HB_WORK_DONE;
414     }
415
416     /* Check for end of point-to-point frame encoding */
417     if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
418     {
419         // Drop an empty buffer into our output to ensure that things
420         // get flushed all the way out.
421         hb_buffer_close( &sync->cur );
422         hb_buffer_close( &next );
423         *buf_out = hb_buffer_init( 0 );
424         hb_log( "sync: reached %d frames, exiting early",
425                 pv->common->count_frames );
426         return HB_WORK_DONE;
427     }
428
429     /* Check for end of point-to-point pts encoding */
430     if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
431     {
432         // Drop an empty buffer into our output to ensure that things
433         // get flushed all the way out.
434         hb_log( "sync: reached pts %"PRId64", exiting early",
435                 sync->cur->start );
436         hb_buffer_close( &sync->cur );
437         hb_buffer_close( &next );
438         *buf_out = hb_buffer_init( 0 );
439         return HB_WORK_DONE;
440     }
441
442     if( sync->first_frame )
443     {
444         /* This is our first frame */
445         if ( cur->start > pv->common->pts_offset )
446         {
447             /*
448              * The first pts from a dvd should always be zero but
449              * can be non-zero with a transport or program stream since
450              * we're not guaranteed to start on an IDR frame. If we get
451              * a non-zero initial PTS extend its duration so it behaves
452              * as if it started at zero so that our audio timing will
453              * be in sync.
454              */
455             hb_log( "sync: first pts is %"PRId64, cur->start );
456             cur->start = pv->common->pts_offset;
457         }
458         sync->first_frame = 0;
459     }
460
461     /*
462      * since the first frame is always 0 and the upstream reader code
463      * is taking care of adjusting for pts discontinuities, we just have
464      * to deal with the next frame's start being in the past. This can
465      * happen when the PTS is adjusted after data loss but video frame
466      * reordering causes some frames with the old clock to appear after
467      * the clock change. This creates frames that overlap in time which
468      * looks to us like time going backward. The downstream muxing code
469      * can deal with overlaps of up to a frame time but anything larger
470      * we handle by dropping frames here.
471      */
472     hb_lock( pv->common->mutex );
473     if ( (int64_t)( next->start - pv->common->video_pts_slip - cur->start ) <= 0 )
474     {
475         if ( sync->first_drop == 0 )
476         {
477             sync->first_drop = next->start;
478         }
479         ++sync->drop_count;
480         if (next->start - cur->start > 0)
481         {
482             sync->pts_skip += next->start - cur->start;
483             pv->common->video_pts_slip -= next->start - cur->start;
484         }
485         hb_unlock( pv->common->mutex );
486         if ( next->new_chap )
487         {
488             // don't drop a chapter mark when we drop the buffer
489             sync->chap_mark = next->new_chap;
490         }
491         hb_buffer_close( &next );
492         return HB_WORK_OK;
493     }
494     hb_unlock( pv->common->mutex );
495     if ( sync->first_drop )
496     {
497         hb_log( "sync: video time didn't advance - dropped %d frames "
498                 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
499                 sync->drop_count, (int)( cur->start - sync->first_drop ) / 90,
500                 cur->start, next->start, (int)( next->start - cur->start ) );
501         sync->first_drop = 0;
502         sync->drop_count = 0;
503     }
504
505     /*
506      * Track the video sequence number localy so that we can sync the audio
507      * to it using the sequence number as well as the PTS.
508      */
509     sync->video_sequence = cur->sequence;
510
511     /*
512      * Look for a subtitle for this frame.
513      *
514      * If found then it will be tagged onto a video buffer of the correct time and 
515      * sent in to the render pipeline. This only needs to be done for VOBSUBs which
516      * get rendered, other types of subtitles can just sit in their raw_queue until
517      * delt with at muxing.
518      */
519     for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
520     {
521         subtitle = hb_list_item( job->list_subtitle, i );
522
523         /*
524          * Rewrite timestamps on subtitles that need it (on raw queue).
525          */
526         if( subtitle->source == CC608SUB ||
527             subtitle->source == CC708SUB ||
528             subtitle->source == SRTSUB )
529         {
530             /*
531              * Rewrite timestamps on subtitles that came from Closed Captions
532              * since they are using the MPEG2 timestamps.
533              */
534             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
535             {
536                 /*
537                  * Rewrite the timestamps as and when the video
538                  * (cur->start) reaches the same timestamp as a
539                  * closed caption (sub->start).
540                  *
541                  * What about discontinuity boundaries - not delt
542                  * with here - Van?
543                  *
544                  * Bypass the sync fifo altogether.
545                  */
546                 if( sub->size <= 0 )
547                 {
548                     sub = hb_fifo_get( subtitle->fifo_raw );
549                     hb_fifo_push( subtitle->fifo_out, sub );
550                     sub = NULL;
551                     break;
552                 } else {
553                     /*
554                      * Sync the subtitles to the incoming video, and use
555                      * the matching converted video timestamp.
556                      *
557                      * Note that it doesn't appear that we need to convert 
558                      * timestamps, I guess that they were already correct,
559                      * so just push them through for rendering.
560                      *
561                      */
562                     if( sub->start < cur->start )
563                     {
564                         sub = hb_fifo_get( subtitle->fifo_raw );
565                         hb_fifo_push( subtitle->fifo_out, sub );
566                     } else {
567                         sub = NULL;
568                         break;
569                     }
570                 }
571             }
572         }
573
574         if( subtitle->source == VOBSUB ) 
575         {
576             hb_buffer_t * sub2;
577             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
578             {
579                 if( sub->size == 0 )
580                 {
581                     /*
582                      * EOF, pass it through immediately.
583                      */
584                     break;
585                 }
586
587                 /* If two subtitles overlap, make the first one stop
588                    when the second one starts */
589                 sub2 = hb_fifo_see2( subtitle->fifo_raw );
590                 if( sub2 && sub->stop > sub2->start )
591                 {
592                     sub->stop = sub2->start;
593                 }
594                 
595                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
596                 //       sub, cur->sequence, sub->sequence);
597                 
598                 if( sub->sequence > cur->sequence )
599                 {
600                     /*
601                      * The video is behind where we are, so wait until
602                      * it catches up to the same reader point on the
603                      * DVD. Then our PTS should be in the same region
604                      * as the video.
605                      */
606                     sub = NULL;
607                     break;
608                 }
609                 
610                 if( sub->stop > cur->start ) {
611                     /*
612                      * The stop time is in the future, so fall through
613                      * and we'll deal with it in the next block of
614                      * code.
615                      */
616
617                     /*
618                      * There is a valid subtitle, is it time to display it?
619                      */
620                     if( sub->stop > sub->start)
621                     {
622                         /*
623                          * Normal subtitle which ends after it starts, 
624                          * check to see that the current video is between 
625                          * the start and end.
626                          */
627                         if( cur->start > sub->start &&
628                             cur->start < sub->stop )
629                         {
630                             /*
631                             * We should be playing this, so leave the
632                             * subtitle in place.
633                             *
634                             * fall through to display
635                             */
636                             if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
637                             {
638                                 /*
639                                  * Subtitle is on for less than three 
640                                  * seconds, extend the time that it is 
641                                  * displayed to make it easier to read. 
642                                  * Make it 3 seconds or until the next
643                                  * subtitle is displayed.
644                                  *
645                                  * This is in response to Indochine which 
646                                  * only displays subs for 1 second - 
647                                  * too fast to read.
648                                  */
649                                 sub->stop = sub->start + ( 2 * 90000 );
650                             
651                                 sub2 = hb_fifo_see2( subtitle->fifo_raw );
652                             
653                                 if( sub2 && sub->stop > sub2->start )
654                                 {
655                                     sub->stop = sub2->start;
656                                 }
657                             }
658                         }
659                         else
660                         {
661                             /*
662                              * Defer until the play point is within 
663                              * the subtitle
664                              */
665                             sub = NULL;
666                         }
667                     }
668                     else
669                     {
670                         /*
671                          * The end of the subtitle is less than the start, 
672                          * this is a sign of a PTS discontinuity.
673                          */
674                         if( sub->start > cur->start )
675                         {
676                             /*
677                              * we haven't reached the start time yet, or
678                              * we have jumped backwards after having
679                              * already started this subtitle.
680                              */
681                             if( cur->start < sub->stop )
682                             {
683                                 /*
684                                  * We have jumped backwards and so should
685                                  * continue displaying this subtitle.
686                                  *
687                                  * fall through to display.
688                                  */
689                             }
690                             else
691                             {
692                                 /*
693                                  * Defer until the play point is 
694                                  * within the subtitle
695                                  */
696                                 sub = NULL;
697                             }
698                         } else {
699                             /*
700                             * Play this subtitle as the start is 
701                             * greater than our video point.
702                             *
703                             * fall through to display/
704                             */
705                         }
706                     }
707                         break;
708                 }
709                 else
710                 {
711                     
712                     /*
713                      * The subtitle is older than this picture, trash it
714                      */
715                     sub = hb_fifo_get( subtitle->fifo_raw );
716                     hb_buffer_close( &sub );
717                 }
718             }
719             
720             /* If we have a subtitle for this picture, copy it */
721             /* FIXME: we should avoid this memcpy */
722             if( sub )
723             {
724                 if( sub->size > 0 )
725                 {
726                     if( subtitle->config.dest == RENDERSUB )
727                     {
728                         if ( cur->sub == NULL )
729                         {
730                             /*
731                              * Tack onto the video buffer for rendering
732                              */
733                             cur->sub         = hb_buffer_init( sub->size );
734                             cur->sub->x      = sub->x;
735                             cur->sub->y      = sub->y;
736                             cur->sub->width  = sub->width;
737                             cur->sub->height = sub->height;
738                             memcpy( cur->sub->data, sub->data, sub->size ); 
739                         }
740                     } else {
741                         /*
742                          * Pass-Through, pop it off of the raw queue, 
743                          */
744                         sub = hb_fifo_get( subtitle->fifo_raw );
745                         hb_fifo_push( subtitle->fifo_sync, sub );
746                     }
747                 } else {
748                     /*
749                     * EOF - consume for rendered, else pass through
750                     */
751                     if( subtitle->config.dest == RENDERSUB )
752                     {
753                         sub = hb_fifo_get( subtitle->fifo_raw );
754                         hb_buffer_close( &sub );
755                     } else {
756                         sub = hb_fifo_get( subtitle->fifo_raw );
757                         hb_fifo_push( subtitle->fifo_sync, sub );
758                     }
759                 }
760             }
761         }
762     } // end subtitles
763
764     /*
765      * Adjust the pts of the current frame so that it's contiguous
766      * with the previous frame. The start time of the current frame
767      * has to be the end time of the previous frame and the stop
768      * time has to be the start of the next frame.  We don't
769      * make any adjustments to the source timestamps other than removing
770      * the clock offsets (which also removes pts discontinuities).
771      * This means we automatically encode at the source's frame rate.
772      * MP2 uses an implicit duration (frames end when the next frame
773      * starts) but more advanced containers like MP4 use an explicit
774      * duration. Since we're looking ahead one frame we set the
775      * explicit stop time from the start time of the next frame.
776      */
777     *buf_out = cur;
778     sync->cur = cur = next;
779     cur->sub = NULL;
780     sync->next_pts = cur->start;
781     int64_t duration = cur->start - sync->pts_skip - (*buf_out)->start;
782     sync->pts_skip = 0;
783     if ( duration <= 0 )
784     {
785         hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
786                 duration, (*buf_out)->start, next->start );
787     }
788
789     (*buf_out)->start = sync->next_start;
790     sync->next_start += duration;
791     (*buf_out)->stop = sync->next_start;
792
793     if ( sync->chap_mark )
794     {
795         // we have a pending chapter mark from a recent drop - put it on this
796         // buffer (this may make it one frame late but we can't do any better).
797         (*buf_out)->new_chap = sync->chap_mark;
798         sync->chap_mark = 0;
799     }
800
801     /* Update UI */
802     UpdateState( w );
803         
804     return HB_WORK_OK;
805 }
806
807 // sync*Init does nothing because sync has a special initializer
808 // that takes care of initializing video and all audio tracks
809 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
810 {
811     return 0;
812 }
813
814 hb_work_object_t hb_sync_video =
815 {
816     WORK_SYNC_VIDEO,
817     "Video Synchronization",
818     syncVideoInit,
819     syncVideoWork,
820     syncVideoClose
821 };
822
823 /***********************************************************************
824  * Close Audio
825  ***********************************************************************
826  *
827  **********************************************************************/
828 void syncAudioClose( hb_work_object_t * w )
829 {
830     hb_work_private_t * pv    = w->private_data;
831     hb_sync_audio_t   * sync  = &pv->type.audio;
832
833     if( w->audio->config.out.codec == HB_ACODEC_AC3 )
834     {
835         free( sync->ac3_buf );
836     }
837     else
838     {
839         src_delete( sync->state );
840     }
841
842     hb_lock( pv->common->mutex );
843     if ( --pv->common->ref == 0 )
844     {
845         hb_unlock( pv->common->mutex );
846         hb_lock_close( &pv->common->mutex );
847         free( pv->common );
848     }
849     else
850     {
851         hb_unlock( pv->common->mutex );
852     }
853
854     free( pv );
855     w->private_data = NULL;
856 }
857
858 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
859 {
860     return 0;
861 }
862
863 /***********************************************************************
864  * SyncAudio
865  ***********************************************************************
866  *
867  **********************************************************************/
868 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
869                        hb_buffer_t ** buf_out )
870 {
871     hb_work_private_t * pv = w->private_data;
872     hb_job_t        * job = pv->job;
873     hb_sync_audio_t * sync = &pv->type.audio;
874     hb_buffer_t     * buf;
875     //hb_fifo_t       * fifo;
876     int64_t start;
877
878     *buf_out = NULL;
879     buf = *buf_in;
880     *buf_in = NULL;
881     /* if the next buffer is an eof send it downstream */
882     if ( buf->size <= 0 )
883     {
884         hb_buffer_close( &buf );
885         *buf_out = hb_buffer_init( 0 );
886         return HB_WORK_DONE;
887     }
888
889     /* Wait for start frame if doing point-to-point */
890     hb_lock( pv->common->mutex );
891     while ( !pv->common->start_found )
892     {
893         if ( buf->start < pv->common->audio_pts_thresh )
894         {
895             hb_buffer_close( &buf );
896             hb_unlock( pv->common->mutex );
897             return HB_WORK_OK;
898         }
899         while ( !pv->common->start_found && 
900                 buf->start >= pv->common->audio_pts_thresh )
901         {
902             hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
903         }
904     }
905     if ( buf->start < pv->common->audio_pts_thresh )
906     {
907         hb_buffer_close( &buf );
908         hb_unlock( pv->common->mutex );
909         return HB_WORK_OK;
910     }
911     hb_unlock( pv->common->mutex );
912
913     /* Wait till we can determine the initial pts of all streams */
914     if( pv->common->pts_offset == INT64_MIN )
915     {
916         pv->common->first_pts[sync->index+1] = buf->start;
917         hb_lock( pv->common->mutex );
918         while( pv->common->pts_offset == INT64_MIN )
919         {
920             // Full fifos will make us wait forever, so get the
921             // pts offset from the available streams if full
922             if (hb_fifo_is_full(w->fifo_in))
923             {
924                 getPtsOffset( w );
925                 hb_cond_broadcast( pv->common->next_frame );
926             }
927             else if ( checkPtsOffset( w ) )
928                 hb_cond_broadcast( pv->common->next_frame );
929             else
930                 hb_cond_timedwait( pv->common->next_frame, pv->common->mutex, 200 );
931         }
932         hb_unlock( pv->common->mutex );
933     }
934
935     if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
936     {
937         hb_buffer_close( &buf );
938         *buf_out = hb_buffer_init( 0 );
939         return HB_WORK_DONE;
940     }
941
942     if( job->pts_to_stop && sync->next_start >= job->pts_to_stop )
943     {
944         hb_buffer_close( &buf );
945         *buf_out = hb_buffer_init( 0 );
946         return HB_WORK_DONE;
947     }
948
949     hb_lock( pv->common->mutex );
950     start = buf->start - pv->common->audio_passthru_slip;
951     hb_unlock( pv->common->mutex );
952     if ( (int64_t)( start - sync->next_pts ) < 0 )
953     {
954         // audio time went backwards.
955         // If our output clock is more than a half frame ahead of the
956         // input clock drop this frame to move closer to sync.
957         // Otherwise drop frames until the input clock matches the output clock.
958         if ( sync->first_drop || sync->next_start - start > 90*15 )
959         {
960             // Discard data that's in the past.
961             if ( sync->first_drop == 0 )
962             {
963                 sync->first_drop = sync->next_pts;
964             }
965             ++sync->drop_count;
966             hb_buffer_close( &buf );
967             return HB_WORK_OK;
968         }
969         sync->next_pts = start;
970     }
971     if ( sync->first_drop )
972     {
973         // we were dropping old data but input buf time is now current
974         hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
975                 "(next %"PRId64", current %"PRId64")", w->audio->id,
976                 (int)( sync->next_pts - sync->first_drop ) / 90,
977                 sync->drop_count, sync->first_drop, sync->next_pts );
978         sync->first_drop = 0;
979         sync->drop_count = 0;
980         sync->next_pts = start;
981     }
982     if ( start - sync->next_pts >= (90 * 70) )
983     {
984         if ( start - sync->next_pts > (90000LL * 60) )
985         {
986             // there's a gap of more than a minute between the last
987             // frame and this. assume we got a corrupted timestamp
988             // and just drop the next buf.
989             hb_log( "sync: %d minute time gap in audio %d - dropping buf"
990                     "  start %"PRId64", next %"PRId64,
991                     (int)((start - sync->next_pts) / (90000*60)),
992                     w->audio->id, start, sync->next_pts );
993             hb_buffer_close( &buf );
994             return HB_WORK_OK;
995         }
996         /*
997          * there's a gap of at least 70ms between the last
998          * frame we processed & the next. Fill it with silence.
999          * Or in the case of DCA, skip some frames from the
1000          * other streams.
1001          */
1002         if( w->audio->config.out.codec == HB_ACODEC_DCA )
1003         {
1004             hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
1005                     "  start %"PRId64", next %"PRId64,
1006                     (int)((start - sync->next_pts) / 90),
1007                     w->audio->id, start, sync->next_pts );
1008             hb_lock( pv->common->mutex );
1009             pv->common->audio_passthru_slip += (start - sync->next_pts);
1010             pv->common->video_pts_slip += (start - sync->next_pts);
1011             hb_unlock( pv->common->mutex );
1012             *buf_out = buf;
1013             return HB_WORK_OK;
1014         }
1015         hb_log( "sync: adding %d ms of silence to audio %d"
1016                 "  start %"PRId64", next %"PRId64,
1017                 (int)((start - sync->next_pts) / 90),
1018                 w->audio->id, start, sync->next_pts );
1019         InsertSilence( w, start - sync->next_pts );
1020         *buf_out = buf;
1021         return HB_WORK_OK;
1022     }
1023
1024     /*
1025      * When we get here we've taken care of all the dups and gaps in the
1026      * audio stream and are ready to inject the next input frame into
1027      * the output stream.
1028      */
1029     *buf_out = OutputAudioFrame( w->audio, buf, sync );
1030     return HB_WORK_OK;
1031 }
1032
1033 hb_work_object_t hb_sync_audio =
1034 {
1035     WORK_SYNC_AUDIO,
1036     "AudioSynchronization",
1037     syncAudioInit,
1038     syncAudioWork,
1039     syncAudioClose
1040 };
1041
1042 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
1043 {
1044     hb_work_object_t  * w;
1045     hb_work_private_t * pv;
1046     hb_title_t        * title = job->title;
1047     hb_sync_audio_t   * sync;
1048
1049     pv = calloc( 1, sizeof( hb_work_private_t ) );
1050     sync = &pv->type.audio;
1051     sync->index = i;
1052     pv->job    = job;
1053     pv->common = common;
1054     pv->common->ref++;
1055     pv->common->pts_count++;
1056
1057     w = hb_get_work( WORK_SYNC_AUDIO );
1058     w->private_data = pv;
1059     w->audio = hb_list_item( title->list_audio, i );
1060     w->fifo_in = w->audio->priv.fifo_raw;
1061
1062     if( w->audio->config.out.codec == HB_ACODEC_AC3 ||
1063         w->audio->config.out.codec == HB_ACODEC_DCA )
1064     {
1065         w->fifo_out = w->audio->priv.fifo_out;
1066     }
1067     else
1068     {
1069         w->fifo_out = w->audio->priv.fifo_sync;
1070     }
1071
1072     if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1073     {
1074         /* Have a silent AC-3 frame ready in case we have to fill a
1075            gap */
1076         AVCodec        * codec;
1077         AVCodecContext * c;
1078         short          * zeros;
1079
1080         codec = avcodec_find_encoder( CODEC_ID_AC3 );
1081         c     = avcodec_alloc_context();
1082
1083         c->bit_rate    = w->audio->config.in.bitrate;
1084         c->sample_rate = w->audio->config.in.samplerate;
1085         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
1086
1087         if( hb_avcodec_open( c, codec ) < 0 )
1088         {
1089             hb_log( "sync: avcodec_open failed" );
1090             return;
1091         }
1092
1093         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
1094                                  sizeof( short ) * c->channels, 1 );
1095         sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
1096                              w->audio->config.in.samplerate / 8;
1097         sync->ac3_buf  = malloc( sync->ac3_size );
1098
1099         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
1100                                   zeros ) != sync->ac3_size )
1101         {
1102             hb_log( "sync: avcodec_encode_audio failed" );
1103         }
1104
1105         free( zeros );
1106         hb_avcodec_close( c );
1107         av_free( c );
1108     }
1109     else
1110     {
1111         /* Initialize libsamplerate */
1112         int error;
1113         sync->state = src_new( SRC_SINC_MEDIUM_QUALITY, 
1114             HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1115                 w->audio->config.out.mixdown), &error );
1116         sync->data.end_of_input = 0;
1117     }
1118     hb_list_add( job->list_work, w );
1119 }
1120
1121 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
1122                                        hb_sync_audio_t *sync )
1123 {
1124     int64_t start = sync->next_start;
1125     int64_t duration = buf->stop - buf->start;
1126
1127     sync->next_pts += duration;
1128
1129     if( audio->config.in.samplerate == audio->config.out.samplerate ||
1130         audio->config.out.codec == HB_ACODEC_AC3 ||
1131         audio->config.out.codec == HB_ACODEC_DCA )
1132     {
1133         /*
1134          * If we don't have to do sample rate conversion or this audio is 
1135          * pass-thru just send the input buffer downstream after adjusting
1136          * its timestamps to make the output stream continuous.
1137          */
1138     }
1139     else
1140     {
1141         /* Not pass-thru - do sample rate conversion */
1142         int count_in, count_out;
1143         hb_buffer_t * buf_raw = buf;
1144         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
1145                             sizeof( float );
1146
1147         count_in  = buf_raw->size / channel_count;
1148         /*
1149          * When using stupid rates like 44.1 there will always be some
1150          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
1151          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
1152          * the error will build up over time and eventually the audio will
1153          * substantially lag the video. libsamplerate will keep track of the
1154          * fractional sample & give it to us when appropriate if we give it
1155          * an extra sample of space in the output buffer.
1156          */
1157         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
1158
1159         sync->data.input_frames = count_in;
1160         sync->data.output_frames = count_out;
1161         sync->data.src_ratio = (double)audio->config.out.samplerate /
1162                                (double)audio->config.in.samplerate;
1163
1164         buf = hb_buffer_init( count_out * channel_count );
1165         sync->data.data_in  = (float *) buf_raw->data;
1166         sync->data.data_out = (float *) buf->data;
1167         if( src_process( sync->state, &sync->data ) )
1168         {
1169             /* XXX If this happens, we're screwed */
1170             hb_log( "sync: audio %d src_process failed", audio->id );
1171         }
1172         hb_buffer_close( &buf_raw );
1173
1174         buf->size = sync->data.output_frames_gen * channel_count;
1175         duration = ( sync->data.output_frames_gen * 90000 ) /
1176                    audio->config.out.samplerate;
1177     }
1178     buf->frametype = HB_FRAME_AUDIO;
1179     buf->start = start;
1180     buf->stop  = start + duration;
1181     sync->next_start = start + duration;
1182     return buf;
1183 }
1184
1185 static void InsertSilence( hb_work_object_t * w, int64_t duration )
1186 {
1187     hb_work_private_t * pv = w->private_data;
1188     hb_sync_audio_t *sync = &pv->type.audio;
1189     hb_buffer_t     *buf;
1190     hb_fifo_t       *fifo;
1191
1192     // to keep pass-thru and regular audio in sync we generate silence in
1193     // AC3 frame-sized units. If the silence duration isn't an integer multiple
1194     // of the AC3 frame duration we will truncate or round up depending on
1195     // which minimizes the timing error.
1196     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1197                           w->audio->config.in.samplerate;
1198     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1199
1200     while ( --frame_count >= 0 )
1201     {
1202         if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1203         {
1204             buf        = hb_buffer_init( sync->ac3_size );
1205             buf->start = sync->next_pts;
1206             buf->stop  = buf->start + frame_dur;
1207             memcpy( buf->data, sync->ac3_buf, buf->size );
1208             fifo = w->audio->priv.fifo_out;
1209         }
1210         else
1211         {
1212             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1213                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1214                                          w->audio->config.out.mixdown) );
1215             buf->start = sync->next_pts;
1216             buf->stop  = buf->start + frame_dur;
1217             memset( buf->data, 0, buf->size );
1218             fifo = w->audio->priv.fifo_sync;
1219         }
1220         buf = OutputAudioFrame( w->audio, buf, sync );
1221         hb_fifo_push( fifo, buf );
1222     }
1223 }
1224
1225 static void UpdateState( hb_work_object_t * w )
1226 {
1227     hb_work_private_t * pv = w->private_data;
1228     hb_sync_video_t   * sync = &pv->type.video;
1229     hb_state_t state;
1230
1231     if( !pv->common->count_frames )
1232     {
1233         sync->st_first = hb_get_date();
1234         pv->job->st_pause_date = -1;
1235         pv->job->st_paused = 0;
1236     }
1237     pv->common->count_frames++;
1238
1239     if( hb_get_date() > sync->st_dates[3] + 1000 )
1240     {
1241         memmove( &sync->st_dates[0], &sync->st_dates[1],
1242                  3 * sizeof( uint64_t ) );
1243         memmove( &sync->st_counts[0], &sync->st_counts[1],
1244                  3 * sizeof( uint64_t ) );
1245         sync->st_dates[3]  = hb_get_date();
1246         sync->st_counts[3] = pv->common->count_frames;
1247     }
1248
1249 #define p state.param.working
1250     state.state = HB_STATE_WORKING;
1251     p.progress  = (float) pv->common->count_frames / (float) sync->count_frames_max;
1252     if( p.progress > 1.0 )
1253     {
1254         p.progress = 1.0;
1255     }
1256     p.rate_cur   = 1000.0 *
1257         (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1258         (float) ( sync->st_dates[3] - sync->st_dates[0] );
1259     if( hb_get_date() > sync->st_first + 4000 )
1260     {
1261         int eta;
1262         p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1263             (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1264         eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1265             p.rate_avg;
1266         p.hours   = eta / 3600;
1267         p.minutes = ( eta % 3600 ) / 60;
1268         p.seconds = eta % 60;
1269     }
1270     else
1271     {
1272         p.rate_avg = 0.0;
1273         p.hours    = -1;
1274         p.minutes  = -1;
1275         p.seconds  = -1;
1276     }
1277 #undef p
1278
1279     hb_set_state( pv->job->h, &state );
1280 }
1281
1282 static void UpdateSearchState( hb_work_object_t * w, int64_t start )
1283 {
1284     hb_work_private_t * pv = w->private_data;
1285     hb_sync_video_t   * sync = &pv->type.video;
1286     hb_state_t state;
1287     uint64_t now;
1288     double avg;
1289
1290     now = hb_get_date();
1291     if( !pv->common->count_frames )
1292     {
1293         sync->st_first = now;
1294         pv->job->st_pause_date = -1;
1295         pv->job->st_paused = 0;
1296     }
1297     pv->common->count_frames++;
1298
1299 #define p state.param.working
1300     state.state = HB_STATE_SEARCHING;
1301     if ( pv->job->frame_to_start )
1302         p.progress  = (float) pv->common->count_frames / 
1303                       (float) pv->job->frame_to_start;
1304     else if ( pv->job->pts_to_start )
1305         p.progress  = (float) start / (float) pv->job->pts_to_start;
1306     else
1307         p.progress = 0;
1308     if( p.progress > 1.0 )
1309     {
1310         p.progress = 1.0;
1311     }
1312     if (now > sync->st_first)
1313     {
1314         int eta;
1315
1316         if ( pv->job->frame_to_start )
1317         {
1318             avg = 1000.0 * (double)pv->common->count_frames / (now - sync->st_first);
1319             eta = ( pv->job->frame_to_start - pv->common->count_frames ) / avg;
1320         }
1321         else if ( pv->job->pts_to_start )
1322         {
1323             avg = 1000.0 * (double)start / (now - sync->st_first);
1324             eta = ( pv->job->pts_to_start - start ) / avg;
1325         }
1326         p.hours   = eta / 3600;
1327         p.minutes = ( eta % 3600 ) / 60;
1328         p.seconds = eta % 60;
1329     }
1330     else
1331     {
1332         p.rate_avg = 0.0;
1333         p.hours    = -1;
1334         p.minutes  = -1;
1335         p.seconds  = -1;
1336     }
1337 #undef p
1338
1339     hb_set_state( pv->job->h, &state );
1340 }
1341
1342 static void getPtsOffset( hb_work_object_t * w )
1343 {
1344     hb_work_private_t * pv = w->private_data;
1345     int           i ;
1346     int64_t       first_pts = INT64_MAX;
1347
1348     for( i = 0; i < pv->common->pts_count; i++ )
1349     {
1350         if ( pv->common->first_pts[i] < first_pts )
1351             first_pts = pv->common->first_pts[i];
1352     }
1353     pv->common->audio_passthru_slip = pv->common->pts_offset = first_pts;
1354     return;
1355 }
1356
1357 static int checkPtsOffset( hb_work_object_t * w )
1358 {
1359     hb_work_private_t * pv = w->private_data;
1360     int           i ;
1361
1362     for( i = 0; i < pv->common->pts_count; i++ )
1363     {
1364         if ( pv->common->first_pts[i] == INT64_MAX )
1365             return 0;
1366     }
1367     getPtsOffset( w );
1368     return 1;
1369 }