libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.m0k.org/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8
   9 #include "samplerate.h"
  10 #include "ffmpeg/avcodec.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22     int64_t      count_frames;
  23
  24     /* Raw */
  25     SRC_STATE  * state;
  26     SRC_DATA     data;
  27
  28     /* AC-3 */
  29     int          ac3_size;
  30     uint8_t    * ac3_buf;
  31
  32 } hb_sync_audio_t;
  33
  34 struct hb_work_private_s
  35 {
  36     hb_job_t * job;
  37     int        done;
  38
  39     /* Video */
  40     hb_subtitle_t * subtitle;
  41     int64_t pts_offset;
  42     int64_t pts_offset_old;
  43     int64_t next_start;
  44     int64_t count_frames;
  45     int64_t count_frames_max;
  46     int64_t video_sequence;
  47     hb_buffer_t * cur; /* The next picture to process */
  48
  49     /* Audio */
  50     hb_sync_audio_t sync_audio[8];
  51
  52     /* Flags */
  53     int discontinuity;
  54
  55     /* Statistics */
  56     uint64_t st_counts[4];
  57     uint64_t st_dates[4];
  58     uint64_t st_first;
  59
  60     /* Throttle message flags */
  61     int   trashing_audio;
  62     int   inserting_silence;
  63     int   way_out_of_sync;
  64 };
  65
  66 /***********************************************************************
  67  * Local prototypes
  68  **********************************************************************/
  69 static void InitAudio( hb_work_object_t * w, int i );
  70 static int  SyncVideo( hb_work_object_t * w );
  71 static void SyncAudio( hb_work_object_t * w, int i );
  72 static int  NeedSilence( hb_work_object_t * w, hb_audio_t * );
  73 static void InsertSilence( hb_work_object_t * w, int i );
  74 static void UpdateState( hb_work_object_t * w );
  75
  76 /***********************************************************************
  77  * hb_work_sync_init
  78  ***********************************************************************
  79  * Initialize the work object
  80  **********************************************************************/
  81 int syncInit( hb_work_object_t * w, hb_job_t * job )
  82 {
  83     hb_title_t       * title = job->title;
  84     hb_chapter_t     * chapter;
  85     int                i;
  86     uint64_t           duration;
  87     hb_work_private_t * pv;
  88
  89     pv = calloc( 1, sizeof( hb_work_private_t ) );
  90     w->private_data = pv;
  91
  92     pv->job            = job;
  93     pv->pts_offset     = INT64_MIN;
  94     pv->pts_offset_old = INT64_MIN;
  95     pv->count_frames   = 0;
  96
  97     pv->discontinuity = 0;
  98
  99     pv->trashing_audio = 0;
 100     pv->inserting_silence = 0;
 101     pv->way_out_of_sync = 0;
 102
 103     /* Calculate how many video frames we are expecting */
 104     duration = 0;
 105     for( i = job->chapter_start; i <= job->chapter_end; i++ )
 106     {
 107         chapter   = hb_list_item( title->list_chapter, i - 1 );
 108         duration += chapter->duration;
 109     }
 110     duration += 90000;
 111         /* 1 second safety so we're sure we won't miss anything */
 112     pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
 113
 114     hb_log( "sync: expecting %lld video frames", pv->count_frames_max );
 115
 116     /* Initialize libsamplerate for every audio track we have */
 117     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 118     {
 119         InitAudio( w, i );
 120     }
 121
 122     /* Get subtitle info, if any */
 123     pv->subtitle = hb_list_item( title->list_subtitle, 0 );
 124
 125     pv->video_sequence = 0;
 126
 127     return 0;
 128 }
 129
 130 /***********************************************************************
 131  * Close
 132  ***********************************************************************
 133  *
 134  **********************************************************************/
 135 void syncClose( hb_work_object_t * w )
 136 {
 137     hb_work_private_t * pv = w->private_data;
 138     hb_job_t          * job   = pv->job;
 139     hb_title_t        * title = job->title;
 140
 141     int i;
 142
 143     if( pv->cur ) hb_buffer_close( &pv->cur );
 144
 145     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 146     {
 147         if( job->acodec & HB_ACODEC_AC3 )
 148         {
 149             free( pv->sync_audio[i].ac3_buf );
 150         }
 151         else
 152         {
 153             src_delete( pv->sync_audio[i].state );
 154         }
 155     }
 156
 157     free( pv );
 158     w->private_data = NULL;
 159 }
 160
 161 /***********************************************************************
 162  * Work
 163  ***********************************************************************
 164  * The root routine of this work abject
 165  *
 166  * The way this works is that we are syncing the audio to the PTS of
 167  * the last video that we processed. That's why we skip the audio sync
 168  * if we haven't got a valid PTS from the video yet.
 169  *
 170  **********************************************************************/
 171 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 172               hb_buffer_t ** unused2 )
 173 {
 174     hb_work_private_t * pv = w->private_data;
 175     int i;
 176
 177     /* If we ever got a video frame, handle audio now */
 178     if( pv->pts_offset != INT64_MIN )
 179     {
 180         for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 181         {
 182             SyncAudio( w, i );
 183         }
 184     }
 185
 186     /* Handle video */
 187     return SyncVideo( w );
 188 }
 189
 190 hb_work_object_t hb_sync =
 191 {
 192     WORK_SYNC,
 193     "Synchronization",
 194     syncInit,
 195     syncWork,
 196     syncClose
 197 };
 198
 199 static void InitAudio( hb_work_object_t * w, int i )
 200 {
 201     hb_work_private_t * pv = w->private_data;
 202     hb_job_t        * job   = pv->job;
 203     hb_title_t      * title = job->title;
 204     hb_sync_audio_t * sync;
 205
 206     sync        = &pv->sync_audio[i];
 207     sync->audio = hb_list_item( title->list_audio, i );
 208
 209     if( job->acodec & HB_ACODEC_AC3 )
 210     {
 211         /* Have a silent AC-3 frame ready in case we have to fill a
 212            gap */
 213         AVCodec        * codec;
 214         AVCodecContext * c;
 215         short          * zeros;
 216
 217         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 218         c     = avcodec_alloc_context();
 219
 220         c->bit_rate    = sync->audio->bitrate;
 221         c->sample_rate = sync->audio->rate;
 222         c->channels    = 2;
 223
 224         if( avcodec_open( c, codec ) < 0 )
 225         {
 226             hb_log( "sync: avcodec_open failed" );
 227             return;
 228         }
 229
 230         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 231                                  sizeof( short ) * c->channels, 1 );
 232         sync->ac3_size = sync->audio->bitrate * AC3_SAMPLES_PER_FRAME /
 233                              sync->audio->rate / 8;
 234         sync->ac3_buf  = malloc( sync->ac3_size );
 235
 236         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 237                                   zeros ) != sync->ac3_size )
 238         {
 239             hb_log( "sync: avcodec_encode_audio failed" );
 240         }
 241
 242         free( zeros );
 243         avcodec_close( c );
 244         av_free( c );
 245     }
 246     else
 247     {
 248         /* Initialize libsamplerate */
 249         int error;
 250         sync->state             = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown), &error );
 251         sync->data.end_of_input = 0;
 252     }
 253 }
 254
 255
 256
 257 #define PTS_DISCONTINUITY_TOLERANCE 90000
 258
 259 /***********************************************************************
 260  * SyncVideo
 261  ***********************************************************************
 262  *
 263  **********************************************************************/
 264 static int SyncVideo( hb_work_object_t * w )
 265 {
 266     hb_work_private_t * pv = w->private_data;
 267     hb_buffer_t * cur, * next, * sub = NULL;
 268     hb_job_t * job = pv->job;
 269     int64_t pts_expected;
 270     int chap_break;
 271
 272     if( pv->done )
 273     {
 274         return HB_WORK_DONE;
 275     }
 276
 277     if( hb_thread_has_exited( job->reader ) &&
 278         !hb_fifo_size( job->fifo_mpeg2 ) &&
 279         !hb_fifo_size( job->fifo_raw ) )
 280     {
 281         /* All video data has been processed already, we won't get
 282            more */
 283         hb_log( "sync: got %lld frames, %lld expected",
 284                 pv->count_frames, pv->count_frames_max );
 285         pv->done = 1;
 286
 287         hb_buffer_t * buf_tmp;
 288
 289        // Drop an empty buffer into our output to ensure that things
 290        // get flushed all the way out.
 291         buf_tmp = hb_buffer_init(0); // Empty end buffer
 292         hb_fifo_push( job->fifo_sync, buf_tmp );
 293
 294         return HB_WORK_DONE;
 295     }
 296
 297     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 298     {
 299         /* We haven't even got a frame yet */
 300         return HB_WORK_OK;
 301     }
 302     cur = pv->cur;
 303
 304     /* At this point we have a frame to process. Let's check
 305         1) if we will be able to push into the fifo ahead
 306         2) if the next frame is there already, since we need it to
 307            know whether we'll have to repeat the current frame or not */
 308     while( !hb_fifo_is_full( job->fifo_sync ) &&
 309            ( next = hb_fifo_see( job->fifo_raw ) ) )
 310     {
 311         hb_buffer_t * buf_tmp;
 312
 313         if( pv->pts_offset == INT64_MIN )
 314         {
 315             /* This is our first frame */
 316             hb_log( "sync: first pts is %lld", cur->start );
 317             pv->pts_offset = cur->start;
 318         }
 319
 320         /*
 321          * Track the video sequence number localy so that we can sync the audio
 322          * to it using the sequence number as well as the PTS.
 323          */
 324         pv->video_sequence = cur->sequence;
 325
 326         /* Check for PTS jumps over 0.5 second */
 327         if( next->start < cur->start - PTS_DISCONTINUITY_TOLERANCE ||
 328             next->start > cur->start + PTS_DISCONTINUITY_TOLERANCE )
 329         {
 330             hb_log( "Sync: Video PTS discontinuity %s (current buffer start=%lld, next buffer start=%lld)",
 331                     pv->discontinuity ? "second" : "first", cur->start, next->start );
 332
 333             /*
 334              * Do we need to trash the subtitle, is it from the next->start period
 335              * or is it from our old position. If the latter then trash it.
 336              */
 337             if( pv->subtitle )
 338             {
 339                 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 340                 {
 341                     if( ( sub->start > ( cur->start - PTS_DISCONTINUITY_TOLERANCE ) ) &&
 342                         ( sub->start < ( cur->start + PTS_DISCONTINUITY_TOLERANCE ) ) )
 343                     {
 344                         /*
 345                          * The subtitle is from our current time region which we are
 346                          * jumping from. So trash it as we are about to jump backwards
 347                          * or forwards and don't want it blocking the subtitle fifo.
 348                          */
 349                         hb_log("Trashing subtitle 0x%x due to PTS discontinuity", sub);
 350                         sub = hb_fifo_get( pv->subtitle->fifo_raw );
 351                         hb_buffer_close( &sub );
 352                     } else {
 353                         break;
 354                     }
 355                 }
 356             }
 357
 358             /* Trash current picture */
 359             /* Also, make sure we don't trash a chapter break */
 360             chap_break = cur->new_chap;
 361             hb_buffer_close( &cur );
 362             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 363             cur->new_chap |= chap_break; // Don't stomp existing chapter breaks
 364
 365             /* Calculate new offset */
 366             pv->pts_offset_old = pv->pts_offset;
 367             if ( job->vfr )
 368             {
 369                 pv->pts_offset = cur->start - pv->next_start;
 370             } else {
 371                 pv->pts_offset = cur->start -
 372                     pv->count_frames * pv->job->vrate_base / 300;
 373             }
 374
 375             if( !pv->discontinuity )
 376             {
 377                 pv->discontinuity = 1;
 378             }
 379
 380             pv->video_sequence = cur->sequence;
 381             continue;
 382         }
 383
 384         /* Look for a subtitle for this frame */
 385         if( pv->subtitle )
 386         {
 387             hb_buffer_t * sub2;
 388             while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 389             {
 390                 /* If two subtitles overlap, make the first one stop
 391                    when the second one starts */
 392                 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
 393                 if( sub2 && sub->stop > sub2->start )
 394                     sub->stop = sub2->start;
 395
 396                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 397                 //       sub, cur->sequence, sub->sequence);
 398
 399                 if( sub->sequence > cur->sequence )
 400                 {
 401                     /*
 402                      * The video is behind where we are, so wait until
 403                      * it catches up to the same reader point on the
 404                      * DVD. Then our PTS should be in the same region
 405                      * as the video.
 406                      */
 407                     sub = NULL;
 408                     break;
 409                 }
 410
 411                 if( sub->stop > cur->start ) {
 412                     /*
 413                      * The stop time is in the future, so fall through
 414                      * and we'll deal with it in the next block of
 415                      * code.
 416                      */
 417                     break;
 418                 }
 419                 else
 420                 {
 421                     /*
 422                      * The stop time is in the past. But is it due to
 423                      * it having been played already, or has the PTS
 424                      * been reset to 0?
 425                      */
 426                     if( ( cur->start - sub->stop ) > PTS_DISCONTINUITY_TOLERANCE ) {
 427                         /*
 428                          * There is a lot of time between our current
 429                          * video and where this subtitle is ending,
 430                          * assume that we are about to reset the PTS
 431                          * and do not throw away this subtitle.
 432                          */
 433                         break;
 434                     }
 435                 }
 436
 437                 /*
 438                  * The subtitle is older than this picture, trash it
 439                  */
 440                 sub = hb_fifo_get( pv->subtitle->fifo_raw );
 441                 hb_buffer_close( &sub );
 442             }
 443
 444             /*
 445              * There is a valid subtitle, is it time to display it?
 446              */
 447             if( sub )
 448             {
 449                 if( sub->stop > sub->start)
 450                 {
 451                     /*
 452                      * Normal subtitle which ends after it starts, check to
 453                      * see that the current video is between the start and end.
 454                      */
 455                     if( cur->start > sub->start &&
 456                         cur->start < sub->stop )
 457                     {
 458                         /*
 459                          * We should be playing this, so leave the
 460                          * subtitle in place.
 461                          *
 462                          * fall through to display
 463                          */
 464                     }
 465                     else
 466                     {
 467                         /*
 468                          * Defer until the play point is within the subtitle
 469                          */
 470                         sub = NULL;
 471                     }
 472                 }
 473                 else
 474                 {
 475                     /*
 476                      * The end of the subtitle is less than the start, this is a
 477                      * sign of a PTS discontinuity.
 478                      */
 479                     if( sub->start > cur->start )
 480                     {
 481                         /*
 482                          * we haven't reached the start time yet, or
 483                          * we have jumped backwards after having
 484                          * already started this subtitle.
 485                          */
 486                         if( cur->start < sub->stop )
 487                         {
 488                             /*
 489                              * We have jumped backwards and so should
 490                              * continue displaying this subtitle.
 491                              *
 492                              * fall through to display.
 493                              */
 494                         }
 495                         else
 496                         {
 497                             /*
 498                              * Defer until the play point is within the subtitle
 499                              */
 500                             sub = NULL;
 501                         }
 502                     } else {
 503                         /*
 504                          * Play this subtitle as the start is greater than our
 505                          * video point.
 506                          *
 507                          * fall through to display/
 508                          */
 509                     }
 510                 }
 511             }
 512         }
 513
 514         if ( job->vfr )
 515         {
 516             /*
 517              * adjust the pts of the current frame so that it's contiguous
 518              * with the previous frame. pts_offset tracks the time difference
 519              * between the pts values in the input content (which start at some
 520              * random time) and our timestamps (which start at zero). We don't
 521              * make any adjustments to the source timestamps other than removing
 522              * the clock offsets (which also removes pts discontinuities).
 523              * This means we automatically encode at the source's frame rate.
 524              * MP2 uses an implicit duration (frames end when the next frame
 525              * starts) but more advanced containers like MP4 use an explicit
 526              * duration. Since we're looking ahead one frame we set the
 527              * explicit stop time from the start time of the next frame.
 528              */
 529             buf_tmp = cur;
 530             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 531             buf_tmp->start = pv->next_start;
 532             pv->next_start = next->start - pv->pts_offset;
 533             buf_tmp->stop = pv->next_start;
 534         }
 535         else
 536         {
 537             /* The PTS of the frame we are expecting now */
 538             pts_expected = pv->pts_offset +
 539                 pv->count_frames * pv->job->vrate_base / 300;
 540
 541             //hb_log("Video expecting PTS %lld, current frame: %lld, next frame: %lld, cf: %lld",
 542             //       pts_expected, cur->start, next->start, pv->count_frames * pv->job->vrate_base / 300 );
 543
 544             if( cur->start < pts_expected - pv->job->vrate_base / 300 / 2 &&
 545                 next->start < pts_expected + pv->job->vrate_base / 300 / 2 )
 546             {
 547                 /* The current frame is too old but the next one matches,
 548                    let's trash */
 549                 /* Also, make sure we don't trash a chapter break */
 550                 chap_break = cur->new_chap;
 551                 hb_buffer_close( &cur );
 552                 pv->cur = cur = hb_fifo_get( job->fifo_raw );
 553                 cur->new_chap |= chap_break; // Make sure we don't stomp the existing one.
 554
 555                 continue;
 556             }
 557
 558             if( next->start > pts_expected + 3 * pv->job->vrate_base / 300 / 2 )
 559             {
 560                 /* We'll need the current frame more than one time. Make a
 561                    copy of it and keep it */
 562                 buf_tmp = hb_buffer_init( cur->size );
 563                 memcpy( buf_tmp->data, cur->data, cur->size );
 564                 buf_tmp->sequence = cur->sequence;
 565             }
 566             else
 567             {
 568                 /* The frame has the expected date and won't have to be
 569                    duplicated, just put it through */
 570                 buf_tmp = cur;
 571                 pv->cur = cur = hb_fifo_get( job->fifo_raw );
 572             }
 573
 574             /* Replace those MPEG-2 dates with our dates */
 575             buf_tmp->start = (uint64_t) pv->count_frames *
 576                 pv->job->vrate_base / 300;
 577             buf_tmp->stop  = (uint64_t) ( pv->count_frames + 1 ) *
 578                 pv->job->vrate_base / 300;
 579         }
 580
 581         /* If we have a subtitle for this picture, copy it */
 582         /* FIXME: we should avoid this memcpy */
 583         if( sub )
 584         {
 585             buf_tmp->sub         = hb_buffer_init( sub->size );
 586             buf_tmp->sub->x      = sub->x;
 587             buf_tmp->sub->y      = sub->y;
 588             buf_tmp->sub->width  = sub->width;
 589             buf_tmp->sub->height = sub->height;
 590             memcpy( buf_tmp->sub->data, sub->data, sub->size );
 591         }
 592
 593         /* Push the frame to the renderer */
 594         hb_fifo_push( job->fifo_sync, buf_tmp );
 595
 596         /* Update UI */
 597         UpdateState( w );
 598
 599         /* Make sure we won't get more frames then expected */
 600         if( pv->count_frames >= pv->count_frames_max * 2)
 601         {
 602             hb_log( "sync: got too many frames (%lld), exiting early", pv->count_frames );
 603             pv->done = 1;
 604
 605            // Drop an empty buffer into our output to ensure that things
 606            // get flushed all the way out.
 607            buf_tmp = hb_buffer_init(0); // Empty end buffer
 608            hb_fifo_push( job->fifo_sync, buf_tmp );
 609
 610             break;
 611         }
 612     }
 613
 614     return HB_WORK_OK;
 615 }
 616
 617 /***********************************************************************
 618  * SyncAudio
 619  ***********************************************************************
 620  *
 621  **********************************************************************/
 622 static void SyncAudio( hb_work_object_t * w, int i )
 623 {
 624     hb_work_private_t * pv = w->private_data;
 625     hb_job_t        * job;
 626     hb_audio_t      * audio;
 627     hb_buffer_t     * buf;
 628     hb_sync_audio_t * sync;
 629
 630     hb_fifo_t       * fifo;
 631     int               rate;
 632
 633     int64_t           pts_expected;
 634     int64_t           start;
 635
 636     job    = pv->job;
 637     sync   = &pv->sync_audio[i];
 638     audio  = sync->audio;
 639
 640     if( job->acodec & HB_ACODEC_AC3 )
 641     {
 642         fifo = audio->fifo_out;
 643         rate = audio->rate;
 644     }
 645     else
 646     {
 647         fifo = audio->fifo_sync;
 648         rate = job->arate;
 649     }
 650
 651     while( !hb_fifo_is_full( fifo ) &&
 652            ( buf = hb_fifo_see( audio->fifo_raw ) ) )
 653     {
 654         /* The PTS of the samples we are expecting now */
 655         pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 656
 657         // hb_log("Video Sequence: %lld, Audio Sequence: %lld", pv->video_sequence, buf->sequence);
 658
 659         /*
 660          * Using the same logic as the Video have we crossed a VOB
 661          * boundary as detected by the expected PTS and the PTS of our
 662          * audio being out by more than the tolerance value.
 663          */
 664         if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 665             buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 666         {
 667             /* There has been a PTS discontinuity, and this frame might
 668                be from before the discontinuity*/
 669
 670             if( pv->discontinuity )
 671             {
 672                 /*
 673                  * There is an outstanding discontinuity, so use the offset from
 674                  * that discontinuity.
 675                  */
 676                 pts_expected = pv->pts_offset_old + sync->count_frames *
 677                     90000 / rate;
 678             }
 679             else
 680             {
 681                 /*
 682                  * No outstanding discontinuity, so the audio must be leading the
 683                  * video (or the PTS values are really stuffed). So lets mark this
 684                  * as a discontinuity ourselves for the audio to use until
 685                  * the video also crosses the discontinuity.
 686                  *
 687                  * pts_offset is used when we are in the same time space as the video
 688                  * pts_offset_old when in a discontinuity.
 689                  *
 690                  * Therefore set the pts_offset_old given the new pts_offset for this
 691                  * current buffer.
 692                  */
 693                 pv->discontinuity = 1;
 694                 pv->pts_offset_old = buf->start - sync->count_frames *
 695                     90000 / rate;
 696                 pts_expected = pv->pts_offset_old + sync->count_frames *
 697                     90000 / rate;
 698
 699                 hb_log("Sync: Audio discontinuity (sequence: vid %lld aud %lld) (pts %lld < %lld < %lld)",
 700                        pv->video_sequence, buf->sequence,
 701                        pts_expected - PTS_DISCONTINUITY_TOLERANCE, buf->start,
 702                        pts_expected + PTS_DISCONTINUITY_TOLERANCE );
 703             }
 704
 705             /*
 706              * Is the audio from a valid period given the previous
 707              * Video PTS. I.e. has there just been a video PTS
 708              * discontinuity and this audio belongs to the vdeo from
 709              * before?
 710              */
 711             if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 712                 buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 713             {
 714                 /*
 715                  * It's outside of our tolerance for where the video
 716                  * is now, and it's outside of the tolerance for
 717                  * where we have been in the case of a VOB change.
 718                  * Try and reconverge regardless. so continue on to
 719                  * our convergence code below which will kick in as
 720                  * it will be more than 100ms out.
 721                  *
 722                  * Note that trashing the Audio could make things
 723                  * worse if the Audio is in front because we will end
 724                  * up diverging even more. We need to hold on to the
 725                  * audio until the video catches up.
 726                  */
 727                 if( !pv->way_out_of_sync )
 728                 {
 729                     hb_log("Sync: Audio is way out of sync, attempt to reconverge from current video PTS");
 730                     pv->way_out_of_sync = 1;
 731                 }
 732
 733                 /*
 734                  * It wasn't from the old place, so we must be from
 735                  * the new, but just too far out. So attempt to
 736                  * reconverge by resetting the point we want to be to
 737                  * where we are currently wanting to be.
 738                  */
 739                 pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 740                 start = pts_expected - pv->pts_offset;
 741             } else {
 742                  /* Use the older offset */
 743                 start = pts_expected - pv->pts_offset_old;
 744             }
 745         }
 746         else
 747         {
 748             start = pts_expected - pv->pts_offset;
 749
 750             if( pv->discontinuity )
 751             {
 752                 /*
 753                  * The Audio is tracking the Video again using the normal pts_offset, so the
 754                  * discontinuity is over.
 755                  */
 756                 hb_log( "Sync: Audio joined Video after discontinuity at PTS %lld", buf->start );
 757                 pv->discontinuity = 0;
 758             }
 759         }
 760
 761         /* Tolerance: 100 ms */
 762         if( buf->start < pts_expected - 9000 )
 763         {
 764             if( !pv->trashing_audio )
 765             {
 766                 /* Audio is behind the Video, trash it, can't use it now. */
 767                 hb_log( "Sync: Audio PTS (%lld) < Video PTS (%lld) by greater than 100ms, trashing audio to reconverge",
 768                         buf->start, pts_expected);
 769                 pv->trashing_audio = 1;
 770             }
 771             buf = hb_fifo_get( audio->fifo_raw );
 772             hb_buffer_close( &buf );
 773             continue;
 774         }
 775         else if( buf->start > pts_expected + 9000 )
 776         {
 777             /* Audio is ahead of the Video, insert silence until we catch up*/
 778             if( !pv->inserting_silence )
 779             {
 780                 hb_log("Sync: Audio PTS (%lld) >  Video PTS (%lld) by greater than 100ms insert silence until reconverged", buf->start, pts_expected);
 781                 pv->inserting_silence = 1;
 782             }
 783             InsertSilence( w, i );
 784             continue;
 785         }
 786         else
 787         {
 788             if( pv->trashing_audio || pv->inserting_silence )
 789             {
 790                 hb_log( "Sync: Audio back in Sync at PTS %lld", buf->start );
 791                 pv->trashing_audio = 0;
 792                 pv->inserting_silence = 0;
 793             }
 794             if( pv->way_out_of_sync )
 795             {
 796                 hb_log( "Sync: Audio no longer way out of sync at PTS %lld",
 797                         buf->start );
 798                 pv->way_out_of_sync = 0;
 799             }
 800         }
 801
 802         if( job->acodec & HB_ACODEC_AC3 )
 803         {
 804             buf        = hb_fifo_get( audio->fifo_raw );
 805             buf->start = start;
 806             buf->stop  = start + 90000 * AC3_SAMPLES_PER_FRAME / rate;
 807
 808             sync->count_frames += AC3_SAMPLES_PER_FRAME;
 809         }
 810         else
 811         {
 812             hb_buffer_t * buf_raw = hb_fifo_get( audio->fifo_raw );
 813
 814             int count_in, count_out;
 815
 816             count_in  = buf_raw->size / HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) / sizeof( float );
 817             count_out = ( buf_raw->stop - buf_raw->start ) * job->arate / 90000;
 818             if( buf->start < pts_expected - 1500 )
 819                 count_out--;
 820             else if( buf->start > pts_expected + 1500 )
 821                 count_out++;
 822
 823             sync->data.data_in      = (float *) buf_raw->data;
 824             sync->data.input_frames = count_in;
 825             sync->data.output_frames = count_out;
 826
 827             sync->data.src_ratio = (double) sync->data.output_frames /
 828                                    (double) sync->data.input_frames;
 829
 830             buf = hb_buffer_init( sync->data.output_frames * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) *
 831                                   sizeof( float ) );
 832             sync->data.data_out = (float *) buf->data;
 833             if( src_process( sync->state, &sync->data ) )
 834             {
 835                 /* XXX If this happens, we're screwed */
 836                 hb_log( "sync: src_process failed" );
 837             }
 838             hb_buffer_close( &buf_raw );
 839
 840             buf->size = sync->data.output_frames_gen * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) * sizeof( float );
 841
 842             /* Set dates for resampled data */
 843             buf->start = start;
 844             buf->stop  = start + sync->data.output_frames_gen *
 845                             90000 / job->arate;
 846
 847             sync->count_frames += sync->data.output_frames_gen;
 848         }
 849
 850         buf->frametype = HB_FRAME_AUDIO;
 851         hb_fifo_push( fifo, buf );
 852     }
 853
 854     if( hb_fifo_is_full( fifo ) &&
 855         pv->way_out_of_sync )
 856     {
 857         /*
 858          * Trash the top audio packet to avoid dead lock as we reconverge.
 859          */
 860         if ( (buf = hb_fifo_get( audio->fifo_raw ) ) != NULL)
 861             hb_buffer_close( &buf );
 862     }
 863
 864     if( NeedSilence( w, audio ) )
 865     {
 866         InsertSilence( w, i );
 867     }
 868 }
 869
 870 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio )
 871 {
 872     hb_work_private_t * pv = w->private_data;
 873     hb_job_t * job = pv->job;
 874
 875     if( hb_fifo_size( audio->fifo_in ) ||
 876         hb_fifo_size( audio->fifo_raw ) ||
 877         hb_fifo_size( audio->fifo_sync ) ||
 878         hb_fifo_size( audio->fifo_out ) )
 879     {
 880         /* We have some audio, we are fine */
 881         return 0;
 882     }
 883
 884     /* No audio left in fifos */
 885
 886     if( hb_thread_has_exited( job->reader ) )
 887     {
 888         /* We might miss some audio to complete encoding and muxing
 889            the video track */
 890         hb_log("Reader has exited early, inserting silence.");
 891         return 1;
 892     }
 893
 894     if( hb_fifo_is_full( job->fifo_mpeg2 ) &&
 895         hb_fifo_is_full( job->fifo_raw ) &&
 896         hb_fifo_is_full( job->fifo_sync ) &&
 897         hb_fifo_is_full( job->fifo_render ) &&
 898         hb_fifo_is_full( job->fifo_mpeg4 ) )
 899     {
 900         /* Too much video and no audio, oh-oh */
 901         hb_log("Still got some video - and nothing in the audio fifo, insert silence");
 902         return 1;
 903     }
 904
 905     return 0;
 906 }
 907
 908 static void InsertSilence( hb_work_object_t * w, int i )
 909 {
 910     hb_work_private_t * pv = w->private_data;
 911     hb_job_t        * job;
 912     hb_sync_audio_t * sync;
 913     hb_buffer_t     * buf;
 914
 915     job    = pv->job;
 916     sync   = &pv->sync_audio[i];
 917
 918     if( job->acodec & HB_ACODEC_AC3 )
 919     {
 920         buf        = hb_buffer_init( sync->ac3_size );
 921         buf->start = sync->count_frames * 90000 / sync->audio->rate;
 922         buf->stop  = buf->start + 90000 * AC3_SAMPLES_PER_FRAME /
 923                      sync->audio->rate;
 924         memcpy( buf->data, sync->ac3_buf, buf->size );
 925
 926         hb_log( "sync: adding a silent AC-3 frame for track %x",
 927                 sync->audio->id );
 928         hb_fifo_push( sync->audio->fifo_out, buf );
 929
 930         sync->count_frames += AC3_SAMPLES_PER_FRAME;
 931
 932     }
 933     else
 934     {
 935         buf        = hb_buffer_init( HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown) * job->arate / 20 *
 936                                      sizeof( float ) );
 937         buf->start = sync->count_frames * 90000 / job->arate;
 938         buf->stop  = buf->start + 90000 / 20;
 939         memset( buf->data, 0, buf->size );
 940
 941         hb_fifo_push( sync->audio->fifo_sync, buf );
 942
 943         sync->count_frames += job->arate / 20;
 944     }
 945 }
 946
 947 static void UpdateState( hb_work_object_t * w )
 948 {
 949     hb_work_private_t * pv = w->private_data;
 950     hb_state_t state;
 951
 952     if( !pv->count_frames )
 953     {
 954         pv->st_first = hb_get_date();
 955     }
 956     pv->count_frames++;
 957
 958     if( hb_get_date() > pv->st_dates[3] + 1000 )
 959     {
 960         memmove( &pv->st_dates[0], &pv->st_dates[1],
 961                  3 * sizeof( uint64_t ) );
 962         memmove( &pv->st_counts[0], &pv->st_counts[1],
 963                  3 * sizeof( uint64_t ) );
 964         pv->st_dates[3]  = hb_get_date();
 965         pv->st_counts[3] = pv->count_frames;
 966     }
 967
 968 #define p state.param.working
 969     state.state = HB_STATE_WORKING;
 970     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
 971     if( p.progress > 1.0 )
 972     {
 973         p.progress = 1.0;
 974     }
 975     p.rate_cur   = 1000.0 *
 976         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
 977         (float) ( pv->st_dates[3] - pv->st_dates[0] );
 978     if( hb_get_date() > pv->st_first + 4000 )
 979     {
 980         int eta;
 981         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
 982             (float) ( pv->st_dates[3] - pv->st_first );
 983         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
 984             p.rate_avg;
 985         p.hours   = eta / 3600;
 986         p.minutes = ( eta % 3600 ) / 60;
 987         p.seconds = eta % 60;
 988     }
 989     else
 990     {
 991         p.rate_avg = 0.0;
 992         p.hours    = -1;
 993         p.minutes  = -1;
 994         p.seconds  = -1;
 995     }
 996 #undef p
 997
 998     hb_set_state( pv->job->h, &state );
 999 }