libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.m0k.org/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8
   9 #include "samplerate.h"
  10 #include "ffmpeg/avcodec.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22     int64_t      count_frames;
  23
  24     /* Raw */
  25     SRC_STATE  * state;
  26     SRC_DATA     data;
  27
  28     /* AC-3 */
  29     int          ac3_size;
  30     uint8_t    * ac3_buf;
  31
  32 } hb_sync_audio_t;
  33
  34 struct hb_work_private_s
  35 {
  36     hb_job_t * job;
  37     int        done;
  38
  39     /* Video */
  40     hb_subtitle_t * subtitle;
  41     int64_t pts_offset;
  42     int64_t pts_offset_old;
  43     int64_t next_start;
  44     int64_t count_frames;
  45     int64_t count_frames_max;
  46     int64_t video_sequence;
  47     hb_buffer_t * cur; /* The next picture to process */
  48
  49     /* Audio */
  50     hb_sync_audio_t sync_audio[8];
  51
  52     /* Flags */
  53     int discontinuity;
  54
  55     /* Statistics */
  56     uint64_t st_counts[4];
  57     uint64_t st_dates[4];
  58     uint64_t st_first;
  59
  60     /* Throttle message flags */
  61     int   trashing_audio;
  62     int   inserting_silence;
  63     int   way_out_of_sync;
  64 };
  65
  66 /***********************************************************************
  67  * Local prototypes
  68  **********************************************************************/
  69 static void InitAudio( hb_work_object_t * w, int i );
  70 static int  SyncVideo( hb_work_object_t * w );
  71 static void SyncAudio( hb_work_object_t * w, int i );
  72 static int  NeedSilence( hb_work_object_t * w, hb_audio_t * );
  73 static void InsertSilence( hb_work_object_t * w, int i );
  74 static void UpdateState( hb_work_object_t * w );
  75
  76 /***********************************************************************
  77  * hb_work_sync_init
  78  ***********************************************************************
  79  * Initialize the work object
  80  **********************************************************************/
  81 int syncInit( hb_work_object_t * w, hb_job_t * job )
  82 {
  83     hb_title_t       * title = job->title;
  84     hb_chapter_t     * chapter;
  85     int                i;
  86     uint64_t           duration;
  87     hb_work_private_t * pv;
  88
  89     pv = calloc( 1, sizeof( hb_work_private_t ) );
  90     w->private_data = pv;
  91
  92     pv->job            = job;
  93     pv->pts_offset     = INT64_MIN;
  94     pv->pts_offset_old = INT64_MIN;
  95     pv->count_frames   = 0;
  96
  97     pv->discontinuity = 0;
  98
  99     pv->trashing_audio = 0;
 100     pv->inserting_silence = 0;
 101     pv->way_out_of_sync = 0;
 102
 103     /* Calculate how many video frames we are expecting */
 104     duration = 0;
 105     for( i = job->chapter_start; i <= job->chapter_end; i++ )
 106     {
 107         chapter   = hb_list_item( title->list_chapter, i - 1 );
 108         duration += chapter->duration;
 109     }
 110     duration += 90000;
 111         /* 1 second safety so we're sure we won't miss anything */
 112     pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
 113
 114     hb_log( "sync: expecting %lld video frames", pv->count_frames_max );
 115
 116     /* Initialize libsamplerate for every audio track we have */
 117     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 118     {
 119         InitAudio( w, i );
 120     }
 121
 122     /* Get subtitle info, if any */
 123     pv->subtitle = hb_list_item( title->list_subtitle, 0 );
 124
 125     pv->video_sequence = 0;
 126
 127     return 0;
 128 }
 129
 130 /***********************************************************************
 131  * Close
 132  ***********************************************************************
 133  *
 134  **********************************************************************/
 135 void syncClose( hb_work_object_t * w )
 136 {
 137     hb_work_private_t * pv = w->private_data;
 138     hb_job_t          * job   = pv->job;
 139     hb_title_t        * title = job->title;
 140
 141     int i;
 142
 143     if( pv->cur ) hb_buffer_close( &pv->cur );
 144
 145     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 146     {
 147         if( job->acodec & HB_ACODEC_AC3 ||
 148             job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 149         {
 150             free( pv->sync_audio[i].ac3_buf );
 151         }
 152         else
 153         {
 154             src_delete( pv->sync_audio[i].state );
 155         }
 156     }
 157
 158     free( pv );
 159     w->private_data = NULL;
 160 }
 161
 162 /***********************************************************************
 163  * Work
 164  ***********************************************************************
 165  * The root routine of this work abject
 166  *
 167  * The way this works is that we are syncing the audio to the PTS of
 168  * the last video that we processed. That's why we skip the audio sync
 169  * if we haven't got a valid PTS from the video yet.
 170  *
 171  **********************************************************************/
 172 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 173               hb_buffer_t ** unused2 )
 174 {
 175     hb_work_private_t * pv = w->private_data;
 176     int i;
 177
 178     /* If we ever got a video frame, handle audio now */
 179     if( pv->pts_offset != INT64_MIN )
 180     {
 181         for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 182         {
 183             SyncAudio( w, i );
 184         }
 185     }
 186
 187     /* Handle video */
 188     return SyncVideo( w );
 189 }
 190
 191 hb_work_object_t hb_sync =
 192 {
 193     WORK_SYNC,
 194     "Synchronization",
 195     syncInit,
 196     syncWork,
 197     syncClose
 198 };
 199
 200 static void InitAudio( hb_work_object_t * w, int i )
 201 {
 202     hb_work_private_t * pv = w->private_data;
 203     hb_job_t        * job   = pv->job;
 204     hb_title_t      * title = job->title;
 205     hb_sync_audio_t * sync;
 206
 207     sync        = &pv->sync_audio[i];
 208     sync->audio = hb_list_item( title->list_audio, i );
 209
 210     if( job->acodec & HB_ACODEC_AC3 ||
 211         job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 212     {
 213         /* Have a silent AC-3 frame ready in case we have to fill a
 214            gap */
 215         AVCodec        * codec;
 216         AVCodecContext * c;
 217         short          * zeros;
 218
 219         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 220         c     = avcodec_alloc_context();
 221
 222         c->bit_rate    = sync->audio->bitrate;
 223         c->sample_rate = sync->audio->rate;
 224         c->channels    = 2;
 225
 226         if( avcodec_open( c, codec ) < 0 )
 227         {
 228             hb_log( "sync: avcodec_open failed" );
 229             return;
 230         }
 231
 232         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 233                                  sizeof( short ) * c->channels, 1 );
 234         sync->ac3_size = sync->audio->bitrate * AC3_SAMPLES_PER_FRAME /
 235                              sync->audio->rate / 8;
 236         sync->ac3_buf  = malloc( sync->ac3_size );
 237
 238         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 239                                   zeros ) != sync->ac3_size )
 240         {
 241             hb_log( "sync: avcodec_encode_audio failed" );
 242         }
 243
 244         free( zeros );
 245         avcodec_close( c );
 246         av_free( c );
 247     }
 248     else
 249     {
 250         /* Initialize libsamplerate */
 251         int error;
 252         sync->state             = src_new( SRC_LINEAR, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown), &error );
 253         sync->data.end_of_input = 0;
 254     }
 255 }
 256
 257
 258
 259 #define PTS_DISCONTINUITY_TOLERANCE 90000
 260
 261 /***********************************************************************
 262  * SyncVideo
 263  ***********************************************************************
 264  *
 265  **********************************************************************/
 266 static int SyncVideo( hb_work_object_t * w )
 267 {
 268     hb_work_private_t * pv = w->private_data;
 269     hb_buffer_t * cur, * next, * sub = NULL;
 270     hb_job_t * job = pv->job;
 271     int64_t pts_expected;
 272     int chap_break;
 273
 274     if( pv->done )
 275     {
 276         return HB_WORK_DONE;
 277     }
 278
 279     if( hb_thread_has_exited( job->reader ) &&
 280         !hb_fifo_size( job->fifo_mpeg2 ) &&
 281         !hb_fifo_size( job->fifo_raw ) )
 282     {
 283         /* All video data has been processed already, we won't get
 284            more */
 285         hb_log( "sync: got %lld frames, %lld expected",
 286                 pv->count_frames, pv->count_frames_max );
 287         pv->done = 1;
 288
 289         hb_buffer_t * buf_tmp;
 290
 291        // Drop an empty buffer into our output to ensure that things
 292        // get flushed all the way out.
 293         buf_tmp = hb_buffer_init(0); // Empty end buffer
 294         hb_fifo_push( job->fifo_sync, buf_tmp );
 295
 296         return HB_WORK_DONE;
 297     }
 298
 299     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 300     {
 301         /* We haven't even got a frame yet */
 302         return HB_WORK_OK;
 303     }
 304     cur = pv->cur;
 305
 306     /* At this point we have a frame to process. Let's check
 307         1) if we will be able to push into the fifo ahead
 308         2) if the next frame is there already, since we need it to
 309            know whether we'll have to repeat the current frame or not */
 310     while( !hb_fifo_is_full( job->fifo_sync ) &&
 311            ( next = hb_fifo_see( job->fifo_raw ) ) )
 312     {
 313         hb_buffer_t * buf_tmp;
 314
 315         if( pv->pts_offset == INT64_MIN )
 316         {
 317             /* This is our first frame */
 318             hb_log( "sync: first pts is %lld", cur->start );
 319             pv->pts_offset = cur->start;
 320         }
 321
 322         /*
 323          * Track the video sequence number localy so that we can sync the audio
 324          * to it using the sequence number as well as the PTS.
 325          */
 326         pv->video_sequence = cur->sequence;
 327
 328         /* Check for PTS jumps over 0.5 second */
 329         if( next->start < cur->start - PTS_DISCONTINUITY_TOLERANCE ||
 330             next->start > cur->start + PTS_DISCONTINUITY_TOLERANCE )
 331         {
 332             hb_log( "Sync: Video PTS discontinuity %s (current buffer start=%lld, next buffer start=%lld)",
 333                     pv->discontinuity ? "second" : "first", cur->start, next->start );
 334
 335             /*
 336              * Do we need to trash the subtitle, is it from the next->start period
 337              * or is it from our old position. If the latter then trash it.
 338              */
 339             if( pv->subtitle )
 340             {
 341                 while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 342                 {
 343                     if( ( sub->start > ( cur->start - PTS_DISCONTINUITY_TOLERANCE ) ) &&
 344                         ( sub->start < ( cur->start + PTS_DISCONTINUITY_TOLERANCE ) ) )
 345                     {
 346                         /*
 347                          * The subtitle is from our current time region which we are
 348                          * jumping from. So trash it as we are about to jump backwards
 349                          * or forwards and don't want it blocking the subtitle fifo.
 350                          */
 351                         hb_log("Trashing subtitle 0x%x due to PTS discontinuity", sub);
 352                         sub = hb_fifo_get( pv->subtitle->fifo_raw );
 353                         hb_buffer_close( &sub );
 354                     } else {
 355                         break;
 356                     }
 357                 }
 358             }
 359
 360             /* Trash current picture */
 361             /* Also, make sure we don't trash a chapter break */
 362             chap_break = cur->new_chap;
 363             hb_buffer_close( &cur );
 364             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 365             cur->new_chap |= chap_break; // Don't stomp existing chapter breaks
 366
 367             /* Calculate new offset */
 368             pv->pts_offset_old = pv->pts_offset;
 369             if ( job->vfr )
 370             {
 371                 pv->pts_offset = cur->start - pv->next_start;
 372             } else {
 373                 pv->pts_offset = cur->start -
 374                     pv->count_frames * pv->job->vrate_base / 300;
 375             }
 376
 377             if( !pv->discontinuity )
 378             {
 379                 pv->discontinuity = 1;
 380             }
 381
 382             pv->video_sequence = cur->sequence;
 383             continue;
 384         }
 385
 386         /* Look for a subtitle for this frame */
 387         if( pv->subtitle )
 388         {
 389             hb_buffer_t * sub2;
 390             while( ( sub = hb_fifo_see( pv->subtitle->fifo_raw ) ) )
 391             {
 392                 /* If two subtitles overlap, make the first one stop
 393                    when the second one starts */
 394                 sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
 395                 if( sub2 && sub->stop > sub2->start )
 396                     sub->stop = sub2->start;
 397
 398                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 399                 //       sub, cur->sequence, sub->sequence);
 400
 401                 if( sub->sequence > cur->sequence )
 402                 {
 403                     /*
 404                      * The video is behind where we are, so wait until
 405                      * it catches up to the same reader point on the
 406                      * DVD. Then our PTS should be in the same region
 407                      * as the video.
 408                      */
 409                     sub = NULL;
 410                     break;
 411                 }
 412
 413                 if( sub->stop > cur->start ) {
 414                     /*
 415                      * The stop time is in the future, so fall through
 416                      * and we'll deal with it in the next block of
 417                      * code.
 418                      */
 419                     break;
 420                 }
 421                 else
 422                 {
 423                     /*
 424                      * The stop time is in the past. But is it due to
 425                      * it having been played already, or has the PTS
 426                      * been reset to 0?
 427                      */
 428                     if( ( cur->start - sub->stop ) > PTS_DISCONTINUITY_TOLERANCE ) {
 429                         /*
 430                          * There is a lot of time between our current
 431                          * video and where this subtitle is ending,
 432                          * assume that we are about to reset the PTS
 433                          * and do not throw away this subtitle.
 434                          */
 435                         break;
 436                     }
 437                 }
 438
 439                 /*
 440                  * The subtitle is older than this picture, trash it
 441                  */
 442                 sub = hb_fifo_get( pv->subtitle->fifo_raw );
 443                 hb_buffer_close( &sub );
 444             }
 445
 446             /*
 447              * There is a valid subtitle, is it time to display it?
 448              */
 449             if( sub )
 450             {
 451                 if( sub->stop > sub->start)
 452                 {
 453                     /*
 454                      * Normal subtitle which ends after it starts, check to
 455                      * see that the current video is between the start and end.
 456                      */
 457                     if( cur->start > sub->start &&
 458                         cur->start < sub->stop )
 459                     {
 460                         /*
 461                          * We should be playing this, so leave the
 462                          * subtitle in place.
 463                          *
 464                          * fall through to display
 465                          */
 466                         if( ( sub->stop - sub->start ) < ( 3 * 90000 ) )
 467                         {
 468                             /*
 469                              * Subtitle is on for less than three seconds, extend
 470                              * the time that it is displayed to make it easier
 471                              * to read. Make it 3 seconds or until the next
 472                              * subtitle is displayed.
 473                              *
 474                              * This is in response to Indochine which only
 475                              * displays subs for 1 second - too fast to read.
 476                              */
 477                             sub->stop = sub->start + ( 3 * 90000 );
 478
 479                             sub2 = hb_fifo_see2( pv->subtitle->fifo_raw );
 480
 481                             if( sub2 && sub->stop > sub2->start )
 482                             {
 483                                 sub->stop = sub2->start;
 484                             }
 485                         }
 486                     }
 487                     else
 488                     {
 489                         /*
 490                          * Defer until the play point is within the subtitle
 491                          */
 492                         sub = NULL;
 493                     }
 494                 }
 495                 else
 496                 {
 497                     /*
 498                      * The end of the subtitle is less than the start, this is a
 499                      * sign of a PTS discontinuity.
 500                      */
 501                     if( sub->start > cur->start )
 502                     {
 503                         /*
 504                          * we haven't reached the start time yet, or
 505                          * we have jumped backwards after having
 506                          * already started this subtitle.
 507                          */
 508                         if( cur->start < sub->stop )
 509                         {
 510                             /*
 511                              * We have jumped backwards and so should
 512                              * continue displaying this subtitle.
 513                              *
 514                              * fall through to display.
 515                              */
 516                         }
 517                         else
 518                         {
 519                             /*
 520                              * Defer until the play point is within the subtitle
 521                              */
 522                             sub = NULL;
 523                         }
 524                     } else {
 525                         /*
 526                          * Play this subtitle as the start is greater than our
 527                          * video point.
 528                          *
 529                          * fall through to display/
 530                          */
 531                     }
 532                 }
 533             }
 534         }
 535
 536         if ( job->vfr )
 537         {
 538             /*
 539              * adjust the pts of the current frame so that it's contiguous
 540              * with the previous frame. pts_offset tracks the time difference
 541              * between the pts values in the input content (which start at some
 542              * random time) and our timestamps (which start at zero). We don't
 543              * make any adjustments to the source timestamps other than removing
 544              * the clock offsets (which also removes pts discontinuities).
 545              * This means we automatically encode at the source's frame rate.
 546              * MP2 uses an implicit duration (frames end when the next frame
 547              * starts) but more advanced containers like MP4 use an explicit
 548              * duration. Since we're looking ahead one frame we set the
 549              * explicit stop time from the start time of the next frame.
 550              */
 551             buf_tmp = cur;
 552             pv->cur = cur = hb_fifo_get( job->fifo_raw );
 553             buf_tmp->start = pv->next_start;
 554             pv->next_start = next->start - pv->pts_offset;
 555             buf_tmp->stop = pv->next_start;
 556         }
 557         else
 558         {
 559             /* The PTS of the frame we are expecting now */
 560             pts_expected = pv->pts_offset +
 561                 pv->count_frames * pv->job->vrate_base / 300;
 562
 563             //hb_log("Video expecting PTS %lld, current frame: %lld, next frame: %lld, cf: %lld",
 564             //       pts_expected, cur->start, next->start, pv->count_frames * pv->job->vrate_base / 300 );
 565
 566             if( cur->start < pts_expected - pv->job->vrate_base / 300 / 2 &&
 567                 next->start < pts_expected + pv->job->vrate_base / 300 / 2 )
 568             {
 569                 /* The current frame is too old but the next one matches,
 570                    let's trash */
 571                 /* Also, make sure we don't trash a chapter break */
 572                 chap_break = cur->new_chap;
 573                 hb_buffer_close( &cur );
 574                 pv->cur = cur = hb_fifo_get( job->fifo_raw );
 575                 cur->new_chap |= chap_break; // Make sure we don't stomp the existing one.
 576
 577                 continue;
 578             }
 579
 580             if( next->start > pts_expected + 3 * pv->job->vrate_base / 300 / 2 )
 581             {
 582                 /* We'll need the current frame more than one time. Make a
 583                    copy of it and keep it */
 584                 buf_tmp = hb_buffer_init( cur->size );
 585                 memcpy( buf_tmp->data, cur->data, cur->size );
 586                 buf_tmp->sequence = cur->sequence;
 587             }
 588             else
 589             {
 590                 /* The frame has the expected date and won't have to be
 591                    duplicated, just put it through */
 592                 buf_tmp = cur;
 593                 pv->cur = cur = hb_fifo_get( job->fifo_raw );
 594             }
 595
 596             /* Replace those MPEG-2 dates with our dates */
 597             buf_tmp->start = (uint64_t) pv->count_frames *
 598                 pv->job->vrate_base / 300;
 599             buf_tmp->stop  = (uint64_t) ( pv->count_frames + 1 ) *
 600                 pv->job->vrate_base / 300;
 601         }
 602
 603         /* If we have a subtitle for this picture, copy it */
 604         /* FIXME: we should avoid this memcpy */
 605         if( sub )
 606         {
 607             buf_tmp->sub         = hb_buffer_init( sub->size );
 608             buf_tmp->sub->x      = sub->x;
 609             buf_tmp->sub->y      = sub->y;
 610             buf_tmp->sub->width  = sub->width;
 611             buf_tmp->sub->height = sub->height;
 612             memcpy( buf_tmp->sub->data, sub->data, sub->size );
 613         }
 614
 615         /* Push the frame to the renderer */
 616         hb_fifo_push( job->fifo_sync, buf_tmp );
 617
 618         /* Update UI */
 619         UpdateState( w );
 620
 621         /* Make sure we won't get more frames then expected */
 622         if( pv->count_frames >= pv->count_frames_max * 2)
 623         {
 624             hb_log( "sync: got too many frames (%lld), exiting early", pv->count_frames );
 625             pv->done = 1;
 626
 627            // Drop an empty buffer into our output to ensure that things
 628            // get flushed all the way out.
 629            buf_tmp = hb_buffer_init(0); // Empty end buffer
 630            hb_fifo_push( job->fifo_sync, buf_tmp );
 631
 632             break;
 633         }
 634     }
 635
 636     return HB_WORK_OK;
 637 }
 638
 639 /***********************************************************************
 640  * SyncAudio
 641  ***********************************************************************
 642  *
 643  **********************************************************************/
 644 static void SyncAudio( hb_work_object_t * w, int i )
 645 {
 646     hb_work_private_t * pv = w->private_data;
 647     hb_job_t        * job;
 648     hb_audio_t      * audio;
 649     hb_buffer_t     * buf;
 650     hb_sync_audio_t * sync;
 651
 652     hb_fifo_t       * fifo;
 653     int               rate;
 654
 655     int64_t           pts_expected;
 656     int64_t           start;
 657
 658     job    = pv->job;
 659     sync   = &pv->sync_audio[i];
 660     audio  = sync->audio;
 661
 662     if( job->acodec & HB_ACODEC_AC3 ||
 663         job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 664     {
 665         fifo = audio->fifo_out;
 666         rate = audio->rate;
 667     }
 668     else
 669     {
 670         fifo = audio->fifo_sync;
 671         rate = job->arate;
 672     }
 673
 674     while( !hb_fifo_is_full( fifo ) &&
 675            ( buf = hb_fifo_see( audio->fifo_raw ) ) )
 676     {
 677         /* The PTS of the samples we are expecting now */
 678         pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 679
 680         // hb_log("Video Sequence: %lld, Audio Sequence: %lld", pv->video_sequence, buf->sequence);
 681
 682         /*
 683          * Using the same logic as the Video have we crossed a VOB
 684          * boundary as detected by the expected PTS and the PTS of our
 685          * audio being out by more than the tolerance value.
 686          */
 687         if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 688             buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 689         {
 690             /* There has been a PTS discontinuity, and this frame might
 691                be from before the discontinuity*/
 692
 693             if( pv->discontinuity )
 694             {
 695                 /*
 696                  * There is an outstanding discontinuity, so use the offset from
 697                  * that discontinuity.
 698                  */
 699                 pts_expected = pv->pts_offset_old + sync->count_frames *
 700                     90000 / rate;
 701             }
 702             else
 703             {
 704                 /*
 705                  * No outstanding discontinuity, so the audio must be leading the
 706                  * video (or the PTS values are really stuffed). So lets mark this
 707                  * as a discontinuity ourselves for the audio to use until
 708                  * the video also crosses the discontinuity.
 709                  *
 710                  * pts_offset is used when we are in the same time space as the video
 711                  * pts_offset_old when in a discontinuity.
 712                  *
 713                  * Therefore set the pts_offset_old given the new pts_offset for this
 714                  * current buffer.
 715                  */
 716                 pv->discontinuity = 1;
 717                 pv->pts_offset_old = buf->start - sync->count_frames *
 718                     90000 / rate;
 719                 pts_expected = pv->pts_offset_old + sync->count_frames *
 720                     90000 / rate;
 721
 722                 hb_log("Sync: Audio discontinuity (sequence: vid %lld aud %lld) (pts %lld < %lld < %lld)",
 723                        pv->video_sequence, buf->sequence,
 724                        pts_expected - PTS_DISCONTINUITY_TOLERANCE, buf->start,
 725                        pts_expected + PTS_DISCONTINUITY_TOLERANCE );
 726             }
 727
 728             /*
 729              * Is the audio from a valid period given the previous
 730              * Video PTS. I.e. has there just been a video PTS
 731              * discontinuity and this audio belongs to the vdeo from
 732              * before?
 733              */
 734             if( buf->start > pts_expected + PTS_DISCONTINUITY_TOLERANCE ||
 735                 buf->start < pts_expected - PTS_DISCONTINUITY_TOLERANCE )
 736             {
 737                 /*
 738                  * It's outside of our tolerance for where the video
 739                  * is now, and it's outside of the tolerance for
 740                  * where we have been in the case of a VOB change.
 741                  * Try and reconverge regardless. so continue on to
 742                  * our convergence code below which will kick in as
 743                  * it will be more than 100ms out.
 744                  *
 745                  * Note that trashing the Audio could make things
 746                  * worse if the Audio is in front because we will end
 747                  * up diverging even more. We need to hold on to the
 748                  * audio until the video catches up.
 749                  */
 750                 if( !pv->way_out_of_sync )
 751                 {
 752                     hb_log("Sync: Audio is way out of sync, attempt to reconverge from current video PTS");
 753                     pv->way_out_of_sync = 1;
 754                 }
 755
 756                 /*
 757                  * It wasn't from the old place, so we must be from
 758                  * the new, but just too far out. So attempt to
 759                  * reconverge by resetting the point we want to be to
 760                  * where we are currently wanting to be.
 761                  */
 762                 pts_expected = pv->pts_offset + sync->count_frames * 90000 / rate;
 763                 start = pts_expected - pv->pts_offset;
 764             } else {
 765                  /* Use the older offset */
 766                 start = pts_expected - pv->pts_offset_old;
 767             }
 768         }
 769         else
 770         {
 771             start = pts_expected - pv->pts_offset;
 772
 773             if( pv->discontinuity )
 774             {
 775                 /*
 776                  * The Audio is tracking the Video again using the normal pts_offset, so the
 777                  * discontinuity is over.
 778                  */
 779                 hb_log( "Sync: Audio joined Video after discontinuity at PTS %lld", buf->start );
 780                 pv->discontinuity = 0;
 781             }
 782         }
 783
 784         /* Tolerance: 100 ms */
 785         if( buf->start < pts_expected - 9000 )
 786         {
 787             if( !pv->trashing_audio )
 788             {
 789                 /* Audio is behind the Video, trash it, can't use it now. */
 790                 hb_log( "Sync: Audio PTS (%lld) < Video PTS (%lld) by greater than 100ms, trashing audio to reconverge",
 791                         buf->start, pts_expected);
 792                 pv->trashing_audio = 1;
 793             }
 794             buf = hb_fifo_get( audio->fifo_raw );
 795             hb_buffer_close( &buf );
 796             continue;
 797         }
 798         else if( buf->start > pts_expected + 9000 )
 799         {
 800             /* Audio is ahead of the Video, insert silence until we catch up*/
 801             if( !pv->inserting_silence )
 802             {
 803                 hb_log("Sync: Audio PTS (%lld) >  Video PTS (%lld) by greater than 100ms insert silence until reconverged", buf->start, pts_expected);
 804                 pv->inserting_silence = 1;
 805             }
 806             InsertSilence( w, i );
 807             continue;
 808         }
 809         else
 810         {
 811             if( pv->trashing_audio || pv->inserting_silence )
 812             {
 813                 hb_log( "Sync: Audio back in Sync at PTS %lld", buf->start );
 814                 pv->trashing_audio = 0;
 815                 pv->inserting_silence = 0;
 816             }
 817             if( pv->way_out_of_sync )
 818             {
 819                 hb_log( "Sync: Audio no longer way out of sync at PTS %lld",
 820                         buf->start );
 821                 pv->way_out_of_sync = 0;
 822             }
 823         }
 824
 825         if( job->acodec & HB_ACODEC_AC3 ||
 826             job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 827         {
 828             buf        = hb_fifo_get( audio->fifo_raw );
 829             buf->start = start;
 830             buf->stop  = start + 90000 * AC3_SAMPLES_PER_FRAME / rate;
 831
 832             sync->count_frames += AC3_SAMPLES_PER_FRAME;
 833         }
 834         else
 835         {
 836             hb_buffer_t * buf_raw = hb_fifo_get( audio->fifo_raw );
 837
 838             int count_in, count_out;
 839
 840             count_in  = buf_raw->size / HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) / sizeof( float );
 841             count_out = ( buf_raw->stop - buf_raw->start ) * job->arate / 90000;
 842             if( buf->start < pts_expected - 1500 )
 843                 count_out--;
 844             else if( buf->start > pts_expected + 1500 )
 845                 count_out++;
 846
 847             sync->data.data_in      = (float *) buf_raw->data;
 848             sync->data.input_frames = count_in;
 849             sync->data.output_frames = count_out;
 850
 851             sync->data.src_ratio = (double) sync->data.output_frames /
 852                                    (double) sync->data.input_frames;
 853
 854             buf = hb_buffer_init( sync->data.output_frames * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) *
 855                                   sizeof( float ) );
 856             sync->data.data_out = (float *) buf->data;
 857             if( src_process( sync->state, &sync->data ) )
 858             {
 859                 /* XXX If this happens, we're screwed */
 860                 hb_log( "sync: src_process failed" );
 861             }
 862             hb_buffer_close( &buf_raw );
 863
 864             buf->size = sync->data.output_frames_gen * HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->amixdown) * sizeof( float );
 865
 866             /* Set dates for resampled data */
 867             buf->start = start;
 868             buf->stop  = start + sync->data.output_frames_gen *
 869                             90000 / job->arate;
 870
 871             sync->count_frames += sync->data.output_frames_gen;
 872         }
 873
 874         buf->frametype = HB_FRAME_AUDIO;
 875         hb_fifo_push( fifo, buf );
 876     }
 877
 878     if( hb_fifo_is_full( fifo ) &&
 879         pv->way_out_of_sync )
 880     {
 881         /*
 882          * Trash the top audio packet to avoid dead lock as we reconverge.
 883          */
 884         if ( (buf = hb_fifo_get( audio->fifo_raw ) ) != NULL)
 885             hb_buffer_close( &buf );
 886     }
 887
 888     if( NeedSilence( w, audio ) )
 889     {
 890         InsertSilence( w, i );
 891     }
 892 }
 893
 894 static int NeedSilence( hb_work_object_t * w, hb_audio_t * audio )
 895 {
 896     hb_work_private_t * pv = w->private_data;
 897     hb_job_t * job = pv->job;
 898
 899     if( hb_fifo_size( audio->fifo_in ) ||
 900         hb_fifo_size( audio->fifo_raw ) ||
 901         hb_fifo_size( audio->fifo_sync ) ||
 902         hb_fifo_size( audio->fifo_out ) )
 903     {
 904         /* We have some audio, we are fine */
 905         return 0;
 906     }
 907
 908     /* No audio left in fifos */
 909
 910     if( hb_thread_has_exited( job->reader ) )
 911     {
 912         /* We might miss some audio to complete encoding and muxing
 913            the video track */
 914         hb_log("Reader has exited early, inserting silence.");
 915         return 1;
 916     }
 917
 918     if( hb_fifo_is_full( job->fifo_mpeg2 ) &&
 919         hb_fifo_is_full( job->fifo_raw ) &&
 920         hb_fifo_is_full( job->fifo_sync ) &&
 921         hb_fifo_is_full( job->fifo_render ) &&
 922         hb_fifo_is_full( job->fifo_mpeg4 ) )
 923     {
 924         /* Too much video and no audio, oh-oh */
 925         hb_log("Still got some video - and nothing in the audio fifo, insert silence");
 926         return 1;
 927     }
 928
 929     return 0;
 930 }
 931
 932 static void InsertSilence( hb_work_object_t * w, int i )
 933 {
 934     hb_work_private_t * pv = w->private_data;
 935     hb_job_t        * job;
 936     hb_sync_audio_t * sync;
 937     hb_buffer_t     * buf;
 938
 939     job    = pv->job;
 940     sync   = &pv->sync_audio[i];
 941
 942     if( job->acodec & HB_ACODEC_AC3 ||
 943         job->audio_mixdowns[i] == HB_AMIXDOWN_AC3 )
 944     {
 945         buf        = hb_buffer_init( sync->ac3_size );
 946         buf->start = sync->count_frames * 90000 / sync->audio->rate;
 947         buf->stop  = buf->start + 90000 * AC3_SAMPLES_PER_FRAME /
 948                      sync->audio->rate;
 949         memcpy( buf->data, sync->ac3_buf, buf->size );
 950
 951         hb_log( "sync: adding a silent AC-3 frame for track %x",
 952                 sync->audio->id );
 953         hb_fifo_push( sync->audio->fifo_out, buf );
 954
 955         sync->count_frames += AC3_SAMPLES_PER_FRAME;
 956
 957     }
 958     else
 959     {
 960         buf        = hb_buffer_init( HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->amixdown) * job->arate / 20 *
 961                                      sizeof( float ) );
 962         buf->start = sync->count_frames * 90000 / job->arate;
 963         buf->stop  = buf->start + 90000 / 20;
 964         memset( buf->data, 0, buf->size );
 965
 966         hb_fifo_push( sync->audio->fifo_sync, buf );
 967
 968         sync->count_frames += job->arate / 20;
 969     }
 970 }
 971
 972 static void UpdateState( hb_work_object_t * w )
 973 {
 974     hb_work_private_t * pv = w->private_data;
 975     hb_state_t state;
 976
 977     if( !pv->count_frames )
 978     {
 979         pv->st_first = hb_get_date();
 980     }
 981     pv->count_frames++;
 982
 983     if( hb_get_date() > pv->st_dates[3] + 1000 )
 984     {
 985         memmove( &pv->st_dates[0], &pv->st_dates[1],
 986                  3 * sizeof( uint64_t ) );
 987         memmove( &pv->st_counts[0], &pv->st_counts[1],
 988                  3 * sizeof( uint64_t ) );
 989         pv->st_dates[3]  = hb_get_date();
 990         pv->st_counts[3] = pv->count_frames;
 991     }
 992
 993 #define p state.param.working
 994     state.state = HB_STATE_WORKING;
 995     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
 996     if( p.progress > 1.0 )
 997     {
 998         p.progress = 1.0;
 999     }
1000     p.rate_cur   = 1000.0 *
1001         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
1002         (float) ( pv->st_dates[3] - pv->st_dates[0] );
1003     if( hb_get_date() > pv->st_first + 4000 )
1004     {
1005         int eta;
1006         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
1007             (float) ( pv->st_dates[3] - pv->st_first );
1008         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
1009             p.rate_avg;
1010         p.hours   = eta / 3600;
1011         p.minutes = ( eta % 3600 ) / 60;
1012         p.seconds = eta % 60;
1013     }
1014     else
1015     {
1016         p.rate_avg = 0.0;
1017         p.hours    = -1;
1018         p.minutes  = -1;
1019         p.seconds  = -1;
1020     }
1021 #undef p
1022
1023     hb_set_state( pv->job->h, &state );
1024 }