libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.fr/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8 #include "hbffmpeg.h"
   9 #include <stdio.h>
  10 #include "samplerate.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22
  23     int64_t      next_start;    /* start time of next output frame */
  24     int64_t      next_pts;      /* start time of next input frame */
  25     int64_t      first_drop;    /* PTS of first 'went backwards' frame dropped */
  26     int          drop_count;    /* count of 'time went backwards' drops */
  27
  28     /* Raw */
  29     SRC_STATE  * state;
  30     SRC_DATA     data;
  31
  32     /* AC-3 */
  33     int          ac3_size;
  34     uint8_t    * ac3_buf;
  35
  36 } hb_sync_audio_t;
  37
  38 struct hb_work_private_s
  39 {
  40     hb_job_t * job;
  41     int        busy;            // bitmask with one bit for each active input
  42                                 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
  43                                 // appropriate bit is cleared when input gets
  44                                 // an eof buf. syncWork returns done when all
  45                                 // bits are clear.
  46     /* Video */
  47     int64_t pts_offset;
  48     int64_t next_start;         /* start time of next output frame */
  49     int64_t next_pts;           /* start time of next input frame */
  50     int64_t first_drop;         /* PTS of first 'went backwards' frame dropped */
  51     int drop_count;             /* count of 'time went backwards' drops */
  52     int drops;                  /* frames dropped to make a cbr video stream */
  53     int dups;                   /* frames duplicated to make a cbr video stream */
  54     int video_sequence;
  55     int count_frames;
  56     int count_frames_max;
  57     int chap_mark;              /* to propagate chapter mark across a drop */
  58     hb_buffer_t * cur; /* The next picture to process */
  59
  60     /* Audio */
  61     hb_sync_audio_t sync_audio[8];
  62     int64_t audio_passthru_slip;
  63     int64_t video_pts_slip;
  64
  65     /* Statistics */
  66     uint64_t st_counts[4];
  67     uint64_t st_dates[4];
  68     uint64_t st_first;
  69 };
  70
  71 /***********************************************************************
  72  * Local prototypes
  73  **********************************************************************/
  74 static void InitAudio( hb_work_object_t * w, int i );
  75 static void SyncVideo( hb_work_object_t * w );
  76 static void SyncAudio( hb_work_object_t * w, int i );
  77 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
  78 static void UpdateState( hb_work_object_t * w );
  79
  80 /***********************************************************************
  81  * hb_work_sync_init
  82  ***********************************************************************
  83  * Initialize the work object
  84  **********************************************************************/
  85 int syncInit( hb_work_object_t * w, hb_job_t * job )
  86 {
  87     hb_title_t       * title = job->title;
  88     hb_chapter_t     * chapter;
  89     int                i;
  90     uint64_t           duration;
  91     hb_work_private_t * pv;
  92
  93     pv = calloc( 1, sizeof( hb_work_private_t ) );
  94     w->private_data = pv;
  95
  96     pv->job            = job;
  97     pv->pts_offset     = INT64_MIN;
  98
  99     /* Calculate how many video frames we are expecting */
 100     if (job->pts_to_stop)
 101     {
 102         duration = job->pts_to_stop + 90000;
 103     }
 104     else if( job->frame_to_stop )
 105     {
 106         /* Set the duration to a rough estimate */
 107         duration = ( job->frame_to_stop / ( job->vrate / job->vrate_base ) ) * 90000;
 108     }
 109     else
 110     {
 111         duration = 0;
 112         for( i = job->chapter_start; i <= job->chapter_end; i++ )
 113         {
 114             chapter   = hb_list_item( title->list_chapter, i - 1 );
 115             duration += chapter->duration;
 116         }
 117         duration += 90000;
 118         /* 1 second safety so we're sure we won't miss anything */
 119     }
 120     pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
 121
 122     hb_log( "sync: expecting %d video frames", pv->count_frames_max );
 123     pv->busy |= 1;
 124
 125     /* Initialize libsamplerate for every audio track we have */
 126     if ( ! job->indepth_scan )
 127     {
 128         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
 129         {
 130             pv->busy |= ( 1 << (i + 1) );
 131             InitAudio( w, i );
 132         }
 133     }
 134
 135     return 0;
 136 }
 137
 138 /***********************************************************************
 139  * Close
 140  ***********************************************************************
 141  *
 142  **********************************************************************/
 143 void syncClose( hb_work_object_t * w )
 144 {
 145     hb_work_private_t * pv = w->private_data;
 146     hb_job_t          * job   = pv->job;
 147     hb_title_t        * title = job->title;
 148     hb_audio_t        * audio = NULL;
 149     int i;
 150
 151     if( pv->cur )
 152     {
 153         hb_buffer_close( &pv->cur );
 154     }
 155
 156     hb_log( "sync: got %d frames, %d expected",
 157             pv->count_frames, pv->count_frames_max );
 158
 159     if (pv->drops || pv->dups )
 160     {
 161         hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
 162     }
 163
 164     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 165     {
 166         audio = hb_list_item( title->list_audio, i );
 167         if( audio->config.out.codec == HB_ACODEC_AC3 )
 168         {
 169             free( pv->sync_audio[i].ac3_buf );
 170         }
 171         else
 172         {
 173             src_delete( pv->sync_audio[i].state );
 174         }
 175     }
 176
 177     free( pv );
 178     w->private_data = NULL;
 179 }
 180
 181 /***********************************************************************
 182  * Work
 183  ***********************************************************************
 184  * The root routine of this work abject
 185  *
 186  * The way this works is that we are syncing the audio to the PTS of
 187  * the last video that we processed. That's why we skip the audio sync
 188  * if we haven't got a valid PTS from the video yet.
 189  *
 190  **********************************************************************/
 191 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 192               hb_buffer_t ** unused2 )
 193 {
 194     hb_work_private_t * pv = w->private_data;
 195     int i;
 196
 197     if ( pv->busy & 1 )
 198         SyncVideo( w );
 199
 200     for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 201     {
 202         if ( pv->busy & ( 1 << (i + 1) ) )
 203             SyncAudio( w, i );
 204     }
 205
 206     return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
 207 }
 208
 209 hb_work_object_t hb_sync =
 210 {
 211     WORK_SYNC,
 212     "Synchronization",
 213     syncInit,
 214     syncWork,
 215     syncClose
 216 };
 217
 218 static void InitAudio( hb_work_object_t * w, int i )
 219 {
 220     hb_work_private_t * pv = w->private_data;
 221     hb_job_t        * job   = pv->job;
 222     hb_title_t      * title = job->title;
 223     hb_sync_audio_t * sync;
 224
 225     sync        = &pv->sync_audio[i];
 226     sync->audio = hb_list_item( title->list_audio, i );
 227
 228     if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 229     {
 230         /* Have a silent AC-3 frame ready in case we have to fill a
 231            gap */
 232         AVCodec        * codec;
 233         AVCodecContext * c;
 234         short          * zeros;
 235
 236         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 237         c     = avcodec_alloc_context();
 238
 239         c->bit_rate    = sync->audio->config.in.bitrate;
 240         c->sample_rate = sync->audio->config.in.samplerate;
 241         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
 242
 243         if( hb_avcodec_open( c, codec ) < 0 )
 244         {
 245             hb_log( "sync: avcodec_open failed" );
 246             return;
 247         }
 248
 249         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 250                                  sizeof( short ) * c->channels, 1 );
 251         sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
 252                              sync->audio->config.in.samplerate / 8;
 253         sync->ac3_buf  = malloc( sync->ac3_size );
 254
 255         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 256                                   zeros ) != sync->ac3_size )
 257         {
 258             hb_log( "sync: avcodec_encode_audio failed" );
 259         }
 260
 261         free( zeros );
 262         hb_avcodec_close( c );
 263         av_free( c );
 264     }
 265     else
 266     {
 267         /* Initialize libsamplerate */
 268         int error;
 269         sync->state             = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
 270         sync->data.end_of_input = 0;
 271     }
 272 }
 273
 274 /***********************************************************************
 275  * SyncVideo
 276  ***********************************************************************
 277  *
 278  **********************************************************************/
 279 static void SyncVideo( hb_work_object_t * w )
 280 {
 281     hb_work_private_t * pv = w->private_data;
 282     hb_buffer_t * cur, * next, * sub = NULL;
 283     hb_job_t * job = pv->job;
 284     hb_subtitle_t *subtitle;
 285     int i;
 286     int64_t pts_skip;
 287
 288     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 289     {
 290         /* We haven't even got a frame yet */
 291         return;
 292     }
 293     cur = pv->cur;
 294     pts_skip = 0;
 295     if( cur->size == 0 )
 296     {
 297         /* we got an end-of-stream. Feed it downstream & signal that we're done. */
 298         hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 299
 300         /*
 301          * Push through any subtitle EOFs in case they were not synced through.
 302          */
 303         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 304         {
 305             subtitle = hb_list_item( job->list_subtitle, i );
 306             if( subtitle->config.dest == PASSTHRUSUB )
 307             {
 308                 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 309             }
 310         }
 311
 312         pv->busy &=~ 1;
 313         return;
 314     }
 315
 316     /* At this point we have a frame to process. Let's check
 317         1) if we will be able to push into the fifo ahead
 318         2) if the next frame is there already, since we need it to
 319            compute the duration of the current frame*/
 320     while( !hb_fifo_is_full( job->fifo_sync ) &&
 321            ( next = hb_fifo_see( job->fifo_raw ) ) )
 322     {
 323         hb_buffer_t * buf_tmp;
 324
 325         if( next->size == 0 )
 326         {
 327             /* we got an end-of-stream. Feed it downstream & signal that
 328              * we're done. Note that this means we drop the final frame of
 329              * video (we don't know its duration). On DVDs the final frame
 330              * is often strange and dropping it seems to be a good idea. */
 331             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 332
 333             /*
 334              * Push through any subtitle EOFs in case they were not synced through.
 335              */
 336             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 337             {
 338                 subtitle = hb_list_item( job->list_subtitle, i );
 339                 if( subtitle->config.dest == PASSTHRUSUB )
 340                 {
 341                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 342                 }
 343             }
 344             pv->busy &=~ 1;
 345             return;
 346         }
 347         if( pv->pts_offset == INT64_MIN )
 348         {
 349             /* This is our first frame */
 350             pv->pts_offset = 0;
 351             if ( cur->start != 0 )
 352             {
 353                 /*
 354                  * The first pts from a dvd should always be zero but
 355                  * can be non-zero with a transport or program stream since
 356                  * we're not guaranteed to start on an IDR frame. If we get
 357                  * a non-zero initial PTS extend its duration so it behaves
 358                  * as if it started at zero so that our audio timing will
 359                  * be in sync.
 360                  */
 361                 hb_log( "sync: first pts is %lld", cur->start );
 362                 cur->start = 0;
 363             }
 364         }
 365
 366         /*
 367          * since the first frame is always 0 and the upstream reader code
 368          * is taking care of adjusting for pts discontinuities, we just have
 369          * to deal with the next frame's start being in the past. This can
 370          * happen when the PTS is adjusted after data loss but video frame
 371          * reordering causes some frames with the old clock to appear after
 372          * the clock change. This creates frames that overlap in time which
 373          * looks to us like time going backward. The downstream muxing code
 374          * can deal with overlaps of up to a frame time but anything larger
 375          * we handle by dropping frames here.
 376          */
 377         if ( (int64_t)( next->start - pv->video_pts_slip - cur->start ) <= 0 )
 378         {
 379             if ( pv->first_drop == 0 )
 380             {
 381                 pv->first_drop = next->start;
 382             }
 383             ++pv->drop_count;
 384             if (next->start - cur->start > 0)
 385             {
 386                 pts_skip += next->start - cur->start;
 387                 pv->video_pts_slip -= next->start - cur->start;
 388             }
 389             buf_tmp = hb_fifo_get( job->fifo_raw );
 390             if ( buf_tmp->new_chap )
 391             {
 392                 // don't drop a chapter mark when we drop the buffer
 393                 pv->chap_mark = buf_tmp->new_chap;
 394             }
 395             hb_buffer_close( &buf_tmp );
 396             continue;
 397         }
 398         if ( pv->first_drop )
 399         {
 400             hb_log( "sync: video time didn't advance - dropped %d frames "
 401                     "(delta %d ms, current %lld, next %lld, dur %d)",
 402                     pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
 403                     cur->start, next->start, (int)( next->start - cur->start ) );
 404             pv->first_drop = 0;
 405             pv->drop_count = 0;
 406         }
 407
 408         /*
 409          * Track the video sequence number localy so that we can sync the audio
 410          * to it using the sequence number as well as the PTS.
 411          */
 412         pv->video_sequence = cur->sequence;
 413
 414         /*
 415          * Look for a subtitle for this frame.
 416          *
 417          * If found then it will be tagged onto a video buffer of the correct time and
 418          * sent in to the render pipeline. This only needs to be done for VOBSUBs which
 419          * get rendered, other types of subtitles can just sit in their raw_queue until
 420          * delt with at muxing.
 421          */
 422         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 423         {
 424             subtitle = hb_list_item( job->list_subtitle, i );
 425
 426             /*
 427              * Rewrite timestamps on subtitles that need it (on raw queue).
 428              */
 429             if( subtitle->source == CC608SUB ||
 430                 subtitle->source == CC708SUB )
 431             {
 432                 /*
 433                  * Rewrite timestamps on subtitles that came from Closed Captions
 434                  * since they are using the MPEG2 timestamps.
 435                  */
 436                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 437                 {
 438                     /*
 439                      * Rewrite the timestamps as and when the video
 440                      * (cur->start) reaches the same timestamp as a
 441                      * closed caption (sub->start).
 442                      *
 443                      * What about discontinuity boundaries - not delt
 444                      * with here - Van?
 445                      *
 446                      * Bypass the sync fifo altogether.
 447                      */
 448                     if( sub->size <= 0 )
 449                     {
 450                         sub = hb_fifo_get( subtitle->fifo_raw );
 451                         hb_fifo_push( subtitle->fifo_out, sub );
 452                         sub = NULL;
 453                         break;
 454                     } else {
 455                         /*
 456                          * Sync the subtitles to the incoming video, and use
 457                          * the matching converted video timestamp.
 458                          *
 459                          * Note that it doesn't appear that we need to convert
 460                          * timestamps, I guess that they were already correct,
 461                          * so just push them through for rendering.
 462                          *
 463                          */
 464                         if( sub->start < cur->start )
 465                         {
 466                             uint64_t duration;
 467                             duration = sub->stop - sub->start;
 468                             sub = hb_fifo_get( subtitle->fifo_raw );
 469                             hb_fifo_push( subtitle->fifo_out, sub );
 470                         } else {
 471                             sub = NULL;
 472                             break;
 473                         }
 474                     }
 475                 }
 476             }
 477
 478             if( subtitle->source == VOBSUB )
 479             {
 480                 hb_buffer_t * sub2;
 481                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 482                 {
 483                     if( sub->size == 0 )
 484                     {
 485                         /*
 486                          * EOF, pass it through immediately.
 487                          */
 488                         break;
 489                     }
 490
 491                     /* If two subtitles overlap, make the first one stop
 492                        when the second one starts */
 493                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 494                     if( sub2 && sub->stop > sub2->start )
 495                     {
 496                         sub->stop = sub2->start;
 497                     }
 498
 499                     // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 500                     //       sub, cur->sequence, sub->sequence);
 501
 502                     if( sub->sequence > cur->sequence )
 503                     {
 504                         /*
 505                          * The video is behind where we are, so wait until
 506                          * it catches up to the same reader point on the
 507                          * DVD. Then our PTS should be in the same region
 508                          * as the video.
 509                          */
 510                         sub = NULL;
 511                         break;
 512                     }
 513
 514                     if( sub->stop > cur->start ) {
 515                         /*
 516                          * The stop time is in the future, so fall through
 517                          * and we'll deal with it in the next block of
 518                          * code.
 519                          */
 520
 521                         /*
 522                          * There is a valid subtitle, is it time to display it?
 523                          */
 524                         if( sub->stop > sub->start)
 525                         {
 526                             /*
 527                              * Normal subtitle which ends after it starts,
 528                              * check to see that the current video is between
 529                              * the start and end.
 530                              */
 531                             if( cur->start > sub->start &&
 532                                 cur->start < sub->stop )
 533                             {
 534                                 /*
 535                                 * We should be playing this, so leave the
 536                                 * subtitle in place.
 537                                 *
 538                                 * fall through to display
 539                                 */
 540                                 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
 541                                 {
 542                                     /*
 543                                      * Subtitle is on for less than three
 544                                      * seconds, extend the time that it is
 545                                      * displayed to make it easier to read.
 546                                      * Make it 3 seconds or until the next
 547                                      * subtitle is displayed.
 548                                      *
 549                                      * This is in response to Indochine which
 550                                      * only displays subs for 1 second -
 551                                      * too fast to read.
 552                                      */
 553                                     sub->stop = sub->start + ( 2 * 90000 );
 554
 555                                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 556
 557                                     if( sub2 && sub->stop > sub2->start )
 558                                     {
 559                                         sub->stop = sub2->start;
 560                                     }
 561                                 }
 562                             }
 563                             else
 564                             {
 565                                 /*
 566                                  * Defer until the play point is within
 567                                  * the subtitle
 568                                  */
 569                                 sub = NULL;
 570                             }
 571                         }
 572                         else
 573                         {
 574                             /*
 575                              * The end of the subtitle is less than the start,
 576                              * this is a sign of a PTS discontinuity.
 577                              */
 578                             if( sub->start > cur->start )
 579                             {
 580                                 /*
 581                                  * we haven't reached the start time yet, or
 582                                  * we have jumped backwards after having
 583                                  * already started this subtitle.
 584                                  */
 585                                 if( cur->start < sub->stop )
 586                                 {
 587                                     /*
 588                                      * We have jumped backwards and so should
 589                                      * continue displaying this subtitle.
 590                                      *
 591                                      * fall through to display.
 592                                      */
 593                                 }
 594                                 else
 595                                 {
 596                                     /*
 597                                      * Defer until the play point is
 598                                      * within the subtitle
 599                                      */
 600                                     sub = NULL;
 601                                 }
 602                             } else {
 603                                 /*
 604                                 * Play this subtitle as the start is
 605                                 * greater than our video point.
 606                                 *
 607                                 * fall through to display/
 608                                 */
 609                             }
 610                         }
 611                         break;
 612                     }
 613                     else
 614                     {
 615
 616                         /*
 617                          * The subtitle is older than this picture, trash it
 618                          */
 619                         sub = hb_fifo_get( subtitle->fifo_raw );
 620                         hb_buffer_close( &sub );
 621                     }
 622                 }
 623
 624                 /* If we have a subtitle for this picture, copy it */
 625                 /* FIXME: we should avoid this memcpy */
 626                 if( sub )
 627                 {
 628                     if( sub->size > 0 )
 629                     {
 630                         if( subtitle->config.dest == RENDERSUB )
 631                         {
 632                             if ( cur->sub == NULL )
 633                             {
 634                                 /*
 635                                  * Tack onto the video buffer for rendering
 636                                  */
 637                                 cur->sub         = hb_buffer_init( sub->size );
 638                                 cur->sub->x      = sub->x;
 639                                 cur->sub->y      = sub->y;
 640                                 cur->sub->width  = sub->width;
 641                                 cur->sub->height = sub->height;
 642                                 memcpy( cur->sub->data, sub->data, sub->size );
 643                             }
 644                         } else {
 645                             /*
 646                              * Pass-Through, pop it off of the raw queue,
 647                              * rewrite times and make it available to be
 648                              * reencoded.
 649                              */
 650                             uint64_t sub_duration;
 651                             sub = hb_fifo_get( subtitle->fifo_raw );
 652                             sub_duration = sub->stop - sub->start;
 653                             sub->start = cur->start;
 654                             buf_tmp = hb_fifo_see( job->fifo_raw );
 655                             int64_t duration = buf_tmp->start - cur->start;
 656                             sub->stop = sub->start + duration;
 657                             hb_fifo_push( subtitle->fifo_sync, sub );
 658                         }
 659                     } else {
 660                         /*
 661                         * EOF - consume for rendered, else pass through
 662                         */
 663                         if( subtitle->config.dest == RENDERSUB )
 664                         {
 665                             sub = hb_fifo_get( subtitle->fifo_raw );
 666                             hb_buffer_close( &sub );
 667                         } else {
 668                             sub = hb_fifo_get( subtitle->fifo_raw );
 669                             hb_fifo_push( subtitle->fifo_out, sub );
 670                         }
 671                     }
 672                 }
 673             }
 674         } // end subtitles
 675
 676         /*
 677          * Adjust the pts of the current frame so that it's contiguous
 678          * with the previous frame. The start time of the current frame
 679          * has to be the end time of the previous frame and the stop
 680          * time has to be the start of the next frame.  We don't
 681          * make any adjustments to the source timestamps other than removing
 682          * the clock offsets (which also removes pts discontinuities).
 683          * This means we automatically encode at the source's frame rate.
 684          * MP2 uses an implicit duration (frames end when the next frame
 685          * starts) but more advanced containers like MP4 use an explicit
 686          * duration. Since we're looking ahead one frame we set the
 687          * explicit stop time from the start time of the next frame.
 688          */
 689         buf_tmp = cur;
 690         pv->cur = cur = hb_fifo_get( job->fifo_raw );
 691         cur->sub = NULL;
 692         pv->next_pts = cur->start;
 693         int64_t duration = cur->start - pts_skip - buf_tmp->start;
 694         pts_skip = 0;
 695         if ( duration <= 0 )
 696         {
 697             hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
 698                     duration, buf_tmp->start, next->start );
 699         }
 700
 701         buf_tmp->start = pv->next_start;
 702         pv->next_start += duration;
 703         buf_tmp->stop = pv->next_start;
 704
 705         if ( pv->chap_mark )
 706         {
 707             // we have a pending chapter mark from a recent drop - put it on this
 708             // buffer (this may make it one frame late but we can't do any better).
 709             buf_tmp->new_chap = pv->chap_mark;
 710             pv->chap_mark = 0;
 711         }
 712
 713         /* Push the frame to the renderer */
 714         hb_fifo_push( job->fifo_sync, buf_tmp );
 715
 716         /* Update UI */
 717         UpdateState( w );
 718
 719         if( job->frame_to_stop && pv->count_frames > job->frame_to_stop )
 720         {
 721             // Drop an empty buffer into our output to ensure that things
 722             // get flushed all the way out.
 723             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 724             pv->busy &=~ 1;
 725             hb_log( "sync: reached %d frames, exiting early (%i busy)",
 726                     pv->count_frames, pv->busy );
 727             return;
 728         }
 729
 730         /* Make sure we won't get more frames then expected */
 731         if( pv->count_frames >= pv->count_frames_max * 2)
 732         {
 733             hb_log( "sync: got too many frames (%d), exiting early",
 734                     pv->count_frames );
 735
 736             // Drop an empty buffer into our output to ensure that things
 737             // get flushed all the way out.
 738             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 739             pv->busy &=~ 1;
 740             return;
 741         }
 742     }
 743 }
 744
 745 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
 746                               hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
 747 {
 748     int64_t start = sync->next_start;
 749     int64_t duration = buf->stop - buf->start;
 750
 751     sync->next_pts += duration;
 752
 753     if( audio->config.in.samplerate == audio->config.out.samplerate ||
 754         audio->config.out.codec == HB_ACODEC_AC3 ||
 755         audio->config.out.codec == HB_ACODEC_DCA )
 756     {
 757         /*
 758          * If we don't have to do sample rate conversion or this audio is
 759          * pass-thru just send the input buffer downstream after adjusting
 760          * its timestamps to make the output stream continuous.
 761          */
 762     }
 763     else
 764     {
 765         /* Not pass-thru - do sample rate conversion */
 766         int count_in, count_out;
 767         hb_buffer_t * buf_raw = buf;
 768         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
 769                             sizeof( float );
 770
 771         count_in  = buf_raw->size / channel_count;
 772         /*
 773          * When using stupid rates like 44.1 there will always be some
 774          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
 775          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
 776          * the error will build up over time and eventually the audio will
 777          * substantially lag the video. libsamplerate will keep track of the
 778          * fractional sample & give it to us when appropriate if we give it
 779          * an extra sample of space in the output buffer.
 780          */
 781         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
 782
 783         sync->data.input_frames = count_in;
 784         sync->data.output_frames = count_out;
 785         sync->data.src_ratio = (double)audio->config.out.samplerate /
 786                                (double)audio->config.in.samplerate;
 787
 788         buf = hb_buffer_init( count_out * channel_count );
 789         sync->data.data_in  = (float *) buf_raw->data;
 790         sync->data.data_out = (float *) buf->data;
 791         if( src_process( sync->state, &sync->data ) )
 792         {
 793             /* XXX If this happens, we're screwed */
 794             hb_log( "sync: audio %d src_process failed", i );
 795         }
 796         hb_buffer_close( &buf_raw );
 797
 798         buf->size = sync->data.output_frames_gen * channel_count;
 799         duration = ( sync->data.output_frames_gen * 90000 ) /
 800                    audio->config.out.samplerate;
 801     }
 802     buf->frametype = HB_FRAME_AUDIO;
 803     buf->start = start;
 804     buf->stop  = start + duration;
 805     sync->next_start = start + duration;
 806     hb_fifo_push( fifo, buf );
 807 }
 808
 809 /***********************************************************************
 810  * SyncAudio
 811  ***********************************************************************
 812  *
 813  **********************************************************************/
 814 static void SyncAudio( hb_work_object_t * w, int i )
 815 {
 816     hb_work_private_t * pv = w->private_data;
 817     hb_job_t        * job = pv->job;
 818     hb_sync_audio_t * sync = &pv->sync_audio[i];
 819     hb_audio_t      * audio = sync->audio;
 820     hb_buffer_t     * buf;
 821     hb_fifo_t       * fifo;
 822     int64_t start;
 823
 824     if( audio->config.out.codec == HB_ACODEC_AC3 ||
 825         audio->config.out.codec == HB_ACODEC_DCA )
 826     {
 827         fifo = audio->priv.fifo_out;
 828     }
 829     else
 830     {
 831         fifo = audio->priv.fifo_sync;
 832     }
 833
 834     while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
 835     {
 836         start = buf->start - pv->audio_passthru_slip;
 837         /* if the next buffer is an eof send it downstream */
 838         if ( buf->size <= 0 )
 839         {
 840             buf = hb_fifo_get( audio->priv.fifo_raw );
 841             hb_fifo_push( fifo, buf );
 842             pv->busy &=~ (1 << (i + 1) );
 843             return;
 844         }
 845         if( job->frame_to_stop && pv->count_frames >= job->frame_to_stop )
 846         {
 847             hb_fifo_push( fifo, hb_buffer_init(0) );
 848             pv->busy &=~ (1 << (i + 1) );
 849             return;
 850         }
 851         if ( (int64_t)( start - sync->next_pts ) < 0 )
 852         {
 853             // audio time went backwards.
 854             // If our output clock is more than a half frame ahead of the
 855             // input clock drop this frame to move closer to sync.
 856             // Otherwise drop frames until the input clock matches the output clock.
 857             if ( sync->first_drop || sync->next_start - start > 90*15 )
 858             {
 859                 // Discard data that's in the past.
 860                 if ( sync->first_drop == 0 )
 861                 {
 862                     sync->first_drop = sync->next_pts;
 863                 }
 864                 ++sync->drop_count;
 865                 buf = hb_fifo_get( audio->priv.fifo_raw );
 866                 hb_buffer_close( &buf );
 867                 continue;
 868             }
 869             sync->next_pts = start;
 870         }
 871         if ( sync->first_drop )
 872         {
 873             // we were dropping old data but input buf time is now current
 874             hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
 875                     "(next %lld, current %lld)", i,
 876                     (int)( sync->next_pts - sync->first_drop ) / 90,
 877                     sync->drop_count, sync->first_drop, sync->next_pts );
 878             sync->first_drop = 0;
 879             sync->drop_count = 0;
 880             sync->next_pts = start;
 881         }
 882         if ( start - sync->next_pts >= (90 * 70) )
 883         {
 884             if ( start - sync->next_pts > (90000LL * 60) )
 885             {
 886                 // there's a gap of more than a minute between the last
 887                 // frame and this. assume we got a corrupted timestamp
 888                 // and just drop the next buf.
 889                 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
 890                         "  start %lld, next %lld",
 891                         (int)((start - sync->next_pts) / (90000*60)),
 892                         i, start, sync->next_pts );
 893                 buf = hb_fifo_get( audio->priv.fifo_raw );
 894                 hb_buffer_close( &buf );
 895                 continue;
 896             }
 897             /*
 898              * there's a gap of at least 70ms between the last
 899              * frame we processed & the next. Fill it with silence.
 900              * Or in the case of DCA, skip some frames from the
 901              * other streams.
 902              */
 903             if( sync->audio->config.out.codec == HB_ACODEC_DCA )
 904             {
 905                 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
 906                         "  start %lld, next %lld",
 907                         (int)((start - sync->next_pts) / 90),
 908                         i, start, sync->next_pts );
 909                 pv->audio_passthru_slip += (start - sync->next_pts);
 910                 pv->video_pts_slip += (start - sync->next_pts);
 911                 return;
 912             }
 913             hb_log( "sync: adding %d ms of silence to audio %d"
 914                     "  start %lld, next %lld",
 915                     (int)((start - sync->next_pts) / 90),
 916                     i, start, sync->next_pts );
 917             InsertSilence( w, i, start - sync->next_pts );
 918             return;
 919         }
 920
 921         /*
 922          * When we get here we've taken care of all the dups and gaps in the
 923          * audio stream and are ready to inject the next input frame into
 924          * the output stream.
 925          */
 926         buf = hb_fifo_get( audio->priv.fifo_raw );
 927         OutputAudioFrame( job, audio, buf, sync, fifo, i );
 928     }
 929 }
 930
 931 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
 932 {
 933     hb_work_private_t * pv = w->private_data;
 934     hb_job_t        *job = pv->job;
 935     hb_sync_audio_t *sync = &pv->sync_audio[i];
 936     hb_buffer_t     *buf;
 937     hb_fifo_t       *fifo;
 938
 939     // to keep pass-thru and regular audio in sync we generate silence in
 940     // AC3 frame-sized units. If the silence duration isn't an integer multiple
 941     // of the AC3 frame duration we will truncate or round up depending on
 942     // which minimizes the timing error.
 943     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
 944                           sync->audio->config.in.samplerate;
 945     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
 946
 947     while ( --frame_count >= 0 )
 948     {
 949         if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 950         {
 951             buf        = hb_buffer_init( sync->ac3_size );
 952             buf->start = sync->next_pts;
 953             buf->stop  = buf->start + frame_dur;
 954             memcpy( buf->data, sync->ac3_buf, buf->size );
 955             fifo = sync->audio->priv.fifo_out;
 956         }
 957         else
 958         {
 959             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
 960                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
 961                                          sync->audio->config.out.mixdown) );
 962             buf->start = sync->next_pts;
 963             buf->stop  = buf->start + frame_dur;
 964             memset( buf->data, 0, buf->size );
 965             fifo = sync->audio->priv.fifo_sync;
 966         }
 967         OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
 968     }
 969 }
 970
 971 static void UpdateState( hb_work_object_t * w )
 972 {
 973     hb_work_private_t * pv = w->private_data;
 974     hb_state_t state;
 975
 976     if( !pv->count_frames )
 977     {
 978         pv->st_first = hb_get_date();
 979     }
 980     pv->count_frames++;
 981
 982     if( hb_get_date() > pv->st_dates[3] + 1000 )
 983     {
 984         memmove( &pv->st_dates[0], &pv->st_dates[1],
 985                  3 * sizeof( uint64_t ) );
 986         memmove( &pv->st_counts[0], &pv->st_counts[1],
 987                  3 * sizeof( uint64_t ) );
 988         pv->st_dates[3]  = hb_get_date();
 989         pv->st_counts[3] = pv->count_frames;
 990     }
 991
 992 #define p state.param.working
 993     state.state = HB_STATE_WORKING;
 994     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
 995     if( p.progress > 1.0 )
 996     {
 997         p.progress = 1.0;
 998     }
 999     p.rate_cur   = 1000.0 *
1000         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
1001         (float) ( pv->st_dates[3] - pv->st_dates[0] );
1002     if( hb_get_date() > pv->st_first + 4000 )
1003     {
1004         int eta;
1005         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
1006             (float) ( pv->st_dates[3] - pv->st_first );
1007         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
1008             p.rate_avg;
1009         p.hours   = eta / 3600;
1010         p.minutes = ( eta % 3600 ) / 60;
1011         p.seconds = eta % 60;
1012     }
1013     else
1014     {
1015         p.rate_avg = 0.0;
1016         p.hours    = -1;
1017         p.minutes  = -1;
1018         p.seconds  = -1;
1019     }
1020 #undef p
1021
1022     hb_set_state( pv->job->h, &state );
1023 }