libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.fr/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8 #include "hbffmpeg.h"
   9 #include <stdio.h>
  10 #include "samplerate.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22
  23     int64_t      next_start;    /* start time of next output frame */
  24     int64_t      next_pts;      /* start time of next input frame */
  25     int64_t      first_drop;    /* PTS of first 'went backwards' frame dropped */
  26     int          drop_count;    /* count of 'time went backwards' drops */
  27
  28     /* Raw */
  29     SRC_STATE  * state;
  30     SRC_DATA     data;
  31
  32     /* AC-3 */
  33     int          ac3_size;
  34     uint8_t    * ac3_buf;
  35
  36 } hb_sync_audio_t;
  37
  38 struct hb_work_private_s
  39 {
  40     hb_job_t * job;
  41     int        busy;            // bitmask with one bit for each active input
  42                                 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
  43                                 // appropriate bit is cleared when input gets
  44                                 // an eof buf. syncWork returns done when all
  45                                 // bits are clear.
  46     /* Video */
  47     int64_t pts_offset;
  48     int64_t next_start;         /* start time of next output frame */
  49     int64_t next_pts;           /* start time of next input frame */
  50     int64_t first_drop;         /* PTS of first 'went backwards' frame dropped */
  51     int drop_count;             /* count of 'time went backwards' drops */
  52     int drops;                  /* frames dropped to make a cbr video stream */
  53     int dups;                   /* frames duplicated to make a cbr video stream */
  54     int video_sequence;
  55     int count_frames;
  56     int count_frames_max;
  57     int chap_mark;              /* to propagate chapter mark across a drop */
  58     hb_buffer_t * cur; /* The next picture to process */
  59
  60     /* Audio */
  61     hb_sync_audio_t sync_audio[8];
  62     int64_t audio_passthru_slip;
  63     int64_t video_pts_slip;
  64
  65     /* Statistics */
  66     uint64_t st_counts[4];
  67     uint64_t st_dates[4];
  68     uint64_t st_first;
  69 };
  70
  71 /***********************************************************************
  72  * Local prototypes
  73  **********************************************************************/
  74 static void InitAudio( hb_work_object_t * w, int i );
  75 static void SyncVideo( hb_work_object_t * w );
  76 static void SyncAudio( hb_work_object_t * w, int i );
  77 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
  78 static void UpdateState( hb_work_object_t * w );
  79
  80 /***********************************************************************
  81  * hb_work_sync_init
  82  ***********************************************************************
  83  * Initialize the work object
  84  **********************************************************************/
  85 int syncInit( hb_work_object_t * w, hb_job_t * job )
  86 {
  87     hb_title_t       * title = job->title;
  88     hb_chapter_t     * chapter;
  89     int                i;
  90     uint64_t           duration;
  91     hb_work_private_t * pv;
  92
  93     pv = calloc( 1, sizeof( hb_work_private_t ) );
  94     w->private_data = pv;
  95
  96     pv->job            = job;
  97     pv->pts_offset     = INT64_MIN;
  98
  99     if( job->pass == 2 )
 100     {
 101         /* We already have an accurate frame count from pass 1 */
 102         hb_interjob_t * interjob = hb_interjob_get( job->h );
 103         pv->count_frames_max = interjob->frame_count;
 104     }
 105     else
 106     {
 107         /* Calculate how many video frames we are expecting */
 108         if ( job->pts_to_stop )
 109         {
 110             duration = job->pts_to_stop + 90000;
 111         }
 112         else if( job->frame_to_stop )
 113         {
 114             /* Set the duration to a rough estimate */
 115             duration = ( job->frame_to_stop / ( job->vrate / job->vrate_base ) ) * 90000;
 116         }
 117         else
 118         {
 119             duration = 0;
 120             for( i = job->chapter_start; i <= job->chapter_end; i++ )
 121             {
 122                 chapter   = hb_list_item( title->list_chapter, i - 1 );
 123                 duration += chapter->duration;
 124             }
 125             duration += 90000;
 126             /* 1 second safety so we're sure we won't miss anything */
 127         }
 128         pv->count_frames_max = duration * job->vrate / job->vrate_base / 90000;
 129     }
 130
 131     hb_log( "sync: expecting %d video frames", pv->count_frames_max );
 132     pv->busy |= 1;
 133
 134     /* Initialize libsamplerate for every audio track we have */
 135     if ( ! job->indepth_scan )
 136     {
 137         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
 138         {
 139             pv->busy |= ( 1 << (i + 1) );
 140             InitAudio( w, i );
 141         }
 142     }
 143
 144     return 0;
 145 }
 146
 147 /***********************************************************************
 148  * Close
 149  ***********************************************************************
 150  *
 151  **********************************************************************/
 152 void syncClose( hb_work_object_t * w )
 153 {
 154     hb_work_private_t * pv = w->private_data;
 155     hb_job_t          * job   = pv->job;
 156     hb_title_t        * title = job->title;
 157     hb_audio_t        * audio = NULL;
 158     int i;
 159
 160     if( pv->cur )
 161     {
 162         hb_buffer_close( &pv->cur );
 163     }
 164
 165     hb_log( "sync: got %d frames, %d expected",
 166             pv->count_frames, pv->count_frames_max );
 167
 168     /* save data for second pass */
 169     if( job->pass == 1 )
 170     {
 171         /* Preserve frame count for better accuracy in pass 2 */
 172         hb_interjob_t * interjob = hb_interjob_get( job->h );
 173         interjob->frame_count = pv->count_frames;
 174         interjob->last_job = job->sequence_id;
 175         interjob->total_time = pv->next_start;
 176     }
 177
 178     if (pv->drops || pv->dups )
 179     {
 180         hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
 181     }
 182
 183     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 184     {
 185         audio = hb_list_item( title->list_audio, i );
 186         if( audio->config.out.codec == HB_ACODEC_AC3 )
 187         {
 188             free( pv->sync_audio[i].ac3_buf );
 189         }
 190         else
 191         {
 192             src_delete( pv->sync_audio[i].state );
 193         }
 194     }
 195
 196     free( pv );
 197     w->private_data = NULL;
 198 }
 199
 200 /***********************************************************************
 201  * Work
 202  ***********************************************************************
 203  * The root routine of this work abject
 204  *
 205  * The way this works is that we are syncing the audio to the PTS of
 206  * the last video that we processed. That's why we skip the audio sync
 207  * if we haven't got a valid PTS from the video yet.
 208  *
 209  **********************************************************************/
 210 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 211               hb_buffer_t ** unused2 )
 212 {
 213     hb_work_private_t * pv = w->private_data;
 214     int i;
 215
 216     if ( pv->busy & 1 )
 217         SyncVideo( w );
 218
 219     for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 220     {
 221         if ( pv->busy & ( 1 << (i + 1) ) )
 222             SyncAudio( w, i );
 223     }
 224
 225     return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
 226 }
 227
 228 hb_work_object_t hb_sync =
 229 {
 230     WORK_SYNC,
 231     "Synchronization",
 232     syncInit,
 233     syncWork,
 234     syncClose
 235 };
 236
 237 static void InitAudio( hb_work_object_t * w, int i )
 238 {
 239     hb_work_private_t * pv = w->private_data;
 240     hb_job_t        * job   = pv->job;
 241     hb_title_t      * title = job->title;
 242     hb_sync_audio_t * sync;
 243
 244     sync        = &pv->sync_audio[i];
 245     sync->audio = hb_list_item( title->list_audio, i );
 246
 247     if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 248     {
 249         /* Have a silent AC-3 frame ready in case we have to fill a
 250            gap */
 251         AVCodec        * codec;
 252         AVCodecContext * c;
 253         short          * zeros;
 254
 255         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 256         c     = avcodec_alloc_context();
 257
 258         c->bit_rate    = sync->audio->config.in.bitrate;
 259         c->sample_rate = sync->audio->config.in.samplerate;
 260         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
 261
 262         if( hb_avcodec_open( c, codec ) < 0 )
 263         {
 264             hb_log( "sync: avcodec_open failed" );
 265             return;
 266         }
 267
 268         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 269                                  sizeof( short ) * c->channels, 1 );
 270         sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
 271                              sync->audio->config.in.samplerate / 8;
 272         sync->ac3_buf  = malloc( sync->ac3_size );
 273
 274         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 275                                   zeros ) != sync->ac3_size )
 276         {
 277             hb_log( "sync: avcodec_encode_audio failed" );
 278         }
 279
 280         free( zeros );
 281         hb_avcodec_close( c );
 282         av_free( c );
 283     }
 284     else
 285     {
 286         /* Initialize libsamplerate */
 287         int error;
 288         sync->state             = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
 289         sync->data.end_of_input = 0;
 290     }
 291 }
 292
 293 /***********************************************************************
 294  * SyncVideo
 295  ***********************************************************************
 296  *
 297  **********************************************************************/
 298 static void SyncVideo( hb_work_object_t * w )
 299 {
 300     hb_work_private_t * pv = w->private_data;
 301     hb_buffer_t * cur, * next, * sub = NULL;
 302     hb_job_t * job = pv->job;
 303     hb_subtitle_t *subtitle;
 304     int i;
 305     int64_t pts_skip;
 306
 307     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 308     {
 309         /* We haven't even got a frame yet */
 310         return;
 311     }
 312     cur = pv->cur;
 313     pts_skip = 0;
 314     if( cur->size == 0 )
 315     {
 316         /* we got an end-of-stream. Feed it downstream & signal that we're done. */
 317         hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 318
 319         /*
 320          * Push through any subtitle EOFs in case they were not synced through.
 321          */
 322         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 323         {
 324             subtitle = hb_list_item( job->list_subtitle, i );
 325             if( subtitle->config.dest == PASSTHRUSUB )
 326             {
 327                 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 328             }
 329         }
 330
 331         pv->busy &=~ 1;
 332         return;
 333     }
 334
 335     /* At this point we have a frame to process. Let's check
 336         1) if we will be able to push into the fifo ahead
 337         2) if the next frame is there already, since we need it to
 338            compute the duration of the current frame*/
 339     while( !hb_fifo_is_full( job->fifo_sync ) &&
 340            ( next = hb_fifo_see( job->fifo_raw ) ) )
 341     {
 342         hb_buffer_t * buf_tmp;
 343
 344         if( next->size == 0 )
 345         {
 346             /* we got an end-of-stream. Feed it downstream & signal that
 347              * we're done. Note that this means we drop the final frame of
 348              * video (we don't know its duration). On DVDs the final frame
 349              * is often strange and dropping it seems to be a good idea. */
 350             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 351
 352             /*
 353              * Push through any subtitle EOFs in case they were not synced through.
 354              */
 355             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 356             {
 357                 subtitle = hb_list_item( job->list_subtitle, i );
 358                 if( subtitle->config.dest == PASSTHRUSUB )
 359                 {
 360                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 361                 }
 362             }
 363             pv->busy &=~ 1;
 364             return;
 365         }
 366         if( pv->pts_offset == INT64_MIN )
 367         {
 368             /* This is our first frame */
 369             pv->pts_offset = 0;
 370             if ( cur->start != 0 )
 371             {
 372                 /*
 373                  * The first pts from a dvd should always be zero but
 374                  * can be non-zero with a transport or program stream since
 375                  * we're not guaranteed to start on an IDR frame. If we get
 376                  * a non-zero initial PTS extend its duration so it behaves
 377                  * as if it started at zero so that our audio timing will
 378                  * be in sync.
 379                  */
 380                 hb_log( "sync: first pts is %lld", cur->start );
 381                 cur->start = 0;
 382             }
 383         }
 384
 385         /*
 386          * since the first frame is always 0 and the upstream reader code
 387          * is taking care of adjusting for pts discontinuities, we just have
 388          * to deal with the next frame's start being in the past. This can
 389          * happen when the PTS is adjusted after data loss but video frame
 390          * reordering causes some frames with the old clock to appear after
 391          * the clock change. This creates frames that overlap in time which
 392          * looks to us like time going backward. The downstream muxing code
 393          * can deal with overlaps of up to a frame time but anything larger
 394          * we handle by dropping frames here.
 395          */
 396         if ( (int64_t)( next->start - pv->video_pts_slip - cur->start ) <= 0 )
 397         {
 398             if ( pv->first_drop == 0 )
 399             {
 400                 pv->first_drop = next->start;
 401             }
 402             ++pv->drop_count;
 403             if (next->start - cur->start > 0)
 404             {
 405                 pts_skip += next->start - cur->start;
 406                 pv->video_pts_slip -= next->start - cur->start;
 407             }
 408             buf_tmp = hb_fifo_get( job->fifo_raw );
 409             if ( buf_tmp->new_chap )
 410             {
 411                 // don't drop a chapter mark when we drop the buffer
 412                 pv->chap_mark = buf_tmp->new_chap;
 413             }
 414             hb_buffer_close( &buf_tmp );
 415             continue;
 416         }
 417         if ( pv->first_drop )
 418         {
 419             hb_log( "sync: video time didn't advance - dropped %d frames "
 420                     "(delta %d ms, current %lld, next %lld, dur %d)",
 421                     pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
 422                     cur->start, next->start, (int)( next->start - cur->start ) );
 423             pv->first_drop = 0;
 424             pv->drop_count = 0;
 425         }
 426
 427         /*
 428          * Track the video sequence number localy so that we can sync the audio
 429          * to it using the sequence number as well as the PTS.
 430          */
 431         pv->video_sequence = cur->sequence;
 432
 433         /*
 434          * Look for a subtitle for this frame.
 435          *
 436          * If found then it will be tagged onto a video buffer of the correct time and
 437          * sent in to the render pipeline. This only needs to be done for VOBSUBs which
 438          * get rendered, other types of subtitles can just sit in their raw_queue until
 439          * delt with at muxing.
 440          */
 441         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 442         {
 443             subtitle = hb_list_item( job->list_subtitle, i );
 444
 445             /*
 446              * Rewrite timestamps on subtitles that need it (on raw queue).
 447              */
 448             if( subtitle->source == CC608SUB ||
 449                 subtitle->source == CC708SUB )
 450             {
 451                 /*
 452                  * Rewrite timestamps on subtitles that came from Closed Captions
 453                  * since they are using the MPEG2 timestamps.
 454                  */
 455                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 456                 {
 457                     /*
 458                      * Rewrite the timestamps as and when the video
 459                      * (cur->start) reaches the same timestamp as a
 460                      * closed caption (sub->start).
 461                      *
 462                      * What about discontinuity boundaries - not delt
 463                      * with here - Van?
 464                      *
 465                      * Bypass the sync fifo altogether.
 466                      */
 467                     if( sub->size <= 0 )
 468                     {
 469                         sub = hb_fifo_get( subtitle->fifo_raw );
 470                         hb_fifo_push( subtitle->fifo_out, sub );
 471                         sub = NULL;
 472                         break;
 473                     } else {
 474                         /*
 475                          * Sync the subtitles to the incoming video, and use
 476                          * the matching converted video timestamp.
 477                          *
 478                          * Note that it doesn't appear that we need to convert
 479                          * timestamps, I guess that they were already correct,
 480                          * so just push them through for rendering.
 481                          *
 482                          */
 483                         if( sub->start < cur->start )
 484                         {
 485                             uint64_t duration;
 486                             duration = sub->stop - sub->start;
 487                             sub = hb_fifo_get( subtitle->fifo_raw );
 488                             hb_fifo_push( subtitle->fifo_out, sub );
 489                         } else {
 490                             sub = NULL;
 491                             break;
 492                         }
 493                     }
 494                 }
 495             }
 496
 497             if( subtitle->source == VOBSUB )
 498             {
 499                 hb_buffer_t * sub2;
 500                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 501                 {
 502                     if( sub->size == 0 )
 503                     {
 504                         /*
 505                          * EOF, pass it through immediately.
 506                          */
 507                         break;
 508                     }
 509
 510                     /* If two subtitles overlap, make the first one stop
 511                        when the second one starts */
 512                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 513                     if( sub2 && sub->stop > sub2->start )
 514                     {
 515                         sub->stop = sub2->start;
 516                     }
 517
 518                     // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 519                     //       sub, cur->sequence, sub->sequence);
 520
 521                     if( sub->sequence > cur->sequence )
 522                     {
 523                         /*
 524                          * The video is behind where we are, so wait until
 525                          * it catches up to the same reader point on the
 526                          * DVD. Then our PTS should be in the same region
 527                          * as the video.
 528                          */
 529                         sub = NULL;
 530                         break;
 531                     }
 532
 533                     if( sub->stop > cur->start ) {
 534                         /*
 535                          * The stop time is in the future, so fall through
 536                          * and we'll deal with it in the next block of
 537                          * code.
 538                          */
 539
 540                         /*
 541                          * There is a valid subtitle, is it time to display it?
 542                          */
 543                         if( sub->stop > sub->start)
 544                         {
 545                             /*
 546                              * Normal subtitle which ends after it starts,
 547                              * check to see that the current video is between
 548                              * the start and end.
 549                              */
 550                             if( cur->start > sub->start &&
 551                                 cur->start < sub->stop )
 552                             {
 553                                 /*
 554                                 * We should be playing this, so leave the
 555                                 * subtitle in place.
 556                                 *
 557                                 * fall through to display
 558                                 */
 559                                 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
 560                                 {
 561                                     /*
 562                                      * Subtitle is on for less than three
 563                                      * seconds, extend the time that it is
 564                                      * displayed to make it easier to read.
 565                                      * Make it 3 seconds or until the next
 566                                      * subtitle is displayed.
 567                                      *
 568                                      * This is in response to Indochine which
 569                                      * only displays subs for 1 second -
 570                                      * too fast to read.
 571                                      */
 572                                     sub->stop = sub->start + ( 2 * 90000 );
 573
 574                                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 575
 576                                     if( sub2 && sub->stop > sub2->start )
 577                                     {
 578                                         sub->stop = sub2->start;
 579                                     }
 580                                 }
 581                             }
 582                             else
 583                             {
 584                                 /*
 585                                  * Defer until the play point is within
 586                                  * the subtitle
 587                                  */
 588                                 sub = NULL;
 589                             }
 590                         }
 591                         else
 592                         {
 593                             /*
 594                              * The end of the subtitle is less than the start,
 595                              * this is a sign of a PTS discontinuity.
 596                              */
 597                             if( sub->start > cur->start )
 598                             {
 599                                 /*
 600                                  * we haven't reached the start time yet, or
 601                                  * we have jumped backwards after having
 602                                  * already started this subtitle.
 603                                  */
 604                                 if( cur->start < sub->stop )
 605                                 {
 606                                     /*
 607                                      * We have jumped backwards and so should
 608                                      * continue displaying this subtitle.
 609                                      *
 610                                      * fall through to display.
 611                                      */
 612                                 }
 613                                 else
 614                                 {
 615                                     /*
 616                                      * Defer until the play point is
 617                                      * within the subtitle
 618                                      */
 619                                     sub = NULL;
 620                                 }
 621                             } else {
 622                                 /*
 623                                 * Play this subtitle as the start is
 624                                 * greater than our video point.
 625                                 *
 626                                 * fall through to display/
 627                                 */
 628                             }
 629                         }
 630                         break;
 631                     }
 632                     else
 633                     {
 634
 635                         /*
 636                          * The subtitle is older than this picture, trash it
 637                          */
 638                         sub = hb_fifo_get( subtitle->fifo_raw );
 639                         hb_buffer_close( &sub );
 640                     }
 641                 }
 642
 643                 /* If we have a subtitle for this picture, copy it */
 644                 /* FIXME: we should avoid this memcpy */
 645                 if( sub )
 646                 {
 647                     if( sub->size > 0 )
 648                     {
 649                         if( subtitle->config.dest == RENDERSUB )
 650                         {
 651                             if ( cur->sub == NULL )
 652                             {
 653                                 /*
 654                                  * Tack onto the video buffer for rendering
 655                                  */
 656                                 cur->sub         = hb_buffer_init( sub->size );
 657                                 cur->sub->x      = sub->x;
 658                                 cur->sub->y      = sub->y;
 659                                 cur->sub->width  = sub->width;
 660                                 cur->sub->height = sub->height;
 661                                 memcpy( cur->sub->data, sub->data, sub->size );
 662                             }
 663                         } else {
 664                             /*
 665                              * Pass-Through, pop it off of the raw queue,
 666                              * rewrite times and make it available to be
 667                              * reencoded.
 668                              */
 669                             uint64_t sub_duration;
 670                             sub = hb_fifo_get( subtitle->fifo_raw );
 671                             sub_duration = sub->stop - sub->start;
 672                             sub->start = cur->start;
 673                             buf_tmp = hb_fifo_see( job->fifo_raw );
 674                             int64_t duration = buf_tmp->start - cur->start;
 675                             sub->stop = sub->start + duration;
 676                             hb_fifo_push( subtitle->fifo_sync, sub );
 677                         }
 678                     } else {
 679                         /*
 680                         * EOF - consume for rendered, else pass through
 681                         */
 682                         if( subtitle->config.dest == RENDERSUB )
 683                         {
 684                             sub = hb_fifo_get( subtitle->fifo_raw );
 685                             hb_buffer_close( &sub );
 686                         } else {
 687                             sub = hb_fifo_get( subtitle->fifo_raw );
 688                             hb_fifo_push( subtitle->fifo_out, sub );
 689                         }
 690                     }
 691                 }
 692             }
 693         } // end subtitles
 694
 695         /*
 696          * Adjust the pts of the current frame so that it's contiguous
 697          * with the previous frame. The start time of the current frame
 698          * has to be the end time of the previous frame and the stop
 699          * time has to be the start of the next frame.  We don't
 700          * make any adjustments to the source timestamps other than removing
 701          * the clock offsets (which also removes pts discontinuities).
 702          * This means we automatically encode at the source's frame rate.
 703          * MP2 uses an implicit duration (frames end when the next frame
 704          * starts) but more advanced containers like MP4 use an explicit
 705          * duration. Since we're looking ahead one frame we set the
 706          * explicit stop time from the start time of the next frame.
 707          */
 708         buf_tmp = cur;
 709         pv->cur = cur = hb_fifo_get( job->fifo_raw );
 710         cur->sub = NULL;
 711         pv->next_pts = cur->start;
 712         int64_t duration = cur->start - pts_skip - buf_tmp->start;
 713         pts_skip = 0;
 714         if ( duration <= 0 )
 715         {
 716             hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
 717                     duration, buf_tmp->start, next->start );
 718         }
 719
 720         buf_tmp->start = pv->next_start;
 721         pv->next_start += duration;
 722         buf_tmp->stop = pv->next_start;
 723
 724         if ( pv->chap_mark )
 725         {
 726             // we have a pending chapter mark from a recent drop - put it on this
 727             // buffer (this may make it one frame late but we can't do any better).
 728             buf_tmp->new_chap = pv->chap_mark;
 729             pv->chap_mark = 0;
 730         }
 731
 732         /* Push the frame to the renderer */
 733         hb_fifo_push( job->fifo_sync, buf_tmp );
 734
 735         /* Update UI */
 736         UpdateState( w );
 737
 738         if( job->frame_to_stop && pv->count_frames > job->frame_to_stop )
 739         {
 740             // Drop an empty buffer into our output to ensure that things
 741             // get flushed all the way out.
 742             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 743             pv->busy &=~ 1;
 744             hb_log( "sync: reached %d frames, exiting early (%i busy)",
 745                     pv->count_frames, pv->busy );
 746             return;
 747         }
 748
 749         /* Make sure we won't get more frames then expected */
 750         if( pv->count_frames >= pv->count_frames_max * 2)
 751         {
 752             hb_log( "sync: got too many frames (%d), exiting early",
 753                     pv->count_frames );
 754
 755             // Drop an empty buffer into our output to ensure that things
 756             // get flushed all the way out.
 757             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 758             pv->busy &=~ 1;
 759             return;
 760         }
 761     }
 762 }
 763
 764 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
 765                               hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
 766 {
 767     int64_t start = sync->next_start;
 768     int64_t duration = buf->stop - buf->start;
 769
 770     sync->next_pts += duration;
 771
 772     if( audio->config.in.samplerate == audio->config.out.samplerate ||
 773         audio->config.out.codec == HB_ACODEC_AC3 ||
 774         audio->config.out.codec == HB_ACODEC_DCA )
 775     {
 776         /*
 777          * If we don't have to do sample rate conversion or this audio is
 778          * pass-thru just send the input buffer downstream after adjusting
 779          * its timestamps to make the output stream continuous.
 780          */
 781     }
 782     else
 783     {
 784         /* Not pass-thru - do sample rate conversion */
 785         int count_in, count_out;
 786         hb_buffer_t * buf_raw = buf;
 787         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
 788                             sizeof( float );
 789
 790         count_in  = buf_raw->size / channel_count;
 791         /*
 792          * When using stupid rates like 44.1 there will always be some
 793          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
 794          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
 795          * the error will build up over time and eventually the audio will
 796          * substantially lag the video. libsamplerate will keep track of the
 797          * fractional sample & give it to us when appropriate if we give it
 798          * an extra sample of space in the output buffer.
 799          */
 800         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
 801
 802         sync->data.input_frames = count_in;
 803         sync->data.output_frames = count_out;
 804         sync->data.src_ratio = (double)audio->config.out.samplerate /
 805                                (double)audio->config.in.samplerate;
 806
 807         buf = hb_buffer_init( count_out * channel_count );
 808         sync->data.data_in  = (float *) buf_raw->data;
 809         sync->data.data_out = (float *) buf->data;
 810         if( src_process( sync->state, &sync->data ) )
 811         {
 812             /* XXX If this happens, we're screwed */
 813             hb_log( "sync: audio %d src_process failed", i );
 814         }
 815         hb_buffer_close( &buf_raw );
 816
 817         buf->size = sync->data.output_frames_gen * channel_count;
 818         duration = ( sync->data.output_frames_gen * 90000 ) /
 819                    audio->config.out.samplerate;
 820     }
 821     buf->frametype = HB_FRAME_AUDIO;
 822     buf->start = start;
 823     buf->stop  = start + duration;
 824     sync->next_start = start + duration;
 825     hb_fifo_push( fifo, buf );
 826 }
 827
 828 /***********************************************************************
 829  * SyncAudio
 830  ***********************************************************************
 831  *
 832  **********************************************************************/
 833 static void SyncAudio( hb_work_object_t * w, int i )
 834 {
 835     hb_work_private_t * pv = w->private_data;
 836     hb_job_t        * job = pv->job;
 837     hb_sync_audio_t * sync = &pv->sync_audio[i];
 838     hb_audio_t      * audio = sync->audio;
 839     hb_buffer_t     * buf;
 840     hb_fifo_t       * fifo;
 841     int64_t start;
 842
 843     if( audio->config.out.codec == HB_ACODEC_AC3 ||
 844         audio->config.out.codec == HB_ACODEC_DCA )
 845     {
 846         fifo = audio->priv.fifo_out;
 847     }
 848     else
 849     {
 850         fifo = audio->priv.fifo_sync;
 851     }
 852
 853     while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
 854     {
 855         start = buf->start - pv->audio_passthru_slip;
 856         /* if the next buffer is an eof send it downstream */
 857         if ( buf->size <= 0 )
 858         {
 859             buf = hb_fifo_get( audio->priv.fifo_raw );
 860             hb_fifo_push( fifo, buf );
 861             pv->busy &=~ (1 << (i + 1) );
 862             return;
 863         }
 864         if( job->frame_to_stop && pv->count_frames >= job->frame_to_stop )
 865         {
 866             hb_fifo_push( fifo, hb_buffer_init(0) );
 867             pv->busy &=~ (1 << (i + 1) );
 868             return;
 869         }
 870         if ( (int64_t)( start - sync->next_pts ) < 0 )
 871         {
 872             // audio time went backwards.
 873             // If our output clock is more than a half frame ahead of the
 874             // input clock drop this frame to move closer to sync.
 875             // Otherwise drop frames until the input clock matches the output clock.
 876             if ( sync->first_drop || sync->next_start - start > 90*15 )
 877             {
 878                 // Discard data that's in the past.
 879                 if ( sync->first_drop == 0 )
 880                 {
 881                     sync->first_drop = sync->next_pts;
 882                 }
 883                 ++sync->drop_count;
 884                 buf = hb_fifo_get( audio->priv.fifo_raw );
 885                 hb_buffer_close( &buf );
 886                 continue;
 887             }
 888             sync->next_pts = start;
 889         }
 890         if ( sync->first_drop )
 891         {
 892             // we were dropping old data but input buf time is now current
 893             hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
 894                     "(next %lld, current %lld)", i,
 895                     (int)( sync->next_pts - sync->first_drop ) / 90,
 896                     sync->drop_count, sync->first_drop, sync->next_pts );
 897             sync->first_drop = 0;
 898             sync->drop_count = 0;
 899             sync->next_pts = start;
 900         }
 901         if ( start - sync->next_pts >= (90 * 70) )
 902         {
 903             if ( start - sync->next_pts > (90000LL * 60) )
 904             {
 905                 // there's a gap of more than a minute between the last
 906                 // frame and this. assume we got a corrupted timestamp
 907                 // and just drop the next buf.
 908                 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
 909                         "  start %lld, next %lld",
 910                         (int)((start - sync->next_pts) / (90000*60)),
 911                         i, start, sync->next_pts );
 912                 buf = hb_fifo_get( audio->priv.fifo_raw );
 913                 hb_buffer_close( &buf );
 914                 continue;
 915             }
 916             /*
 917              * there's a gap of at least 70ms between the last
 918              * frame we processed & the next. Fill it with silence.
 919              * Or in the case of DCA, skip some frames from the
 920              * other streams.
 921              */
 922             if( sync->audio->config.out.codec == HB_ACODEC_DCA )
 923             {
 924                 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
 925                         "  start %lld, next %lld",
 926                         (int)((start - sync->next_pts) / 90),
 927                         i, start, sync->next_pts );
 928                 pv->audio_passthru_slip += (start - sync->next_pts);
 929                 pv->video_pts_slip += (start - sync->next_pts);
 930                 return;
 931             }
 932             hb_log( "sync: adding %d ms of silence to audio %d"
 933                     "  start %lld, next %lld",
 934                     (int)((start - sync->next_pts) / 90),
 935                     i, start, sync->next_pts );
 936             InsertSilence( w, i, start - sync->next_pts );
 937             return;
 938         }
 939
 940         /*
 941          * When we get here we've taken care of all the dups and gaps in the
 942          * audio stream and are ready to inject the next input frame into
 943          * the output stream.
 944          */
 945         buf = hb_fifo_get( audio->priv.fifo_raw );
 946         OutputAudioFrame( job, audio, buf, sync, fifo, i );
 947     }
 948 }
 949
 950 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
 951 {
 952     hb_work_private_t * pv = w->private_data;
 953     hb_job_t        *job = pv->job;
 954     hb_sync_audio_t *sync = &pv->sync_audio[i];
 955     hb_buffer_t     *buf;
 956     hb_fifo_t       *fifo;
 957
 958     // to keep pass-thru and regular audio in sync we generate silence in
 959     // AC3 frame-sized units. If the silence duration isn't an integer multiple
 960     // of the AC3 frame duration we will truncate or round up depending on
 961     // which minimizes the timing error.
 962     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
 963                           sync->audio->config.in.samplerate;
 964     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
 965
 966     while ( --frame_count >= 0 )
 967     {
 968         if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 969         {
 970             buf        = hb_buffer_init( sync->ac3_size );
 971             buf->start = sync->next_pts;
 972             buf->stop  = buf->start + frame_dur;
 973             memcpy( buf->data, sync->ac3_buf, buf->size );
 974             fifo = sync->audio->priv.fifo_out;
 975         }
 976         else
 977         {
 978             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
 979                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
 980                                          sync->audio->config.out.mixdown) );
 981             buf->start = sync->next_pts;
 982             buf->stop  = buf->start + frame_dur;
 983             memset( buf->data, 0, buf->size );
 984             fifo = sync->audio->priv.fifo_sync;
 985         }
 986         OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
 987     }
 988 }
 989
 990 static void UpdateState( hb_work_object_t * w )
 991 {
 992     hb_work_private_t * pv = w->private_data;
 993     hb_state_t state;
 994
 995     if( !pv->count_frames )
 996     {
 997         pv->st_first = hb_get_date();
 998     }
 999     pv->count_frames++;
1000
1001     if( hb_get_date() > pv->st_dates[3] + 1000 )
1002     {
1003         memmove( &pv->st_dates[0], &pv->st_dates[1],
1004                  3 * sizeof( uint64_t ) );
1005         memmove( &pv->st_counts[0], &pv->st_counts[1],
1006                  3 * sizeof( uint64_t ) );
1007         pv->st_dates[3]  = hb_get_date();
1008         pv->st_counts[3] = pv->count_frames;
1009     }
1010
1011 #define p state.param.working
1012     state.state = HB_STATE_WORKING;
1013     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
1014     if( p.progress > 1.0 )
1015     {
1016         p.progress = 1.0;
1017     }
1018     p.rate_cur   = 1000.0 *
1019         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
1020         (float) ( pv->st_dates[3] - pv->st_dates[0] );
1021     if( hb_get_date() > pv->st_first + 4000 )
1022     {
1023         int eta;
1024         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
1025             (float) ( pv->st_dates[3] - pv->st_first );
1026         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
1027             p.rate_avg;
1028         p.hours   = eta / 3600;
1029         p.minutes = ( eta % 3600 ) / 60;
1030         p.seconds = eta % 60;
1031     }
1032     else
1033     {
1034         p.rate_avg = 0.0;
1035         p.hours    = -1;
1036         p.minutes  = -1;
1037         p.seconds  = -1;
1038     }
1039 #undef p
1040
1041     hb_set_state( pv->job->h, &state );
1042 }