libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.fr/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8 #include "hbffmpeg.h"
   9 #include <stdio.h>
  10 #include "samplerate.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_audio_t * audio;
  22
  23     int64_t      next_start;    /* start time of next output frame */
  24     int64_t      next_pts;      /* start time of next input frame */
  25     int64_t      first_drop;    /* PTS of first 'went backwards' frame dropped */
  26     int          drop_count;    /* count of 'time went backwards' drops */
  27
  28     /* Raw */
  29     SRC_STATE  * state;
  30     SRC_DATA     data;
  31
  32     /* AC-3 */
  33     int          ac3_size;
  34     uint8_t    * ac3_buf;
  35
  36 } hb_sync_audio_t;
  37
  38 struct hb_work_private_s
  39 {
  40     hb_job_t * job;
  41     int        busy;            // bitmask with one bit for each active input
  42                                 // (bit 0 = video; 1 = audio 0, 2 = audio 1, ...
  43                                 // appropriate bit is cleared when input gets
  44                                 // an eof buf. syncWork returns done when all
  45                                 // bits are clear.
  46     /* Video */
  47     int64_t pts_offset;
  48     int64_t next_start;         /* start time of next output frame */
  49     int64_t next_pts;           /* start time of next input frame */
  50     int64_t first_drop;         /* PTS of first 'went backwards' frame dropped */
  51     int drop_count;             /* count of 'time went backwards' drops */
  52     int drops;                  /* frames dropped to make a cbr video stream */
  53     int dups;                   /* frames duplicated to make a cbr video stream */
  54     int video_sequence;
  55     int count_frames;
  56     int count_frames_max;
  57     int chap_mark;              /* to propagate chapter mark across a drop */
  58     hb_buffer_t * cur; /* The next picture to process */
  59
  60     /* Audio */
  61     hb_sync_audio_t sync_audio[8];
  62     int64_t audio_passthru_slip;
  63     int64_t video_pts_slip;
  64
  65     /* Statistics */
  66     uint64_t st_counts[4];
  67     uint64_t st_dates[4];
  68     uint64_t st_first;
  69 };
  70
  71 /***********************************************************************
  72  * Local prototypes
  73  **********************************************************************/
  74 static void InitAudio( hb_work_object_t * w, int i );
  75 static void SyncVideo( hb_work_object_t * w );
  76 static void SyncAudio( hb_work_object_t * w, int i );
  77 static void InsertSilence( hb_work_object_t * w, int i, int64_t d );
  78 static void UpdateState( hb_work_object_t * w );
  79
  80 /***********************************************************************
  81  * hb_work_sync_init
  82  ***********************************************************************
  83  * Initialize the work object
  84  **********************************************************************/
  85 int syncInit( hb_work_object_t * w, hb_job_t * job )
  86 {
  87     hb_title_t       * title = job->title;
  88     hb_chapter_t     * chapter;
  89     int                i;
  90     uint64_t           duration;
  91     hb_work_private_t * pv;
  92
  93     pv = calloc( 1, sizeof( hb_work_private_t ) );
  94     w->private_data = pv;
  95
  96     pv->job            = job;
  97     pv->pts_offset     = INT64_MIN;
  98
  99     if( job->pass == 2 )
 100     {
 101         /* We already have an accurate frame count from pass 1 */
 102         hb_interjob_t * interjob = hb_interjob_get( job->h );
 103         pv->count_frames_max = interjob->frame_count;
 104     }
 105     else
 106     {
 107         /* Calculate how many video frames we are expecting */
 108         if ( job->pts_to_stop )
 109         {
 110             duration = job->pts_to_stop + 90000;
 111         }
 112         else if( job->frame_to_stop )
 113         {
 114             /* Set the duration to a rough estimate */
 115             duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
 116         }
 117         else
 118         {
 119             duration = 0;
 120             for( i = job->chapter_start; i <= job->chapter_end; i++ )
 121             {
 122                 chapter   = hb_list_item( title->list_chapter, i - 1 );
 123                 duration += chapter->duration;
 124             }
 125             duration += 90000;
 126             /* 1 second safety so we're sure we won't miss anything */
 127         }
 128         pv->count_frames_max = duration * title->rate / title->rate_base / 90000;
 129     }
 130
 131     hb_log( "sync: expecting %d video frames", pv->count_frames_max );
 132     pv->busy |= 1;
 133
 134     /* Initialize libsamplerate for every audio track we have */
 135     if ( ! job->indepth_scan )
 136     {
 137         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
 138         {
 139             pv->busy |= ( 1 << (i + 1) );
 140             InitAudio( w, i );
 141         }
 142     }
 143
 144     return 0;
 145 }
 146
 147 /***********************************************************************
 148  * Close
 149  ***********************************************************************
 150  *
 151  **********************************************************************/
 152 void syncClose( hb_work_object_t * w )
 153 {
 154     hb_work_private_t * pv = w->private_data;
 155     hb_job_t          * job   = pv->job;
 156     hb_title_t        * title = job->title;
 157     hb_audio_t        * audio = NULL;
 158     int i;
 159
 160     if( pv->cur )
 161     {
 162         hb_buffer_close( &pv->cur );
 163     }
 164
 165     hb_log( "sync: got %d frames, %d expected",
 166             pv->count_frames, pv->count_frames_max );
 167
 168     /* save data for second pass */
 169     if( job->pass == 1 )
 170     {
 171         /* Preserve frame count for better accuracy in pass 2 */
 172         hb_interjob_t * interjob = hb_interjob_get( job->h );
 173         interjob->frame_count = pv->count_frames;
 174         interjob->last_job = job->sequence_id;
 175         interjob->total_time = pv->next_start;
 176     }
 177
 178     if (pv->drops || pv->dups )
 179     {
 180         hb_log( "sync: %d frames dropped, %d duplicated", pv->drops, pv->dups );
 181     }
 182
 183     for( i = 0; i < hb_list_count( title->list_audio ); i++ )
 184     {
 185         audio = hb_list_item( title->list_audio, i );
 186         if( audio->config.out.codec == HB_ACODEC_AC3 )
 187         {
 188             free( pv->sync_audio[i].ac3_buf );
 189         }
 190         else
 191         {
 192             src_delete( pv->sync_audio[i].state );
 193         }
 194     }
 195
 196     free( pv );
 197     w->private_data = NULL;
 198 }
 199
 200 /***********************************************************************
 201  * Work
 202  ***********************************************************************
 203  * The root routine of this work abject
 204  *
 205  * The way this works is that we are syncing the audio to the PTS of
 206  * the last video that we processed. That's why we skip the audio sync
 207  * if we haven't got a valid PTS from the video yet.
 208  *
 209  **********************************************************************/
 210 int syncWork( hb_work_object_t * w, hb_buffer_t ** unused1,
 211               hb_buffer_t ** unused2 )
 212 {
 213     hb_work_private_t * pv = w->private_data;
 214     int i;
 215
 216     if ( pv->busy & 1 )
 217         SyncVideo( w );
 218
 219     for( i = 0; i < hb_list_count( pv->job->title->list_audio ); i++ )
 220     {
 221         if ( pv->busy & ( 1 << (i + 1) ) )
 222             SyncAudio( w, i );
 223     }
 224
 225     return ( pv->busy? HB_WORK_OK : HB_WORK_DONE );
 226 }
 227
 228 hb_work_object_t hb_sync =
 229 {
 230     WORK_SYNC,
 231     "Synchronization",
 232     syncInit,
 233     syncWork,
 234     syncClose
 235 };
 236
 237 static void InitAudio( hb_work_object_t * w, int i )
 238 {
 239     hb_work_private_t * pv = w->private_data;
 240     hb_job_t        * job   = pv->job;
 241     hb_title_t      * title = job->title;
 242     hb_sync_audio_t * sync;
 243
 244     sync        = &pv->sync_audio[i];
 245     sync->audio = hb_list_item( title->list_audio, i );
 246
 247     if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 248     {
 249         /* Have a silent AC-3 frame ready in case we have to fill a
 250            gap */
 251         AVCodec        * codec;
 252         AVCodecContext * c;
 253         short          * zeros;
 254
 255         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 256         c     = avcodec_alloc_context();
 257
 258         c->bit_rate    = sync->audio->config.in.bitrate;
 259         c->sample_rate = sync->audio->config.in.samplerate;
 260         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( sync->audio->config.in.channel_layout );
 261
 262         if( hb_avcodec_open( c, codec ) < 0 )
 263         {
 264             hb_log( "sync: avcodec_open failed" );
 265             return;
 266         }
 267
 268         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 269                                  sizeof( short ) * c->channels, 1 );
 270         sync->ac3_size = sync->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
 271                              sync->audio->config.in.samplerate / 8;
 272         sync->ac3_buf  = malloc( sync->ac3_size );
 273
 274         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 275                                   zeros ) != sync->ac3_size )
 276         {
 277             hb_log( "sync: avcodec_encode_audio failed" );
 278         }
 279
 280         free( zeros );
 281         hb_avcodec_close( c );
 282         av_free( c );
 283     }
 284     else
 285     {
 286         /* Initialize libsamplerate */
 287         int error;
 288         sync->state             = src_new( SRC_SINC_MEDIUM_QUALITY, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync->audio->config.out.mixdown), &error );
 289         sync->data.end_of_input = 0;
 290     }
 291 }
 292
 293 /***********************************************************************
 294  * SyncVideo
 295  ***********************************************************************
 296  *
 297  **********************************************************************/
 298 static void SyncVideo( hb_work_object_t * w )
 299 {
 300     hb_work_private_t * pv = w->private_data;
 301     hb_buffer_t * cur, * next, * sub = NULL;
 302     hb_job_t * job = pv->job;
 303     hb_subtitle_t *subtitle;
 304     int i;
 305     int64_t pts_skip;
 306
 307     if( !pv->cur && !( pv->cur = hb_fifo_get( job->fifo_raw ) ) )
 308     {
 309         /* We haven't even got a frame yet */
 310         return;
 311     }
 312     cur = pv->cur;
 313     pts_skip = 0;
 314     if( cur->size == 0 )
 315     {
 316         /* we got an end-of-stream. Feed it downstream & signal that we're done. */
 317         hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 318
 319         /*
 320          * Push through any subtitle EOFs in case they were not synced through.
 321          */
 322         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 323         {
 324             subtitle = hb_list_item( job->list_subtitle, i );
 325             if( subtitle->config.dest == PASSTHRUSUB )
 326             {
 327                 hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 328             }
 329         }
 330
 331         pv->busy &=~ 1;
 332         return;
 333     }
 334
 335     /* At this point we have a frame to process. Let's check
 336         1) if we will be able to push into the fifo ahead
 337         2) if the next frame is there already, since we need it to
 338            compute the duration of the current frame*/
 339     while( !hb_fifo_is_full( job->fifo_sync ) &&
 340            ( next = hb_fifo_see( job->fifo_raw ) ) )
 341     {
 342         hb_buffer_t * buf_tmp;
 343
 344         if( next->size == 0 )
 345         {
 346             /* we got an end-of-stream. Feed it downstream & signal that
 347              * we're done. Note that this means we drop the final frame of
 348              * video (we don't know its duration). On DVDs the final frame
 349              * is often strange and dropping it seems to be a good idea. */
 350             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 351
 352             /*
 353              * Push through any subtitle EOFs in case they were not synced through.
 354              */
 355             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 356             {
 357                 subtitle = hb_list_item( job->list_subtitle, i );
 358                 if( subtitle->config.dest == PASSTHRUSUB )
 359                 {
 360                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 361                 }
 362             }
 363             pv->busy &=~ 1;
 364             return;
 365         }
 366         if( pv->pts_offset == INT64_MIN )
 367         {
 368             /* This is our first frame */
 369             pv->pts_offset = 0;
 370             if ( cur->start != 0 )
 371             {
 372                 /*
 373                  * The first pts from a dvd should always be zero but
 374                  * can be non-zero with a transport or program stream since
 375                  * we're not guaranteed to start on an IDR frame. If we get
 376                  * a non-zero initial PTS extend its duration so it behaves
 377                  * as if it started at zero so that our audio timing will
 378                  * be in sync.
 379                  */
 380                 hb_log( "sync: first pts is %lld", cur->start );
 381                 cur->start = 0;
 382             }
 383         }
 384
 385         /*
 386          * since the first frame is always 0 and the upstream reader code
 387          * is taking care of adjusting for pts discontinuities, we just have
 388          * to deal with the next frame's start being in the past. This can
 389          * happen when the PTS is adjusted after data loss but video frame
 390          * reordering causes some frames with the old clock to appear after
 391          * the clock change. This creates frames that overlap in time which
 392          * looks to us like time going backward. The downstream muxing code
 393          * can deal with overlaps of up to a frame time but anything larger
 394          * we handle by dropping frames here.
 395          */
 396         if ( (int64_t)( next->start - pv->video_pts_slip - cur->start ) <= 0 )
 397         {
 398             if ( pv->first_drop == 0 )
 399             {
 400                 pv->first_drop = next->start;
 401             }
 402             ++pv->drop_count;
 403             if (next->start - cur->start > 0)
 404             {
 405                 pts_skip += next->start - cur->start;
 406                 pv->video_pts_slip -= next->start - cur->start;
 407             }
 408             buf_tmp = hb_fifo_get( job->fifo_raw );
 409             if ( buf_tmp->new_chap )
 410             {
 411                 // don't drop a chapter mark when we drop the buffer
 412                 pv->chap_mark = buf_tmp->new_chap;
 413             }
 414             hb_buffer_close( &buf_tmp );
 415             continue;
 416         }
 417         if ( pv->first_drop )
 418         {
 419             hb_log( "sync: video time didn't advance - dropped %d frames "
 420                     "(delta %d ms, current %lld, next %lld, dur %d)",
 421                     pv->drop_count, (int)( cur->start - pv->first_drop ) / 90,
 422                     cur->start, next->start, (int)( next->start - cur->start ) );
 423             pv->first_drop = 0;
 424             pv->drop_count = 0;
 425         }
 426
 427         /*
 428          * Track the video sequence number localy so that we can sync the audio
 429          * to it using the sequence number as well as the PTS.
 430          */
 431         pv->video_sequence = cur->sequence;
 432
 433         /*
 434          * Look for a subtitle for this frame.
 435          *
 436          * If found then it will be tagged onto a video buffer of the correct time and
 437          * sent in to the render pipeline. This only needs to be done for VOBSUBs which
 438          * get rendered, other types of subtitles can just sit in their raw_queue until
 439          * delt with at muxing.
 440          */
 441         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 442         {
 443             subtitle = hb_list_item( job->list_subtitle, i );
 444
 445             /*
 446              * Rewrite timestamps on subtitles that need it (on raw queue).
 447              */
 448             if( subtitle->source == CC608SUB ||
 449                 subtitle->source == CC708SUB ||
 450                 subtitle->source == SRTSUB )
 451             {
 452                 /*
 453                  * Rewrite timestamps on subtitles that came from Closed Captions
 454                  * since they are using the MPEG2 timestamps.
 455                  */
 456                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 457                 {
 458                     /*
 459                      * Rewrite the timestamps as and when the video
 460                      * (cur->start) reaches the same timestamp as a
 461                      * closed caption (sub->start).
 462                      *
 463                      * What about discontinuity boundaries - not delt
 464                      * with here - Van?
 465                      *
 466                      * Bypass the sync fifo altogether.
 467                      */
 468                     if( sub->size <= 0 )
 469                     {
 470                         sub = hb_fifo_get( subtitle->fifo_raw );
 471                         hb_fifo_push( subtitle->fifo_out, sub );
 472                         sub = NULL;
 473                         break;
 474                     } else {
 475                         /*
 476                          * Sync the subtitles to the incoming video, and use
 477                          * the matching converted video timestamp.
 478                          *
 479                          * Note that it doesn't appear that we need to convert
 480                          * timestamps, I guess that they were already correct,
 481                          * so just push them through for rendering.
 482                          *
 483                          */
 484                         if( sub->start < cur->start )
 485                         {
 486                             uint64_t duration;
 487                             duration = sub->stop - sub->start;
 488                             sub = hb_fifo_get( subtitle->fifo_raw );
 489                             hb_fifo_push( subtitle->fifo_out, sub );
 490                         } else {
 491                             sub = NULL;
 492                             break;
 493                         }
 494                     }
 495                 }
 496             }
 497
 498             if( subtitle->source == VOBSUB )
 499             {
 500                 hb_buffer_t * sub2;
 501                 while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 502                 {
 503                     if( sub->size == 0 )
 504                     {
 505                         /*
 506                          * EOF, pass it through immediately.
 507                          */
 508                         break;
 509                     }
 510
 511                     /* If two subtitles overlap, make the first one stop
 512                        when the second one starts */
 513                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 514                     if( sub2 && sub->stop > sub2->start )
 515                     {
 516                         sub->stop = sub2->start;
 517                     }
 518
 519                     // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 520                     //       sub, cur->sequence, sub->sequence);
 521
 522                     if( sub->sequence > cur->sequence )
 523                     {
 524                         /*
 525                          * The video is behind where we are, so wait until
 526                          * it catches up to the same reader point on the
 527                          * DVD. Then our PTS should be in the same region
 528                          * as the video.
 529                          */
 530                         sub = NULL;
 531                         break;
 532                     }
 533
 534                     if( sub->stop > cur->start ) {
 535                         /*
 536                          * The stop time is in the future, so fall through
 537                          * and we'll deal with it in the next block of
 538                          * code.
 539                          */
 540
 541                         /*
 542                          * There is a valid subtitle, is it time to display it?
 543                          */
 544                         if( sub->stop > sub->start)
 545                         {
 546                             /*
 547                              * Normal subtitle which ends after it starts,
 548                              * check to see that the current video is between
 549                              * the start and end.
 550                              */
 551                             if( cur->start > sub->start &&
 552                                 cur->start < sub->stop )
 553                             {
 554                                 /*
 555                                 * We should be playing this, so leave the
 556                                 * subtitle in place.
 557                                 *
 558                                 * fall through to display
 559                                 */
 560                                 if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
 561                                 {
 562                                     /*
 563                                      * Subtitle is on for less than three
 564                                      * seconds, extend the time that it is
 565                                      * displayed to make it easier to read.
 566                                      * Make it 3 seconds or until the next
 567                                      * subtitle is displayed.
 568                                      *
 569                                      * This is in response to Indochine which
 570                                      * only displays subs for 1 second -
 571                                      * too fast to read.
 572                                      */
 573                                     sub->stop = sub->start + ( 2 * 90000 );
 574
 575                                     sub2 = hb_fifo_see2( subtitle->fifo_raw );
 576
 577                                     if( sub2 && sub->stop > sub2->start )
 578                                     {
 579                                         sub->stop = sub2->start;
 580                                     }
 581                                 }
 582                             }
 583                             else
 584                             {
 585                                 /*
 586                                  * Defer until the play point is within
 587                                  * the subtitle
 588                                  */
 589                                 sub = NULL;
 590                             }
 591                         }
 592                         else
 593                         {
 594                             /*
 595                              * The end of the subtitle is less than the start,
 596                              * this is a sign of a PTS discontinuity.
 597                              */
 598                             if( sub->start > cur->start )
 599                             {
 600                                 /*
 601                                  * we haven't reached the start time yet, or
 602                                  * we have jumped backwards after having
 603                                  * already started this subtitle.
 604                                  */
 605                                 if( cur->start < sub->stop )
 606                                 {
 607                                     /*
 608                                      * We have jumped backwards and so should
 609                                      * continue displaying this subtitle.
 610                                      *
 611                                      * fall through to display.
 612                                      */
 613                                 }
 614                                 else
 615                                 {
 616                                     /*
 617                                      * Defer until the play point is
 618                                      * within the subtitle
 619                                      */
 620                                     sub = NULL;
 621                                 }
 622                             } else {
 623                                 /*
 624                                 * Play this subtitle as the start is
 625                                 * greater than our video point.
 626                                 *
 627                                 * fall through to display/
 628                                 */
 629                             }
 630                         }
 631                         break;
 632                     }
 633                     else
 634                     {
 635
 636                         /*
 637                          * The subtitle is older than this picture, trash it
 638                          */
 639                         sub = hb_fifo_get( subtitle->fifo_raw );
 640                         hb_buffer_close( &sub );
 641                     }
 642                 }
 643
 644                 /* If we have a subtitle for this picture, copy it */
 645                 /* FIXME: we should avoid this memcpy */
 646                 if( sub )
 647                 {
 648                     if( sub->size > 0 )
 649                     {
 650                         if( subtitle->config.dest == RENDERSUB )
 651                         {
 652                             if ( cur->sub == NULL )
 653                             {
 654                                 /*
 655                                  * Tack onto the video buffer for rendering
 656                                  */
 657                                 cur->sub         = hb_buffer_init( sub->size );
 658                                 cur->sub->x      = sub->x;
 659                                 cur->sub->y      = sub->y;
 660                                 cur->sub->width  = sub->width;
 661                                 cur->sub->height = sub->height;
 662                                 memcpy( cur->sub->data, sub->data, sub->size );
 663                             }
 664                         } else {
 665                             /*
 666                              * Pass-Through, pop it off of the raw queue,
 667                              * rewrite times and make it available to be
 668                              * reencoded.
 669                              */
 670                             uint64_t sub_duration;
 671                             sub = hb_fifo_get( subtitle->fifo_raw );
 672                             sub_duration = sub->stop - sub->start;
 673                             sub->start = cur->start;
 674                             buf_tmp = hb_fifo_see( job->fifo_raw );
 675                             int64_t duration = buf_tmp->start - cur->start;
 676                             sub->stop = sub->start + duration;
 677                             hb_fifo_push( subtitle->fifo_sync, sub );
 678                         }
 679                     } else {
 680                         /*
 681                         * EOF - consume for rendered, else pass through
 682                         */
 683                         if( subtitle->config.dest == RENDERSUB )
 684                         {
 685                             sub = hb_fifo_get( subtitle->fifo_raw );
 686                             hb_buffer_close( &sub );
 687                         } else {
 688                             sub = hb_fifo_get( subtitle->fifo_raw );
 689                             hb_fifo_push( subtitle->fifo_out, sub );
 690                         }
 691                     }
 692                 }
 693             }
 694         } // end subtitles
 695
 696         /*
 697          * Adjust the pts of the current frame so that it's contiguous
 698          * with the previous frame. The start time of the current frame
 699          * has to be the end time of the previous frame and the stop
 700          * time has to be the start of the next frame.  We don't
 701          * make any adjustments to the source timestamps other than removing
 702          * the clock offsets (which also removes pts discontinuities).
 703          * This means we automatically encode at the source's frame rate.
 704          * MP2 uses an implicit duration (frames end when the next frame
 705          * starts) but more advanced containers like MP4 use an explicit
 706          * duration. Since we're looking ahead one frame we set the
 707          * explicit stop time from the start time of the next frame.
 708          */
 709         buf_tmp = cur;
 710         pv->cur = cur = hb_fifo_get( job->fifo_raw );
 711         cur->sub = NULL;
 712         pv->next_pts = cur->start;
 713         int64_t duration = cur->start - pts_skip - buf_tmp->start;
 714         pts_skip = 0;
 715         if ( duration <= 0 )
 716         {
 717             hb_log( "sync: invalid video duration %lld, start %lld, next %lld",
 718                     duration, buf_tmp->start, next->start );
 719         }
 720
 721         buf_tmp->start = pv->next_start;
 722         pv->next_start += duration;
 723         buf_tmp->stop = pv->next_start;
 724
 725         if ( pv->chap_mark )
 726         {
 727             // we have a pending chapter mark from a recent drop - put it on this
 728             // buffer (this may make it one frame late but we can't do any better).
 729             buf_tmp->new_chap = pv->chap_mark;
 730             pv->chap_mark = 0;
 731         }
 732
 733         /* Push the frame to the renderer */
 734         hb_fifo_push( job->fifo_sync, buf_tmp );
 735
 736         /* Update UI */
 737         UpdateState( w );
 738
 739         if( job->frame_to_stop && pv->count_frames > job->frame_to_stop )
 740         {
 741             // Drop an empty buffer into our output to ensure that things
 742             // get flushed all the way out.
 743             hb_fifo_push( job->fifo_sync, hb_buffer_init( 0 ) );
 744             pv->busy &=~ 1;
 745             hb_log( "sync: reached %d frames, exiting early (%i busy)",
 746                     pv->count_frames, pv->busy );
 747             return;
 748         }
 749     }
 750 }
 751
 752 static void OutputAudioFrame( hb_job_t *job, hb_audio_t *audio, hb_buffer_t *buf,
 753                               hb_sync_audio_t *sync, hb_fifo_t *fifo, int i )
 754 {
 755     int64_t start = sync->next_start;
 756     int64_t duration = buf->stop - buf->start;
 757
 758     sync->next_pts += duration;
 759
 760     if( audio->config.in.samplerate == audio->config.out.samplerate ||
 761         audio->config.out.codec == HB_ACODEC_AC3 ||
 762         audio->config.out.codec == HB_ACODEC_DCA )
 763     {
 764         /*
 765          * If we don't have to do sample rate conversion or this audio is
 766          * pass-thru just send the input buffer downstream after adjusting
 767          * its timestamps to make the output stream continuous.
 768          */
 769     }
 770     else
 771     {
 772         /* Not pass-thru - do sample rate conversion */
 773         int count_in, count_out;
 774         hb_buffer_t * buf_raw = buf;
 775         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
 776                             sizeof( float );
 777
 778         count_in  = buf_raw->size / channel_count;
 779         /*
 780          * When using stupid rates like 44.1 there will always be some
 781          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
 782          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
 783          * the error will build up over time and eventually the audio will
 784          * substantially lag the video. libsamplerate will keep track of the
 785          * fractional sample & give it to us when appropriate if we give it
 786          * an extra sample of space in the output buffer.
 787          */
 788         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
 789
 790         sync->data.input_frames = count_in;
 791         sync->data.output_frames = count_out;
 792         sync->data.src_ratio = (double)audio->config.out.samplerate /
 793                                (double)audio->config.in.samplerate;
 794
 795         buf = hb_buffer_init( count_out * channel_count );
 796         sync->data.data_in  = (float *) buf_raw->data;
 797         sync->data.data_out = (float *) buf->data;
 798         if( src_process( sync->state, &sync->data ) )
 799         {
 800             /* XXX If this happens, we're screwed */
 801             hb_log( "sync: audio %d src_process failed", i );
 802         }
 803         hb_buffer_close( &buf_raw );
 804
 805         buf->size = sync->data.output_frames_gen * channel_count;
 806         duration = ( sync->data.output_frames_gen * 90000 ) /
 807                    audio->config.out.samplerate;
 808     }
 809     buf->frametype = HB_FRAME_AUDIO;
 810     buf->start = start;
 811     buf->stop  = start + duration;
 812     sync->next_start = start + duration;
 813     hb_fifo_push( fifo, buf );
 814 }
 815
 816 /***********************************************************************
 817  * SyncAudio
 818  ***********************************************************************
 819  *
 820  **********************************************************************/
 821 static void SyncAudio( hb_work_object_t * w, int i )
 822 {
 823     hb_work_private_t * pv = w->private_data;
 824     hb_job_t        * job = pv->job;
 825     hb_sync_audio_t * sync = &pv->sync_audio[i];
 826     hb_audio_t      * audio = sync->audio;
 827     hb_buffer_t     * buf;
 828     hb_fifo_t       * fifo;
 829     int64_t start;
 830
 831     if( audio->config.out.codec == HB_ACODEC_AC3 ||
 832         audio->config.out.codec == HB_ACODEC_DCA )
 833     {
 834         fifo = audio->priv.fifo_out;
 835     }
 836     else
 837     {
 838         fifo = audio->priv.fifo_sync;
 839     }
 840
 841     while( !hb_fifo_is_full( fifo ) && ( buf = hb_fifo_see( audio->priv.fifo_raw ) ) )
 842     {
 843         start = buf->start - pv->audio_passthru_slip;
 844         /* if the next buffer is an eof send it downstream */
 845         if ( buf->size <= 0 )
 846         {
 847             buf = hb_fifo_get( audio->priv.fifo_raw );
 848             hb_fifo_push( fifo, buf );
 849             pv->busy &=~ (1 << (i + 1) );
 850             return;
 851         }
 852         if( job->frame_to_stop && pv->count_frames >= job->frame_to_stop )
 853         {
 854             hb_fifo_push( fifo, hb_buffer_init(0) );
 855             pv->busy &=~ (1 << (i + 1) );
 856             return;
 857         }
 858         if ( (int64_t)( start - sync->next_pts ) < 0 )
 859         {
 860             // audio time went backwards.
 861             // If our output clock is more than a half frame ahead of the
 862             // input clock drop this frame to move closer to sync.
 863             // Otherwise drop frames until the input clock matches the output clock.
 864             if ( sync->first_drop || sync->next_start - start > 90*15 )
 865             {
 866                 // Discard data that's in the past.
 867                 if ( sync->first_drop == 0 )
 868                 {
 869                     sync->first_drop = sync->next_pts;
 870                 }
 871                 ++sync->drop_count;
 872                 buf = hb_fifo_get( audio->priv.fifo_raw );
 873                 hb_buffer_close( &buf );
 874                 continue;
 875             }
 876             sync->next_pts = start;
 877         }
 878         if ( sync->first_drop )
 879         {
 880             // we were dropping old data but input buf time is now current
 881             hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
 882                     "(next %lld, current %lld)", i,
 883                     (int)( sync->next_pts - sync->first_drop ) / 90,
 884                     sync->drop_count, sync->first_drop, sync->next_pts );
 885             sync->first_drop = 0;
 886             sync->drop_count = 0;
 887             sync->next_pts = start;
 888         }
 889         if ( start - sync->next_pts >= (90 * 70) )
 890         {
 891             if ( start - sync->next_pts > (90000LL * 60) )
 892             {
 893                 // there's a gap of more than a minute between the last
 894                 // frame and this. assume we got a corrupted timestamp
 895                 // and just drop the next buf.
 896                 hb_log( "sync: %d minute time gap in audio %d - dropping buf"
 897                         "  start %lld, next %lld",
 898                         (int)((start - sync->next_pts) / (90000*60)),
 899                         i, start, sync->next_pts );
 900                 buf = hb_fifo_get( audio->priv.fifo_raw );
 901                 hb_buffer_close( &buf );
 902                 continue;
 903             }
 904             /*
 905              * there's a gap of at least 70ms between the last
 906              * frame we processed & the next. Fill it with silence.
 907              * Or in the case of DCA, skip some frames from the
 908              * other streams.
 909              */
 910             if( sync->audio->config.out.codec == HB_ACODEC_DCA )
 911             {
 912                 hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
 913                         "  start %lld, next %lld",
 914                         (int)((start - sync->next_pts) / 90),
 915                         i, start, sync->next_pts );
 916                 pv->audio_passthru_slip += (start - sync->next_pts);
 917                 pv->video_pts_slip += (start - sync->next_pts);
 918                 return;
 919             }
 920             hb_log( "sync: adding %d ms of silence to audio %d"
 921                     "  start %lld, next %lld",
 922                     (int)((start - sync->next_pts) / 90),
 923                     i, start, sync->next_pts );
 924             InsertSilence( w, i, start - sync->next_pts );
 925             return;
 926         }
 927
 928         /*
 929          * When we get here we've taken care of all the dups and gaps in the
 930          * audio stream and are ready to inject the next input frame into
 931          * the output stream.
 932          */
 933         buf = hb_fifo_get( audio->priv.fifo_raw );
 934         OutputAudioFrame( job, audio, buf, sync, fifo, i );
 935     }
 936 }
 937
 938 static void InsertSilence( hb_work_object_t * w, int i, int64_t duration )
 939 {
 940     hb_work_private_t * pv = w->private_data;
 941     hb_job_t        *job = pv->job;
 942     hb_sync_audio_t *sync = &pv->sync_audio[i];
 943     hb_buffer_t     *buf;
 944     hb_fifo_t       *fifo;
 945
 946     // to keep pass-thru and regular audio in sync we generate silence in
 947     // AC3 frame-sized units. If the silence duration isn't an integer multiple
 948     // of the AC3 frame duration we will truncate or round up depending on
 949     // which minimizes the timing error.
 950     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
 951                           sync->audio->config.in.samplerate;
 952     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
 953
 954     while ( --frame_count >= 0 )
 955     {
 956         if( sync->audio->config.out.codec == HB_ACODEC_AC3 )
 957         {
 958             buf        = hb_buffer_init( sync->ac3_size );
 959             buf->start = sync->next_pts;
 960             buf->stop  = buf->start + frame_dur;
 961             memcpy( buf->data, sync->ac3_buf, buf->size );
 962             fifo = sync->audio->priv.fifo_out;
 963         }
 964         else
 965         {
 966             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
 967                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
 968                                          sync->audio->config.out.mixdown) );
 969             buf->start = sync->next_pts;
 970             buf->stop  = buf->start + frame_dur;
 971             memset( buf->data, 0, buf->size );
 972             fifo = sync->audio->priv.fifo_sync;
 973         }
 974         OutputAudioFrame( job, sync->audio, buf, sync, fifo, i );
 975     }
 976 }
 977
 978 static void UpdateState( hb_work_object_t * w )
 979 {
 980     hb_work_private_t * pv = w->private_data;
 981     hb_state_t state;
 982
 983     if( !pv->count_frames )
 984     {
 985         pv->st_first = hb_get_date();
 986     }
 987     pv->count_frames++;
 988
 989     if( hb_get_date() > pv->st_dates[3] + 1000 )
 990     {
 991         memmove( &pv->st_dates[0], &pv->st_dates[1],
 992                  3 * sizeof( uint64_t ) );
 993         memmove( &pv->st_counts[0], &pv->st_counts[1],
 994                  3 * sizeof( uint64_t ) );
 995         pv->st_dates[3]  = hb_get_date();
 996         pv->st_counts[3] = pv->count_frames;
 997     }
 998
 999 #define p state.param.working
1000     state.state = HB_STATE_WORKING;
1001     p.progress  = (float) pv->count_frames / (float) pv->count_frames_max;
1002     if( p.progress > 1.0 )
1003     {
1004         p.progress = 1.0;
1005     }
1006     p.rate_cur   = 1000.0 *
1007         (float) ( pv->st_counts[3] - pv->st_counts[0] ) /
1008         (float) ( pv->st_dates[3] - pv->st_dates[0] );
1009     if( hb_get_date() > pv->st_first + 4000 )
1010     {
1011         int eta;
1012         p.rate_avg = 1000.0 * (float) pv->st_counts[3] /
1013             (float) ( pv->st_dates[3] - pv->st_first );
1014         eta = (float) ( pv->count_frames_max - pv->st_counts[3] ) /
1015             p.rate_avg;
1016         p.hours   = eta / 3600;
1017         p.minutes = ( eta % 3600 ) / 60;
1018         p.seconds = eta % 60;
1019     }
1020     else
1021     {
1022         p.rate_avg = 0.0;
1023         p.hours    = -1;
1024         p.minutes  = -1;
1025         p.seconds  = -1;
1026     }
1027 #undef p
1028
1029     hb_set_state( pv->job->h, &state );
1030 }