libhb/sync.c

   1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
   2
   3    This file is part of the HandBrake source code.
   4    Homepage: <http://handbrake.fr/>.
   5    It may be used under the terms of the GNU General Public License. */
   6
   7 #include "hb.h"
   8 #include "hbffmpeg.h"
   9 #include <stdio.h>
  10 #include "samplerate.h"
  11
  12 #ifdef INT64_MIN
  13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
  14 #endif
  15 #define INT64_MIN (-9223372036854775807LL-1)
  16
  17 #define AC3_SAMPLES_PER_FRAME 1536
  18
  19 typedef struct
  20 {
  21     hb_lock_t * mutex;
  22     int         ref;        /* Reference count to tell us when it's unused */
  23     int         count_frames;
  24     int64_t     audio_passthru_slip;
  25     int64_t     video_pts_slip;
  26 } hb_sync_common_t;
  27
  28 typedef struct
  29 {
  30     int64_t      next_start;    /* start time of next output frame */
  31     int64_t      next_pts;     /* start time of next input frame */
  32     int64_t      first_drop;   /* PTS of first 'went backwards' frame dropped */
  33     int          drop_count;   /* count of 'time went backwards' drops */
  34
  35     /* Raw */
  36     SRC_STATE  * state;
  37     SRC_DATA     data;
  38
  39     /* AC-3 */
  40     int          ac3_size;
  41     uint8_t    * ac3_buf;
  42 } hb_sync_audio_t;
  43
  44 typedef struct
  45 {
  46     /* Video */
  47     int64_t    pts_offset;
  48     int64_t    pts_skip;
  49     int64_t    next_start;    /* start time of next output frame */
  50     int64_t    next_pts;      /* start time of next input frame */
  51     int64_t    first_drop;    /* PTS of first 'went backwards' frame dropped */
  52     int        drop_count;    /* count of 'time went backwards' drops */
  53     int        drops;         /* frames dropped to make a cbr video stream */
  54     int        dups;          /* frames duplicated to make a cbr video stream */
  55     int        video_sequence;
  56     int        count_frames_max;
  57     int        chap_mark;     /* to propagate chapter mark across a drop */
  58     hb_buffer_t * cur;        /* The next picture to process */
  59
  60     /* Statistics */
  61     uint64_t   st_counts[4];
  62     uint64_t   st_dates[4];
  63     uint64_t   st_first;
  64 } hb_sync_video_t;
  65
  66 struct hb_work_private_s
  67 {
  68     hb_job_t * job;
  69     hb_sync_common_t * common;
  70     union
  71     {
  72         hb_sync_video_t video;
  73         hb_sync_audio_t audio;
  74     } type;
  75 };
  76
  77 /***********************************************************************
  78  * Local prototypes
  79  **********************************************************************/
  80 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i );
  81 static void InsertSilence( hb_work_object_t * w, int64_t d );
  82 static void UpdateState( hb_work_object_t * w );
  83 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
  84                                        hb_sync_audio_t *sync );
  85
  86 /***********************************************************************
  87  * hb_work_sync_init
  88  ***********************************************************************
  89  * Initialize the work object
  90  **********************************************************************/
  91 int hb_sync_init( hb_job_t * job )
  92 {
  93     hb_title_t        * title = job->title;
  94     hb_chapter_t      * chapter;
  95     int                 i;
  96     uint64_t            duration;
  97     hb_work_private_t * pv;
  98     hb_sync_video_t   * sync;
  99     hb_work_object_t  * w;
 100
 101     pv = calloc( 1, sizeof( hb_work_private_t ) );
 102     sync = &pv->type.video;
 103     pv->common = calloc( 1, sizeof( hb_sync_common_t ) );
 104     pv->common->ref++;
 105     pv->common->mutex = hb_lock_init();
 106
 107     w = hb_get_work( WORK_SYNC_VIDEO );
 108     w->private_data = pv;
 109     w->fifo_in = job->fifo_raw;
 110     w->fifo_out = job->fifo_sync;
 111
 112     pv->job            = job;
 113     sync->pts_offset   = INT64_MIN;
 114
 115     if( job->pass == 2 )
 116     {
 117         /* We already have an accurate frame count from pass 1 */
 118         hb_interjob_t * interjob = hb_interjob_get( job->h );
 119         sync->count_frames_max = interjob->frame_count;
 120     }
 121     else
 122     {
 123         /* Calculate how many video frames we are expecting */
 124         if ( job->pts_to_stop )
 125         {
 126             duration = job->pts_to_stop + 90000;
 127         }
 128         else if( job->frame_to_stop )
 129         {
 130             /* Set the duration to a rough estimate */
 131             duration = ( job->frame_to_stop / ( title->rate / title->rate_base ) ) * 90000;
 132         }
 133         else
 134         {
 135             duration = 0;
 136             for( i = job->chapter_start; i <= job->chapter_end; i++ )
 137             {
 138                 chapter   = hb_list_item( title->list_chapter, i - 1 );
 139                 duration += chapter->duration;
 140             }
 141             duration += 90000;
 142             /* 1 second safety so we're sure we won't miss anything */
 143         }
 144         sync->count_frames_max = duration * title->rate / title->rate_base / 90000;
 145     }
 146     hb_list_add( job->list_work, w );
 147
 148     hb_log( "sync: expecting %d video frames", sync->count_frames_max );
 149
 150     /* Initialize libsamplerate for every audio track we have */
 151     if ( ! job->indepth_scan )
 152     {
 153         for( i = 0; i < hb_list_count( title->list_audio ) && i < 8; i++ )
 154         {
 155             InitAudio( job, pv->common, i );
 156         }
 157     }
 158
 159     return 0;
 160 }
 161
 162 /***********************************************************************
 163  * Close Video
 164  ***********************************************************************
 165  *
 166  **********************************************************************/
 167 void syncVideoClose( hb_work_object_t * w )
 168 {
 169     hb_work_private_t * pv = w->private_data;
 170     hb_job_t          * job   = pv->job;
 171     hb_sync_video_t   * sync = &pv->type.video;
 172
 173     if( sync->cur )
 174     {
 175         hb_buffer_close( &sync->cur );
 176     }
 177
 178     hb_log( "sync: got %d frames, %d expected",
 179             pv->common->count_frames, sync->count_frames_max );
 180
 181     /* save data for second pass */
 182     if( job->pass == 1 )
 183     {
 184         /* Preserve frame count for better accuracy in pass 2 */
 185         hb_interjob_t * interjob = hb_interjob_get( job->h );
 186         interjob->frame_count = pv->common->count_frames;
 187         interjob->last_job = job->sequence_id;
 188         interjob->total_time = sync->next_start;
 189     }
 190
 191     if (sync->drops || sync->dups )
 192     {
 193         hb_log( "sync: %d frames dropped, %d duplicated",
 194                 sync->drops, sync->dups );
 195     }
 196
 197     hb_lock( pv->common->mutex );
 198     if ( --pv->common->ref == 0 )
 199     {
 200         hb_unlock( pv->common->mutex );
 201         hb_lock_close( &pv->common->mutex );
 202         free( pv->common );
 203     }
 204     else
 205     {
 206         hb_unlock( pv->common->mutex );
 207     }
 208
 209     free( pv );
 210     w->private_data = NULL;
 211 }
 212
 213 /***********************************************************************
 214  * syncVideoWork
 215  ***********************************************************************
 216  *
 217  **********************************************************************/
 218 int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
 219               hb_buffer_t ** buf_out )
 220 {
 221     hb_buffer_t * cur, * next, * sub = NULL;
 222     hb_work_private_t * pv = w->private_data;
 223     hb_job_t          * job = pv->job;
 224     hb_subtitle_t     * subtitle;
 225     hb_sync_video_t   * sync = &pv->type.video;
 226     int i;
 227
 228     *buf_out = NULL;
 229     if( !sync->cur )
 230     {
 231         sync->cur = *buf_in;
 232         *buf_in = NULL;
 233         if( sync->cur->size == 0 )
 234         {
 235             /* we got an end-of-stream as our first video packet?
 236              * Feed it downstream & signal that we're done.
 237              */
 238             *buf_out = hb_buffer_init( 0 );
 239
 240             /*
 241              * Push through any subtitle EOFs in case they
 242              * were not synced through.
 243              */
 244             for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 245             {
 246                 subtitle = hb_list_item( job->list_subtitle, i );
 247                 if( subtitle->config.dest == PASSTHRUSUB )
 248                 {
 249                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 250                 }
 251             }
 252             return HB_WORK_DONE;
 253         }
 254         return HB_WORK_OK;
 255     }
 256     next = *buf_in;
 257     *buf_in = NULL;
 258     cur = sync->cur;
 259     if( job->frame_to_stop && pv->common->count_frames > job->frame_to_stop )
 260     {
 261         // Drop an empty buffer into our output to ensure that things
 262         // get flushed all the way out.
 263         hb_buffer_close( &sync->cur );
 264         hb_buffer_close( &next );
 265         *buf_out = hb_buffer_init( 0 );
 266         hb_log( "sync: reached %d frames, exiting early",
 267                 pv->common->count_frames );
 268         return HB_WORK_DONE;
 269     }
 270
 271     /* At this point we have a frame to process. Let's check
 272         1) if we will be able to push into the fifo ahead
 273         2) if the next frame is there already, since we need it to
 274            compute the duration of the current frame*/
 275     if( next->size == 0 )
 276     {
 277         hb_buffer_close( &next );
 278
 279         cur->start = sync->next_start;
 280         cur->stop = cur->start + 90000. / ((double)job->vrate / (double)job->vrate_base);
 281
 282         /* Push the frame to the renderer */
 283         hb_fifo_push( job->fifo_sync, cur );
 284         sync->cur = NULL;
 285
 286         /* we got an end-of-stream. Feed it downstream & signal that
 287          * we're done. Note that this means we drop the final frame of
 288          * video (we don't know its duration). On DVDs the final frame
 289          * is often strange and dropping it seems to be a good idea. */
 290         *buf_out = hb_buffer_init( 0 );
 291
 292         /*
 293          * Push through any subtitle EOFs in case they were not synced through.
 294          */
 295         for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 296         {
 297             subtitle = hb_list_item( job->list_subtitle, i );
 298             if( subtitle->config.dest == PASSTHRUSUB )
 299             {
 300                 if( subtitle->source == VOBSUB )
 301                     hb_fifo_push( subtitle->fifo_sync, hb_buffer_init( 0 ) );
 302                 else
 303                     hb_fifo_push( subtitle->fifo_out, hb_buffer_init( 0 ) );
 304             }
 305         }
 306         return HB_WORK_DONE;
 307     }
 308     if( sync->pts_offset == INT64_MIN )
 309     {
 310         /* This is our first frame */
 311         sync->pts_offset = 0;
 312         if ( cur->start != 0 )
 313         {
 314             /*
 315              * The first pts from a dvd should always be zero but
 316              * can be non-zero with a transport or program stream since
 317              * we're not guaranteed to start on an IDR frame. If we get
 318              * a non-zero initial PTS extend its duration so it behaves
 319              * as if it started at zero so that our audio timing will
 320              * be in sync.
 321              */
 322             hb_log( "sync: first pts is %"PRId64, cur->start );
 323             cur->start = 0;
 324         }
 325     }
 326
 327     /*
 328      * since the first frame is always 0 and the upstream reader code
 329      * is taking care of adjusting for pts discontinuities, we just have
 330      * to deal with the next frame's start being in the past. This can
 331      * happen when the PTS is adjusted after data loss but video frame
 332      * reordering causes some frames with the old clock to appear after
 333      * the clock change. This creates frames that overlap in time which
 334      * looks to us like time going backward. The downstream muxing code
 335      * can deal with overlaps of up to a frame time but anything larger
 336      * we handle by dropping frames here.
 337      */
 338     hb_lock( pv->common->mutex );
 339     if ( (int64_t)( next->start - pv->common->video_pts_slip - cur->start ) <= 0 )
 340     {
 341         if ( sync->first_drop == 0 )
 342         {
 343             sync->first_drop = next->start;
 344         }
 345         ++sync->drop_count;
 346         if (next->start - cur->start > 0)
 347         {
 348             sync->pts_skip += next->start - cur->start;
 349             pv->common->video_pts_slip -= next->start - cur->start;
 350         }
 351         hb_unlock( pv->common->mutex );
 352         if ( next->new_chap )
 353         {
 354             // don't drop a chapter mark when we drop the buffer
 355             sync->chap_mark = next->new_chap;
 356         }
 357         hb_buffer_close( &next );
 358         return HB_WORK_OK;
 359     }
 360     hb_unlock( pv->common->mutex );
 361     if ( sync->first_drop )
 362     {
 363         hb_log( "sync: video time didn't advance - dropped %d frames "
 364                 "(delta %d ms, current %"PRId64", next %"PRId64", dur %d)",
 365                 sync->drop_count, (int)( cur->start - sync->first_drop ) / 90,
 366                 cur->start, next->start, (int)( next->start - cur->start ) );
 367         sync->first_drop = 0;
 368         sync->drop_count = 0;
 369     }
 370
 371     /*
 372      * Track the video sequence number localy so that we can sync the audio
 373      * to it using the sequence number as well as the PTS.
 374      */
 375     sync->video_sequence = cur->sequence;
 376
 377     /*
 378      * Look for a subtitle for this frame.
 379      *
 380      * If found then it will be tagged onto a video buffer of the correct time and
 381      * sent in to the render pipeline. This only needs to be done for VOBSUBs which
 382      * get rendered, other types of subtitles can just sit in their raw_queue until
 383      * delt with at muxing.
 384      */
 385     for( i = 0; i < hb_list_count( job->list_subtitle ); i++)
 386     {
 387         subtitle = hb_list_item( job->list_subtitle, i );
 388
 389         /*
 390          * Rewrite timestamps on subtitles that need it (on raw queue).
 391          */
 392         if( subtitle->source == CC608SUB ||
 393             subtitle->source == CC708SUB ||
 394             subtitle->source == SRTSUB )
 395         {
 396             /*
 397              * Rewrite timestamps on subtitles that came from Closed Captions
 398              * since they are using the MPEG2 timestamps.
 399              */
 400             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 401             {
 402                 /*
 403                  * Rewrite the timestamps as and when the video
 404                  * (cur->start) reaches the same timestamp as a
 405                  * closed caption (sub->start).
 406                  *
 407                  * What about discontinuity boundaries - not delt
 408                  * with here - Van?
 409                  *
 410                  * Bypass the sync fifo altogether.
 411                  */
 412                 if( sub->size <= 0 )
 413                 {
 414                     sub = hb_fifo_get( subtitle->fifo_raw );
 415                     hb_fifo_push( subtitle->fifo_out, sub );
 416                     sub = NULL;
 417                     break;
 418                 } else {
 419                     /*
 420                      * Sync the subtitles to the incoming video, and use
 421                      * the matching converted video timestamp.
 422                      *
 423                      * Note that it doesn't appear that we need to convert
 424                      * timestamps, I guess that they were already correct,
 425                      * so just push them through for rendering.
 426                      *
 427                      */
 428                     if( sub->start < cur->start )
 429                     {
 430                         sub = hb_fifo_get( subtitle->fifo_raw );
 431                         hb_fifo_push( subtitle->fifo_out, sub );
 432                     } else {
 433                         sub = NULL;
 434                         break;
 435                     }
 436                 }
 437             }
 438         }
 439
 440         if( subtitle->source == VOBSUB )
 441         {
 442             hb_buffer_t * sub2;
 443             while( ( sub = hb_fifo_see( subtitle->fifo_raw ) ) )
 444             {
 445                 if( sub->size == 0 )
 446                 {
 447                     /*
 448                      * EOF, pass it through immediately.
 449                      */
 450                     break;
 451                 }
 452
 453                 /* If two subtitles overlap, make the first one stop
 454                    when the second one starts */
 455                 sub2 = hb_fifo_see2( subtitle->fifo_raw );
 456                 if( sub2 && sub->stop > sub2->start )
 457                 {
 458                     sub->stop = sub2->start;
 459                 }
 460
 461                 // hb_log("0x%x: video seq: %lld  subtitle sequence: %lld",
 462                 //       sub, cur->sequence, sub->sequence);
 463
 464                 if( sub->sequence > cur->sequence )
 465                 {
 466                     /*
 467                      * The video is behind where we are, so wait until
 468                      * it catches up to the same reader point on the
 469                      * DVD. Then our PTS should be in the same region
 470                      * as the video.
 471                      */
 472                     sub = NULL;
 473                     break;
 474                 }
 475
 476                 if( sub->stop > cur->start ) {
 477                     /*
 478                      * The stop time is in the future, so fall through
 479                      * and we'll deal with it in the next block of
 480                      * code.
 481                      */
 482
 483                     /*
 484                      * There is a valid subtitle, is it time to display it?
 485                      */
 486                     if( sub->stop > sub->start)
 487                     {
 488                         /*
 489                          * Normal subtitle which ends after it starts,
 490                          * check to see that the current video is between
 491                          * the start and end.
 492                          */
 493                         if( cur->start > sub->start &&
 494                             cur->start < sub->stop )
 495                         {
 496                             /*
 497                             * We should be playing this, so leave the
 498                             * subtitle in place.
 499                             *
 500                             * fall through to display
 501                             */
 502                             if( ( sub->stop - sub->start ) < ( 2 * 90000 ) )
 503                             {
 504                                 /*
 505                                  * Subtitle is on for less than three
 506                                  * seconds, extend the time that it is
 507                                  * displayed to make it easier to read.
 508                                  * Make it 3 seconds or until the next
 509                                  * subtitle is displayed.
 510                                  *
 511                                  * This is in response to Indochine which
 512                                  * only displays subs for 1 second -
 513                                  * too fast to read.
 514                                  */
 515                                 sub->stop = sub->start + ( 2 * 90000 );
 516
 517                                 sub2 = hb_fifo_see2( subtitle->fifo_raw );
 518
 519                                 if( sub2 && sub->stop > sub2->start )
 520                                 {
 521                                     sub->stop = sub2->start;
 522                                 }
 523                             }
 524                         }
 525                         else
 526                         {
 527                             /*
 528                              * Defer until the play point is within
 529                              * the subtitle
 530                              */
 531                             sub = NULL;
 532                         }
 533                     }
 534                     else
 535                     {
 536                         /*
 537                          * The end of the subtitle is less than the start,
 538                          * this is a sign of a PTS discontinuity.
 539                          */
 540                         if( sub->start > cur->start )
 541                         {
 542                             /*
 543                              * we haven't reached the start time yet, or
 544                              * we have jumped backwards after having
 545                              * already started this subtitle.
 546                              */
 547                             if( cur->start < sub->stop )
 548                             {
 549                                 /*
 550                                  * We have jumped backwards and so should
 551                                  * continue displaying this subtitle.
 552                                  *
 553                                  * fall through to display.
 554                                  */
 555                             }
 556                             else
 557                             {
 558                                 /*
 559                                  * Defer until the play point is
 560                                  * within the subtitle
 561                                  */
 562                                 sub = NULL;
 563                             }
 564                         } else {
 565                             /*
 566                             * Play this subtitle as the start is
 567                             * greater than our video point.
 568                             *
 569                             * fall through to display/
 570                             */
 571                         }
 572                     }
 573                         break;
 574                 }
 575                 else
 576                 {
 577
 578                     /*
 579                      * The subtitle is older than this picture, trash it
 580                      */
 581                     sub = hb_fifo_get( subtitle->fifo_raw );
 582                     hb_buffer_close( &sub );
 583                 }
 584             }
 585
 586             /* If we have a subtitle for this picture, copy it */
 587             /* FIXME: we should avoid this memcpy */
 588             if( sub )
 589             {
 590                 if( sub->size > 0 )
 591                 {
 592                     if( subtitle->config.dest == RENDERSUB )
 593                     {
 594                         if ( cur->sub == NULL )
 595                         {
 596                             /*
 597                              * Tack onto the video buffer for rendering
 598                              */
 599                             cur->sub         = hb_buffer_init( sub->size );
 600                             cur->sub->x      = sub->x;
 601                             cur->sub->y      = sub->y;
 602                             cur->sub->width  = sub->width;
 603                             cur->sub->height = sub->height;
 604                             memcpy( cur->sub->data, sub->data, sub->size );
 605                         }
 606                     } else {
 607                         /*
 608                          * Pass-Through, pop it off of the raw queue,
 609                          */
 610                         sub = hb_fifo_get( subtitle->fifo_raw );
 611                         hb_fifo_push( subtitle->fifo_sync, sub );
 612                     }
 613                 } else {
 614                     /*
 615                     * EOF - consume for rendered, else pass through
 616                     */
 617                     if( subtitle->config.dest == RENDERSUB )
 618                     {
 619                         sub = hb_fifo_get( subtitle->fifo_raw );
 620                         hb_buffer_close( &sub );
 621                     } else {
 622                         sub = hb_fifo_get( subtitle->fifo_raw );
 623                         hb_fifo_push( subtitle->fifo_sync, sub );
 624                     }
 625                 }
 626             }
 627         }
 628     } // end subtitles
 629
 630     /*
 631      * Adjust the pts of the current frame so that it's contiguous
 632      * with the previous frame. The start time of the current frame
 633      * has to be the end time of the previous frame and the stop
 634      * time has to be the start of the next frame.  We don't
 635      * make any adjustments to the source timestamps other than removing
 636      * the clock offsets (which also removes pts discontinuities).
 637      * This means we automatically encode at the source's frame rate.
 638      * MP2 uses an implicit duration (frames end when the next frame
 639      * starts) but more advanced containers like MP4 use an explicit
 640      * duration. Since we're looking ahead one frame we set the
 641      * explicit stop time from the start time of the next frame.
 642      */
 643     *buf_out = cur;
 644     sync->cur = cur = next;
 645     cur->sub = NULL;
 646     sync->next_pts = cur->start;
 647     int64_t duration = cur->start - sync->pts_skip - (*buf_out)->start;
 648     sync->pts_skip = 0;
 649     if ( duration <= 0 )
 650     {
 651         hb_log( "sync: invalid video duration %"PRId64", start %"PRId64", next %"PRId64"",
 652                 duration, (*buf_out)->start, next->start );
 653     }
 654
 655     (*buf_out)->start = sync->next_start;
 656     sync->next_start += duration;
 657     (*buf_out)->stop = sync->next_start;
 658
 659     if ( sync->chap_mark )
 660     {
 661         // we have a pending chapter mark from a recent drop - put it on this
 662         // buffer (this may make it one frame late but we can't do any better).
 663         (*buf_out)->new_chap = sync->chap_mark;
 664         sync->chap_mark = 0;
 665     }
 666
 667     /* Update UI */
 668     UpdateState( w );
 669
 670     return HB_WORK_OK;
 671 }
 672
 673 // sync*Init does nothing because sync has a special initializer
 674 // that takes care of initializing video and all audio tracks
 675 int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
 676 {
 677     return 0;
 678 }
 679
 680 hb_work_object_t hb_sync_video =
 681 {
 682     WORK_SYNC_VIDEO,
 683     "Video Synchronization",
 684     syncVideoInit,
 685     syncVideoWork,
 686     syncVideoClose
 687 };
 688
 689 /***********************************************************************
 690  * Close Audio
 691  ***********************************************************************
 692  *
 693  **********************************************************************/
 694 void syncAudioClose( hb_work_object_t * w )
 695 {
 696     hb_work_private_t * pv    = w->private_data;
 697     hb_sync_audio_t   * sync  = &pv->type.audio;
 698
 699     if( w->audio->config.out.codec == HB_ACODEC_AC3 )
 700     {
 701         free( sync->ac3_buf );
 702     }
 703     else
 704     {
 705         src_delete( sync->state );
 706     }
 707
 708     hb_lock( pv->common->mutex );
 709     if ( --pv->common->ref == 0 )
 710     {
 711         hb_unlock( pv->common->mutex );
 712         hb_lock_close( &pv->common->mutex );
 713         free( pv->common );
 714     }
 715     else
 716     {
 717         hb_unlock( pv->common->mutex );
 718     }
 719
 720     free( pv );
 721     w->private_data = NULL;
 722 }
 723
 724 int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
 725 {
 726     return 0;
 727 }
 728
 729 /***********************************************************************
 730  * SyncAudio
 731  ***********************************************************************
 732  *
 733  **********************************************************************/
 734 static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
 735                        hb_buffer_t ** buf_out )
 736 {
 737     hb_work_private_t * pv = w->private_data;
 738     hb_job_t        * job = pv->job;
 739     hb_sync_audio_t * sync = &pv->type.audio;
 740     hb_buffer_t     * buf;
 741     //hb_fifo_t       * fifo;
 742     int64_t start;
 743
 744     *buf_out = NULL;
 745     buf = *buf_in;
 746     *buf_in = NULL;
 747     hb_lock( pv->common->mutex );
 748     start = buf->start - pv->common->audio_passthru_slip;
 749     hb_unlock( pv->common->mutex );
 750     /* if the next buffer is an eof send it downstream */
 751     if ( buf->size <= 0 )
 752     {
 753         hb_buffer_close( &buf );
 754         *buf_out = hb_buffer_init( 0 );
 755         return HB_WORK_DONE;
 756     }
 757     if( job->frame_to_stop && pv->common->count_frames >= job->frame_to_stop )
 758     {
 759         hb_buffer_close( &buf );
 760         *buf_out = hb_buffer_init( 0 );
 761         return HB_WORK_DONE;
 762     }
 763     if ( (int64_t)( start - sync->next_pts ) < 0 )
 764     {
 765         // audio time went backwards.
 766         // If our output clock is more than a half frame ahead of the
 767         // input clock drop this frame to move closer to sync.
 768         // Otherwise drop frames until the input clock matches the output clock.
 769         if ( sync->first_drop || sync->next_start - start > 90*15 )
 770         {
 771             // Discard data that's in the past.
 772             if ( sync->first_drop == 0 )
 773             {
 774                 sync->first_drop = sync->next_pts;
 775             }
 776             ++sync->drop_count;
 777             hb_buffer_close( &buf );
 778             return HB_WORK_OK;
 779         }
 780         sync->next_pts = start;
 781     }
 782     if ( sync->first_drop )
 783     {
 784         // we were dropping old data but input buf time is now current
 785         hb_log( "sync: audio %d time went backwards %d ms, dropped %d frames "
 786                 "(next %"PRId64", current %"PRId64")", w->audio->id,
 787                 (int)( sync->next_pts - sync->first_drop ) / 90,
 788                 sync->drop_count, sync->first_drop, sync->next_pts );
 789         sync->first_drop = 0;
 790         sync->drop_count = 0;
 791         sync->next_pts = start;
 792     }
 793     if ( start - sync->next_pts >= (90 * 70) )
 794     {
 795         if ( start - sync->next_pts > (90000LL * 60) )
 796         {
 797             // there's a gap of more than a minute between the last
 798             // frame and this. assume we got a corrupted timestamp
 799             // and just drop the next buf.
 800             hb_log( "sync: %d minute time gap in audio %d - dropping buf"
 801                     "  start %"PRId64", next %"PRId64,
 802                     (int)((start - sync->next_pts) / (90000*60)),
 803                     w->audio->id, start, sync->next_pts );
 804             hb_buffer_close( &buf );
 805             return HB_WORK_OK;
 806         }
 807         /*
 808          * there's a gap of at least 70ms between the last
 809          * frame we processed & the next. Fill it with silence.
 810          * Or in the case of DCA, skip some frames from the
 811          * other streams.
 812          */
 813         if( w->audio->config.out.codec == HB_ACODEC_DCA )
 814         {
 815             hb_log( "sync: audio gap %d ms. Skipping frames. Audio %d"
 816                     "  start %"PRId64", next %"PRId64,
 817                     (int)((start - sync->next_pts) / 90),
 818                     w->audio->id, start, sync->next_pts );
 819             hb_lock( pv->common->mutex );
 820             pv->common->audio_passthru_slip += (start - sync->next_pts);
 821             pv->common->video_pts_slip += (start - sync->next_pts);
 822             hb_unlock( pv->common->mutex );
 823             *buf_out = buf;
 824             return HB_WORK_OK;
 825         }
 826         hb_log( "sync: adding %d ms of silence to audio %d"
 827                 "  start %"PRId64", next %"PRId64,
 828                 (int)((start - sync->next_pts) / 90),
 829                 w->audio->id, start, sync->next_pts );
 830         InsertSilence( w, start - sync->next_pts );
 831         *buf_out = buf;
 832         return HB_WORK_OK;
 833     }
 834
 835     /*
 836      * When we get here we've taken care of all the dups and gaps in the
 837      * audio stream and are ready to inject the next input frame into
 838      * the output stream.
 839      */
 840     *buf_out = OutputAudioFrame( w->audio, buf, sync );
 841     return HB_WORK_OK;
 842 }
 843
 844 hb_work_object_t hb_sync_audio =
 845 {
 846     WORK_SYNC_AUDIO,
 847     "AudioSynchronization",
 848     syncAudioInit,
 849     syncAudioWork,
 850     syncAudioClose
 851 };
 852
 853 static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
 854 {
 855     hb_work_object_t  * w;
 856     hb_work_private_t * pv;
 857     hb_title_t        * title = job->title;
 858     hb_sync_audio_t   * sync;
 859
 860     pv = calloc( 1, sizeof( hb_work_private_t ) );
 861     sync = &pv->type.audio;
 862     pv->job    = job;
 863     pv->common = common;
 864     pv->common->ref++;
 865
 866     w = hb_get_work( WORK_SYNC_AUDIO );
 867     w->private_data = pv;
 868     w->audio = hb_list_item( title->list_audio, i );
 869     w->fifo_in = w->audio->priv.fifo_raw;
 870
 871     if( w->audio->config.out.codec == HB_ACODEC_AC3 ||
 872         w->audio->config.out.codec == HB_ACODEC_DCA )
 873     {
 874         w->fifo_out = w->audio->priv.fifo_out;
 875     }
 876     else
 877     {
 878         w->fifo_out = w->audio->priv.fifo_sync;
 879     }
 880
 881     if( w->audio->config.out.codec == HB_ACODEC_AC3 )
 882     {
 883         /* Have a silent AC-3 frame ready in case we have to fill a
 884            gap */
 885         AVCodec        * codec;
 886         AVCodecContext * c;
 887         short          * zeros;
 888
 889         codec = avcodec_find_encoder( CODEC_ID_AC3 );
 890         c     = avcodec_alloc_context();
 891
 892         c->bit_rate    = w->audio->config.in.bitrate;
 893         c->sample_rate = w->audio->config.in.samplerate;
 894         c->channels    = HB_INPUT_CH_LAYOUT_GET_DISCRETE_COUNT( w->audio->config.in.channel_layout );
 895
 896         if( hb_avcodec_open( c, codec ) < 0 )
 897         {
 898             hb_log( "sync: avcodec_open failed" );
 899             return;
 900         }
 901
 902         zeros          = calloc( AC3_SAMPLES_PER_FRAME *
 903                                  sizeof( short ) * c->channels, 1 );
 904         sync->ac3_size = w->audio->config.in.bitrate * AC3_SAMPLES_PER_FRAME /
 905                              w->audio->config.in.samplerate / 8;
 906         sync->ac3_buf  = malloc( sync->ac3_size );
 907
 908         if( avcodec_encode_audio( c, sync->ac3_buf, sync->ac3_size,
 909                                   zeros ) != sync->ac3_size )
 910         {
 911             hb_log( "sync: avcodec_encode_audio failed" );
 912         }
 913
 914         free( zeros );
 915         hb_avcodec_close( c );
 916         av_free( c );
 917     }
 918     else
 919     {
 920         /* Initialize libsamplerate */
 921         int error;
 922         sync->state = src_new( SRC_SINC_MEDIUM_QUALITY,
 923             HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
 924                 w->audio->config.out.mixdown), &error );
 925         sync->data.end_of_input = 0;
 926     }
 927     hb_list_add( job->list_work, w );
 928 }
 929
 930 static hb_buffer_t * OutputAudioFrame( hb_audio_t *audio, hb_buffer_t *buf,
 931                                        hb_sync_audio_t *sync )
 932 {
 933     int64_t start = sync->next_start;
 934     int64_t duration = buf->stop - buf->start;
 935
 936     sync->next_pts += duration;
 937
 938     if( audio->config.in.samplerate == audio->config.out.samplerate ||
 939         audio->config.out.codec == HB_ACODEC_AC3 ||
 940         audio->config.out.codec == HB_ACODEC_DCA )
 941     {
 942         /*
 943          * If we don't have to do sample rate conversion or this audio is
 944          * pass-thru just send the input buffer downstream after adjusting
 945          * its timestamps to make the output stream continuous.
 946          */
 947     }
 948     else
 949     {
 950         /* Not pass-thru - do sample rate conversion */
 951         int count_in, count_out;
 952         hb_buffer_t * buf_raw = buf;
 953         int channel_count = HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio->config.out.mixdown) *
 954                             sizeof( float );
 955
 956         count_in  = buf_raw->size / channel_count;
 957         /*
 958          * When using stupid rates like 44.1 there will always be some
 959          * truncation error. E.g., a 1536 sample AC3 frame will turn into a
 960          * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2
 961          * the error will build up over time and eventually the audio will
 962          * substantially lag the video. libsamplerate will keep track of the
 963          * fractional sample & give it to us when appropriate if we give it
 964          * an extra sample of space in the output buffer.
 965          */
 966         count_out = ( duration * audio->config.out.samplerate ) / 90000 + 1;
 967
 968         sync->data.input_frames = count_in;
 969         sync->data.output_frames = count_out;
 970         sync->data.src_ratio = (double)audio->config.out.samplerate /
 971                                (double)audio->config.in.samplerate;
 972
 973         buf = hb_buffer_init( count_out * channel_count );
 974         sync->data.data_in  = (float *) buf_raw->data;
 975         sync->data.data_out = (float *) buf->data;
 976         if( src_process( sync->state, &sync->data ) )
 977         {
 978             /* XXX If this happens, we're screwed */
 979             hb_log( "sync: audio %d src_process failed", audio->id );
 980         }
 981         hb_buffer_close( &buf_raw );
 982
 983         buf->size = sync->data.output_frames_gen * channel_count;
 984         duration = ( sync->data.output_frames_gen * 90000 ) /
 985                    audio->config.out.samplerate;
 986     }
 987     buf->frametype = HB_FRAME_AUDIO;
 988     buf->start = start;
 989     buf->stop  = start + duration;
 990     sync->next_start = start + duration;
 991     return buf;
 992 }
 993
 994 static void InsertSilence( hb_work_object_t * w, int64_t duration )
 995 {
 996     hb_work_private_t * pv = w->private_data;
 997     hb_sync_audio_t *sync = &pv->type.audio;
 998     hb_buffer_t     *buf;
 999     hb_fifo_t       *fifo;
1000
1001     // to keep pass-thru and regular audio in sync we generate silence in
1002     // AC3 frame-sized units. If the silence duration isn't an integer multiple
1003     // of the AC3 frame duration we will truncate or round up depending on
1004     // which minimizes the timing error.
1005     const int frame_dur = ( 90000 * AC3_SAMPLES_PER_FRAME ) /
1006                           w->audio->config.in.samplerate;
1007     int frame_count = ( duration + (frame_dur >> 1) ) / frame_dur;
1008
1009     while ( --frame_count >= 0 )
1010     {
1011         if( w->audio->config.out.codec == HB_ACODEC_AC3 )
1012         {
1013             buf        = hb_buffer_init( sync->ac3_size );
1014             buf->start = sync->next_pts;
1015             buf->stop  = buf->start + frame_dur;
1016             memcpy( buf->data, sync->ac3_buf, buf->size );
1017             fifo = w->audio->priv.fifo_out;
1018         }
1019         else
1020         {
1021             buf = hb_buffer_init( AC3_SAMPLES_PER_FRAME * sizeof( float ) *
1022                                      HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(
1023                                          w->audio->config.out.mixdown) );
1024             buf->start = sync->next_pts;
1025             buf->stop  = buf->start + frame_dur;
1026             memset( buf->data, 0, buf->size );
1027             fifo = w->audio->priv.fifo_sync;
1028         }
1029         buf = OutputAudioFrame( w->audio, buf, sync );
1030         hb_fifo_push( fifo, buf );
1031     }
1032 }
1033
1034 static void UpdateState( hb_work_object_t * w )
1035 {
1036     hb_work_private_t * pv = w->private_data;
1037     hb_sync_video_t   * sync = &pv->type.video;
1038     hb_state_t state;
1039
1040     if( !pv->common->count_frames )
1041     {
1042         sync->st_first = hb_get_date();
1043         pv->job->st_pause_date = -1;
1044         pv->job->st_paused = 0;
1045     }
1046     pv->common->count_frames++;
1047
1048     if( hb_get_date() > sync->st_dates[3] + 1000 )
1049     {
1050         memmove( &sync->st_dates[0], &sync->st_dates[1],
1051                  3 * sizeof( uint64_t ) );
1052         memmove( &sync->st_counts[0], &sync->st_counts[1],
1053                  3 * sizeof( uint64_t ) );
1054         sync->st_dates[3]  = hb_get_date();
1055         sync->st_counts[3] = pv->common->count_frames;
1056     }
1057
1058 #define p state.param.working
1059     state.state = HB_STATE_WORKING;
1060     p.progress  = (float) pv->common->count_frames / (float) sync->count_frames_max;
1061     if( p.progress > 1.0 )
1062     {
1063         p.progress = 1.0;
1064     }
1065     p.rate_cur   = 1000.0 *
1066         (float) ( sync->st_counts[3] - sync->st_counts[0] ) /
1067         (float) ( sync->st_dates[3] - sync->st_dates[0] );
1068     if( hb_get_date() > sync->st_first + 4000 )
1069     {
1070         int eta;
1071         p.rate_avg = 1000.0 * (float) sync->st_counts[3] /
1072             (float) ( sync->st_dates[3] - sync->st_first - pv->job->st_paused);
1073         eta = (float) ( sync->count_frames_max - sync->st_counts[3] ) /
1074             p.rate_avg;
1075         p.hours   = eta / 3600;
1076         p.minutes = ( eta % 3600 ) / 60;
1077         p.seconds = eta % 60;
1078     }
1079     else
1080     {
1081         p.rate_avg = 0.0;
1082         p.hours    = -1;
1083         p.minutes  = -1;
1084         p.seconds  = -1;
1085     }
1086 #undef p
1087
1088     hb_set_state( pv->job->h, &state );
1089 }