[video_player] Improve macOS frame management (#5078)

Fixes some logic around getting new frames on macOS:
- The logic to copy a frame had a race condition; the macOS code checks that a frame is available for the current playback time before informing the engine that a frame is available (it's not clear why the iOS code doesn't do this; apparently the engine tolerates returning NULL frames on iOS?), but failed to account for the fact that the current playback time when the (async) request from the engine comes in can be different. This fixes it to remember the last time that was known to be available, and uses that when the current time isn't available. This fixes flickering on playback (since returning NULL on macOS causes the video to vanish until a new frame is available).
- Fixes seek to temporarily restart the display link if the video is paused, rather than telling the engine that a frame is available, because it might not be. This is changed for both macOS and iOS since I don't see any reason this bug couldn't affect iOS as well (although in practice I'm only aware of it being reproducible on macOS).

This extracts the display link code for macOS and iOS into an abstraction, eliminating most of the ifdefing, in order to support the latter (since more code needs to be able to play/pause the display link), which also resolves a TODO from the initial implementation.

There is also some non-trivial addition of factory injection in order to make the code more testable. This is definitely not complete, but it incrementally moves the code toward being more testable than it was before, and allows for testing the display link behavior.

Lastly, this moves some code used by tests to the existing `_Test.h` header, removing redeclarations from unit test files, since we already have a test header and that's our preferred approach for accessing private details in ObjC tests. (Longer term the multi-class mega-file should be broken up more to reduce the need for that.)

Fixes https://github.com/flutter/flutter/issues/136027
Improves https://github.com/flutter/flutter/issues/135999
This commit is contained in:
stuartmorgan
2023-11-16 08:19:15 -08:00
committed by GitHub
parent 52a5342f02
commit 25574f996e
11 changed files with 547 additions and 166 deletions

View File

@ -1,3 +1,7 @@
## 2.5.2
* Fixes flickering and seek-while-paused on macOS.
## 2.5.1
* Updates to Pigeon 13.

View File

@ -0,0 +1,34 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import <Foundation/Foundation.h>
#if TARGET_OS_OSX
#import <FlutterMacOS/FlutterMacOS.h>
#else
#import <Flutter/Flutter.h>
#endif
// A cross-platform display link abstraction.
@interface FVPDisplayLink : NSObject
/**
* Whether the display link is currently running (i.e., firing events).
*
* Defaults to NO.
*/
@property(nonatomic, assign) BOOL running;
/**
* Initializes a display link that calls the given callback when fired.
*
* The display link starts paused, so must be started, by setting 'running' to YES, before the
* callback will fire.
*/
- (instancetype)initWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback NS_DESIGNATED_INITIALIZER;
- (instancetype)init NS_UNAVAILABLE;
@end

View File

@ -9,6 +9,7 @@
#import <GLKit/GLKit.h>
#import "AVAssetTrackUtils.h"
#import "FVPDisplayLink.h"
#import "messages.g.h"
#if !__has_feature(objc_arc)
@ -20,9 +21,15 @@
@property(nonatomic, weak, readonly) NSObject<FlutterTextureRegistry> *registry;
// The output that this updater is managing.
@property(nonatomic, weak) AVPlayerItemVideoOutput *videoOutput;
#if TARGET_OS_IOS
- (void)onDisplayLink:(CADisplayLink *)link;
#endif
// The last time that has been validated as avaliable according to hasNewPixelBufferForItemTime:.
@property(nonatomic, assign) CMTime lastKnownAvailableTime;
// If YES, the engine is informed that a new texture is available any time the display link
// callback is fired, regardless of the videoOutput state.
//
// TODO(stuartmorgan): Investigate removing this; it exists only to preserve existing iOS behavior
// while implementing macOS, but iOS should very likely be doing the check as well. See
// https://github.com/flutter/flutter/issues/138427.
@property(nonatomic, assign) BOOL skipBufferAvailabilityCheck;
@end
@implementation FVPFrameUpdater
@ -30,56 +37,57 @@
NSAssert(self, @"super init cannot be nil");
if (self == nil) return nil;
_registry = registry;
_lastKnownAvailableTime = kCMTimeInvalid;
return self;
}
#if TARGET_OS_IOS
- (void)onDisplayLink:(CADisplayLink *)link {
// TODO(stuartmorgan): Investigate switching this to displayLinkFired; iOS may also benefit from
// the availability check there.
[_registry textureFrameAvailable:_textureId];
}
#endif
- (void)displayLinkFired {
// Only report a new frame if one is actually available.
// Only report a new frame if one is actually available, or the check is being skipped.
BOOL reportFrame = NO;
if (self.skipBufferAvailabilityCheck) {
reportFrame = YES;
} else {
CMTime outputItemTime = [self.videoOutput itemTimeForHostTime:CACurrentMediaTime()];
if ([self.videoOutput hasNewPixelBufferForItemTime:outputItemTime]) {
_lastKnownAvailableTime = outputItemTime;
reportFrame = YES;
}
}
if (reportFrame) {
[_registry textureFrameAvailable:_textureId];
}
}
@end
#if TARGET_OS_OSX
static CVReturn DisplayLinkCallback(CVDisplayLinkRef displayLink, const CVTimeStamp *now,
const CVTimeStamp *outputTime, CVOptionFlags flagsIn,
CVOptionFlags *flagsOut, void *displayLinkSource) {
// Trigger the main-thread dispatch queue, to drive a frame update check.
__weak dispatch_source_t source = (__bridge dispatch_source_t)displayLinkSource;
dispatch_source_merge_data(source, 1);
return kCVReturnSuccess;
}
#endif
@interface FVPDefaultPlayerFactory : NSObject <FVPPlayerFactory>
@interface FVPDefaultAVFactory : NSObject <FVPAVFactory>
@end
@implementation FVPDefaultPlayerFactory
@implementation FVPDefaultAVFactory
- (AVPlayer *)playerWithPlayerItem:(AVPlayerItem *)playerItem {
return [AVPlayer playerWithPlayerItem:playerItem];
}
- (AVPlayerItemVideoOutput *)videoOutputWithPixelBufferAttributes:
(NSDictionary<NSString *, id> *)attributes {
return [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:attributes];
}
@end
/** Non-test implementation of the diplay link factory. */
@interface FVPDefaultDisplayLinkFactory : NSObject <FVPDisplayLinkFactory>
@end
@implementation FVPDefaultDisplayLinkFactory
- (FVPDisplayLink *)displayLinkWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback {
return [[FVPDisplayLink alloc] initWithRegistrar:registrar callback:callback];
}
@end
@interface FVPVideoPlayer : NSObject <FlutterTexture, FlutterStreamHandler>
@property(readonly, nonatomic) AVPlayer *player;
#pragma mark -
@interface FVPVideoPlayer ()
@property(readonly, nonatomic) AVPlayerItemVideoOutput *videoOutput;
// This is to fix 2 bugs: 1. blank video for encrypted video streams on iOS 16
// (https://github.com/flutter/flutter/issues/111457) and 2. swapped width and height for some video
// streams (not just iOS 16). (https://github.com/flutter/flutter/issues/109116).
// An invisible AVPlayerLayer is used to overwrite the protection of pixel buffers in those streams
// for issue #1, and restore the correct width and height for issue #2.
@property(readonly, nonatomic) AVPlayerLayer *playerLayer;
// The plugin registrar, to obtain view information from.
@property(nonatomic, weak) NSObject<FlutterPluginRegistrar> *registrar;
// The CALayer associated with the Flutter view this plugin is associated with, if any.
@ -91,21 +99,20 @@ static CVReturn DisplayLinkCallback(CVDisplayLinkRef displayLink, const CVTimeSt
@property(nonatomic, readonly) BOOL isPlaying;
@property(nonatomic) BOOL isLooping;
@property(nonatomic, readonly) BOOL isInitialized;
// TODO(stuartmorgan): Extract and abstract the display link to remove all the display-link-related
// ifdefs from this file.
#if TARGET_OS_OSX
// The display link to trigger frame reads from the video player.
@property(nonatomic, assign) CVDisplayLinkRef displayLink;
// A dispatch source to move display link callbacks to the main thread.
@property(nonatomic, strong) dispatch_source_t displayLinkSource;
#else
@property(nonatomic) CADisplayLink *displayLink;
#endif
// The updater that drives callbacks to the engine to indicate that a new frame is ready.
@property(nonatomic) FVPFrameUpdater *frameUpdater;
// The display link that drives frameUpdater.
@property(nonatomic) FVPDisplayLink *displayLink;
// Whether a new frame needs to be provided to the engine regardless of the current play/pause state
// (e.g., after a seek while paused). If YES, the display link should continue to run until the next
// frame is successfully provided.
@property(nonatomic, assign) BOOL waitingForFrame;
- (instancetype)initWithURL:(NSURL *)url
frameUpdater:(FVPFrameUpdater *)frameUpdater
displayLink:(FVPDisplayLink *)displayLink
httpHeaders:(nonnull NSDictionary<NSString *, NSString *> *)headers
playerFactory:(id<FVPPlayerFactory>)playerFactory
avFactory:(id<FVPAVFactory>)avFactory
registrar:(NSObject<FlutterPluginRegistrar> *)registrar;
@end
@ -119,7 +126,8 @@ static void *rateContext = &rateContext;
@implementation FVPVideoPlayer
- (instancetype)initWithAsset:(NSString *)asset
frameUpdater:(FVPFrameUpdater *)frameUpdater
playerFactory:(id<FVPPlayerFactory>)playerFactory
displayLink:(FVPDisplayLink *)displayLink
avFactory:(id<FVPAVFactory>)avFactory
registrar:(NSObject<FlutterPluginRegistrar> *)registrar {
NSString *path = [[NSBundle mainBundle] pathForResource:asset ofType:nil];
#if TARGET_OS_OSX
@ -131,8 +139,9 @@ static void *rateContext = &rateContext;
#endif
return [self initWithURL:[NSURL fileURLWithPath:path]
frameUpdater:frameUpdater
displayLink:displayLink
httpHeaders:@{}
playerFactory:playerFactory
avFactory:avFactory
registrar:registrar];
}
@ -243,40 +252,11 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
return videoComposition;
}
- (void)createVideoOutputAndDisplayLink:(FVPFrameUpdater *)frameUpdater {
NSDictionary *pixBuffAttributes = @{
(id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA),
(id)kCVPixelBufferIOSurfacePropertiesKey : @{}
};
_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:pixBuffAttributes];
#if TARGET_OS_OSX
frameUpdater.videoOutput = _videoOutput;
// Create and start the main-thread dispatch queue to drive frameUpdater.
self.displayLinkSource =
dispatch_source_create(DISPATCH_SOURCE_TYPE_DATA_ADD, 0, 0, dispatch_get_main_queue());
dispatch_source_set_event_handler(self.displayLinkSource, ^() {
@autoreleasepool {
[frameUpdater displayLinkFired];
}
});
dispatch_resume(self.displayLinkSource);
if (CVDisplayLinkCreateWithActiveCGDisplays(&_displayLink) == kCVReturnSuccess) {
CVDisplayLinkSetOutputCallback(_displayLink, &DisplayLinkCallback,
(__bridge void *)(self.displayLinkSource));
}
#else
_displayLink = [CADisplayLink displayLinkWithTarget:frameUpdater
selector:@selector(onDisplayLink:)];
[_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
_displayLink.paused = YES;
#endif
}
- (instancetype)initWithURL:(NSURL *)url
frameUpdater:(FVPFrameUpdater *)frameUpdater
displayLink:(FVPDisplayLink *)displayLink
httpHeaders:(nonnull NSDictionary<NSString *, NSString *> *)headers
playerFactory:(id<FVPPlayerFactory>)playerFactory
avFactory:(id<FVPAVFactory>)avFactory
registrar:(NSObject<FlutterPluginRegistrar> *)registrar {
NSDictionary<NSString *, id> *options = nil;
if ([headers count] != 0) {
@ -286,18 +266,21 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
AVPlayerItem *item = [AVPlayerItem playerItemWithAsset:urlAsset];
return [self initWithPlayerItem:item
frameUpdater:frameUpdater
playerFactory:playerFactory
displayLink:(FVPDisplayLink *)displayLink
avFactory:avFactory
registrar:registrar];
}
- (instancetype)initWithPlayerItem:(AVPlayerItem *)item
frameUpdater:(FVPFrameUpdater *)frameUpdater
playerFactory:(id<FVPPlayerFactory>)playerFactory
displayLink:(FVPDisplayLink *)displayLink
avFactory:(id<FVPAVFactory>)avFactory
registrar:(NSObject<FlutterPluginRegistrar> *)registrar {
self = [super init];
NSAssert(self, @"super init cannot be nil");
_registrar = registrar;
_frameUpdater = frameUpdater;
AVAsset *asset = [item asset];
void (^assetCompletionHandler)(void) = ^{
@ -328,7 +311,7 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
}
};
_player = [playerFactory playerWithPlayerItem:item];
_player = [avFactory playerWithPlayerItem:item];
_player.actionAtItemEnd = AVPlayerActionAtItemEndNone;
// This is to fix 2 bugs: 1. blank video for encrypted video streams on iOS 16
@ -339,7 +322,18 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
_playerLayer = [AVPlayerLayer playerLayerWithPlayer:_player];
[self.flutterViewLayer addSublayer:_playerLayer];
[self createVideoOutputAndDisplayLink:frameUpdater];
// Configure output.
_displayLink = displayLink;
NSDictionary *pixBuffAttributes = @{
(id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA),
(id)kCVPixelBufferIOSurfacePropertiesKey : @{}
};
_videoOutput = [avFactory videoOutputWithPixelBufferAttributes:pixBuffAttributes];
frameUpdater.videoOutput = _videoOutput;
#if TARGET_OS_IOS
// See TODO on this property in FVPFrameUpdater.
frameUpdater.skipBufferAvailabilityCheck = YES;
#endif
[self addObserversForItem:item player:_player];
@ -422,23 +416,7 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
} else {
[_player pause];
}
#if TARGET_OS_OSX
if (_displayLink) {
if (_isPlaying) {
NSScreen *screen = self.registrar.view.window.screen;
if (screen) {
CGDirectDisplayID viewDisplayID =
(CGDirectDisplayID)[screen.deviceDescription[@"NSScreenNumber"] unsignedIntegerValue];
CVDisplayLinkSetCurrentCGDisplay(_displayLink, viewDisplayID);
}
CVDisplayLinkStart(_displayLink);
} else {
CVDisplayLinkStop(_displayLink);
}
}
#else
_displayLink.paused = !_isPlaying;
#endif
_displayLink.running = _isPlaying;
}
- (void)setupEventSinkIfReadyToPlay {
@ -513,16 +491,32 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
}
- (void)seekTo:(int64_t)location completionHandler:(void (^)(BOOL))completionHandler {
CMTime locationCMT = CMTimeMake(location, 1000);
CMTime previousCMTime = _player.currentTime;
CMTime targetCMTime = CMTimeMake(location, 1000);
CMTimeValue duration = _player.currentItem.asset.duration.value;
// Without adding tolerance when seeking to duration,
// seekToTime will never complete, and this call will hang.
// see issue https://github.com/flutter/flutter/issues/124475.
CMTime tolerance = location == duration ? CMTimeMake(1, 1000) : kCMTimeZero;
[_player seekToTime:locationCMT
[_player seekToTime:targetCMTime
toleranceBefore:tolerance
toleranceAfter:tolerance
completionHandler:completionHandler];
completionHandler:^(BOOL completed) {
if (CMTimeCompare(self.player.currentTime, previousCMTime) != 0) {
// Ensure that a frame is drawn once available, even if currently paused. In theory a race
// is possible here where the new frame has already drawn by the time this code runs, and
// the display link stays on indefinitely, but that should be relatively harmless. This
// must use the display link rather than just informing the engine that a new frame is
// available because the seek completing doesn't guarantee that the pixel buffer is
// already available.
self.waitingForFrame = YES;
self.displayLink.running = YES;
}
if (completionHandler) {
completionHandler(completed);
}
}];
}
- (void)setIsLooping:(BOOL)isLooping {
@ -558,14 +552,31 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
}
- (CVPixelBufferRef)copyPixelBuffer {
CVPixelBufferRef buffer = NULL;
CMTime outputItemTime = [_videoOutput itemTimeForHostTime:CACurrentMediaTime()];
if ([_videoOutput hasNewPixelBufferForItemTime:outputItemTime]) {
return [_videoOutput copyPixelBufferForItemTime:outputItemTime itemTimeForDisplay:NULL];
buffer = [_videoOutput copyPixelBufferForItemTime:outputItemTime itemTimeForDisplay:NULL];
} else {
return NULL;
// If the current time isn't available yet, use the time that was checked when informing the
// engine that a frame was available (if any).
CMTime lastAvailableTime = self.frameUpdater.lastKnownAvailableTime;
if (CMTIME_IS_VALID(lastAvailableTime)) {
buffer = [_videoOutput copyPixelBufferForItemTime:lastAvailableTime itemTimeForDisplay:NULL];
}
}
if (self.waitingForFrame && buffer) {
self.waitingForFrame = NO;
// If the display link was only running temporarily to pick up a new frame while the video was
// paused, stop it again.
if (!self.isPlaying) {
self.displayLink.running = NO;
}
}
return buffer;
}
- (void)onTextureUnregistered:(NSObject<FlutterTexture> *)texture {
dispatch_async(dispatch_get_main_queue(), ^{
[self dispose];
@ -603,16 +614,7 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
_disposed = YES;
[_playerLayer removeFromSuperlayer];
#if TARGET_OS_OSX
if (_displayLink) {
CVDisplayLinkStop(_displayLink);
CVDisplayLinkRelease(_displayLink);
_displayLink = NULL;
}
dispatch_source_cancel(_displayLinkSource);
#else
[_displayLink invalidate];
#endif
_displayLink = nil;
[self removeKeyValueObservers];
[self.player replaceCurrentItemWithPlayerItem:nil];
@ -653,13 +655,12 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
@end
@interface FVPVideoPlayerPlugin () <FVPAVFoundationVideoPlayerApi>
@interface FVPVideoPlayerPlugin ()
@property(readonly, weak, nonatomic) NSObject<FlutterTextureRegistry> *registry;
@property(readonly, weak, nonatomic) NSObject<FlutterBinaryMessenger> *messenger;
@property(readonly, strong, nonatomic)
NSMutableDictionary<NSNumber *, FVPVideoPlayer *> *playersByTextureId;
@property(readonly, strong, nonatomic) NSObject<FlutterPluginRegistrar> *registrar;
@property(nonatomic, strong) id<FVPPlayerFactory> playerFactory;
@property(nonatomic, strong) id<FVPDisplayLinkFactory> displayLinkFactory;
@property(nonatomic, strong) id<FVPAVFactory> avFactory;
@end
@implementation FVPVideoPlayerPlugin
@ -674,17 +675,21 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
}
- (instancetype)initWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar {
return [self initWithPlayerFactory:[[FVPDefaultPlayerFactory alloc] init] registrar:registrar];
return [self initWithAVFactory:[[FVPDefaultAVFactory alloc] init]
displayLinkFactory:[[FVPDefaultDisplayLinkFactory alloc] init]
registrar:registrar];
}
- (instancetype)initWithPlayerFactory:(id<FVPPlayerFactory>)playerFactory
- (instancetype)initWithAVFactory:(id<FVPAVFactory>)avFactory
displayLinkFactory:(id<FVPDisplayLinkFactory>)displayLinkFactory
registrar:(NSObject<FlutterPluginRegistrar> *)registrar {
self = [super init];
NSAssert(self, @"super init cannot be nil");
_registry = [registrar textures];
_messenger = [registrar messenger];
_registrar = registrar;
_playerFactory = playerFactory;
_displayLinkFactory = displayLinkFactory ?: [[FVPDefaultDisplayLinkFactory alloc] init];
_avFactory = avFactory ?: [[FVPDefaultAVFactory alloc] init];
_playersByTextureId = [NSMutableDictionary dictionaryWithCapacity:1];
return self;
}
@ -729,6 +734,12 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
- (FVPTextureMessage *)create:(FVPCreateMessage *)input error:(FlutterError **)error {
FVPFrameUpdater *frameUpdater = [[FVPFrameUpdater alloc] initWithRegistry:_registry];
FVPDisplayLink *displayLink =
[self.displayLinkFactory displayLinkWithRegistrar:_registrar
callback:^() {
[frameUpdater displayLinkFired];
}];
FVPVideoPlayer *player;
if (input.asset) {
NSString *assetPath;
@ -740,7 +751,8 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
@try {
player = [[FVPVideoPlayer alloc] initWithAsset:assetPath
frameUpdater:frameUpdater
playerFactory:_playerFactory
displayLink:displayLink
avFactory:_avFactory
registrar:self.registrar];
return [self onPlayerSetup:player frameUpdater:frameUpdater];
} @catch (NSException *exception) {
@ -750,8 +762,9 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
} else if (input.uri) {
player = [[FVPVideoPlayer alloc] initWithURL:[NSURL URLWithString:input.uri]
frameUpdater:frameUpdater
displayLink:displayLink
httpHeaders:input.httpHeaders
playerFactory:_playerFactory
avFactory:_avFactory
registrar:self.registrar];
return [self onPlayerSetup:player frameUpdater:frameUpdater];
} else {
@ -816,7 +829,6 @@ NS_INLINE CGFloat radiansToDegrees(CGFloat radians) {
[player seekTo:input.position
completionHandler:^(BOOL finished) {
dispatch_async(dispatch_get_main_queue(), ^{
[self.registry textureFrameAvailable:input.textureId];
completion(nil);
});
}];

View File

@ -6,13 +6,48 @@
#import <AVFoundation/AVFoundation.h>
// Protocol for an AVPlayer instance factory. Used for injecting players in tests.
@protocol FVPPlayerFactory
#import "FVPDisplayLink.h"
#import "messages.g.h"
// Protocol for AVFoundation object instance factory. Used for injecting framework objects in tests.
@protocol FVPAVFactory
@required
- (AVPlayer *)playerWithPlayerItem:(AVPlayerItem *)playerItem;
- (AVPlayerItemVideoOutput *)videoOutputWithPixelBufferAttributes:
(NSDictionary<NSString *, id> *)attributes;
@end
@interface FVPVideoPlayerPlugin ()
// Protocol for an AVPlayer instance factory. Used for injecting display links in tests.
@protocol FVPDisplayLinkFactory
- (FVPDisplayLink *)displayLinkWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback;
@end
- (instancetype)initWithPlayerFactory:(id<FVPPlayerFactory>)playerFactory
#pragma mark -
// TODO(stuartmorgan): Move this whole class to its own files.
@interface FVPVideoPlayer : NSObject <FlutterStreamHandler, FlutterTexture>
@property(readonly, nonatomic) AVPlayer *player;
// This is to fix 2 bugs: 1. blank video for encrypted video streams on iOS 16
// (https://github.com/flutter/flutter/issues/111457) and 2. swapped width and height for some video
// streams (not just iOS 16). (https://github.com/flutter/flutter/issues/109116).
// An invisible AVPlayerLayer is used to overwrite the protection of pixel buffers in those streams
// for issue #1, and restore the correct width and height for issue #2.
@property(readonly, nonatomic) AVPlayerLayer *playerLayer;
@property(readonly, nonatomic) int64_t position;
- (void)onTextureUnregistered:(NSObject<FlutterTexture> *)texture;
@end
#pragma mark -
@interface FVPVideoPlayerPlugin () <FVPAVFoundationVideoPlayerApi>
@property(readonly, strong, nonatomic)
NSMutableDictionary<NSNumber *, FVPVideoPlayer *> *playersByTextureId;
- (instancetype)initWithAVFactory:(id<FVPAVFactory>)avFactory
displayLinkFactory:(id<FVPDisplayLinkFactory>)displayLinkFactory
registrar:(NSObject<FlutterPluginRegistrar> *)registrar;
@end

View File

@ -0,0 +1,72 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import "../FVPDisplayLink.h"
#import <Foundation/Foundation.h>
#import <QuartzCore/QuartzCore.h>
/**
* A proxy object to act as a CADisplayLink target, to avoid retain loops, since FVPDisplayLink
* owns its CADisplayLink, but CADisplayLink retains its target.
*/
@interface FVPDisplayLinkTarget : NSObject
@property(nonatomic) void (^callback)(void);
/** Initializes a target object that runs the given callback when onDisplayLink: is called. */
- (instancetype)initWithCallback:(void (^)(void))callback;
/** Method to be called when a CADisplayLink fires. */
- (void)onDisplayLink:(CADisplayLink *)link;
@end
@implementation FVPDisplayLinkTarget
- (instancetype)initWithCallback:(void (^)(void))callback {
self = [super init];
if (self) {
_callback = callback;
}
return self;
}
- (void)onDisplayLink:(CADisplayLink *)link {
self.callback();
}
@end
#pragma mark -
@interface FVPDisplayLink ()
// The underlying display link implementation.
@property(nonatomic) CADisplayLink *displayLink;
@property(nonatomic) FVPDisplayLinkTarget *target;
@end
@implementation FVPDisplayLink
- (instancetype)initWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback {
self = [super init];
if (self) {
_target = [[FVPDisplayLinkTarget alloc] initWithCallback:callback];
_displayLink = [CADisplayLink displayLinkWithTarget:_target selector:@selector(onDisplayLink:)];
[_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
_displayLink.paused = YES;
}
return self;
}
- (void)dealloc {
[_displayLink invalidate];
}
- (BOOL)running {
return !self.displayLink.paused;
}
- (void)setRunning:(BOOL)running {
self.displayLink.paused = !running;
}
@end

View File

@ -0,0 +1,84 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import "../FVPDisplayLink.h"
#import <CoreVideo/CoreVideo.h>
#import <Foundation/Foundation.h>
@interface FVPDisplayLink ()
// The underlying display link implementation.
@property(nonatomic, assign) CVDisplayLinkRef displayLink;
// A dispatch source to move display link callbacks to the main thread.
@property(nonatomic, strong) dispatch_source_t displayLinkSource;
// The plugin registrar, to get screen information.
@property(nonatomic, weak) NSObject<FlutterPluginRegistrar> *registrar;
@end
static CVReturn DisplayLinkCallback(CVDisplayLinkRef displayLink, const CVTimeStamp *now,
const CVTimeStamp *outputTime, CVOptionFlags flagsIn,
CVOptionFlags *flagsOut, void *displayLinkSource) {
// Trigger the main-thread dispatch queue, to drive the callback there.
__weak dispatch_source_t source = (__bridge dispatch_source_t)displayLinkSource;
dispatch_source_merge_data(source, 1);
return kCVReturnSuccess;
}
@implementation FVPDisplayLink
- (instancetype)initWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback {
self = [super init];
if (self) {
_registrar = registrar;
// Create and start the main-thread dispatch queue to drive frameUpdater.
_displayLinkSource =
dispatch_source_create(DISPATCH_SOURCE_TYPE_DATA_ADD, 0, 0, dispatch_get_main_queue());
dispatch_source_set_event_handler(_displayLinkSource, ^() {
@autoreleasepool {
callback();
}
});
dispatch_resume(_displayLinkSource);
if (CVDisplayLinkCreateWithActiveCGDisplays(&_displayLink) == kCVReturnSuccess) {
CVDisplayLinkSetOutputCallback(_displayLink, &DisplayLinkCallback,
(__bridge void *)(_displayLinkSource));
}
}
return self;
}
- (void)dealloc {
CVDisplayLinkStop(_displayLink);
CVDisplayLinkRelease(_displayLink);
_displayLink = NULL;
dispatch_source_cancel(_displayLinkSource);
}
- (BOOL)running {
return CVDisplayLinkIsRunning(self.displayLink);
}
- (void)setRunning:(BOOL)running {
if (self.running == running) {
return;
}
if (running) {
// TODO(stuartmorgan): Move this to init + a screen change listener; this won't correctly
// handle windows being dragged to another screen until the next pause/play cycle. That will
// likely require new plugin registrar APIs.
NSScreen *screen = self.registrar.view.window.screen;
if (screen) {
CGDirectDisplayID viewDisplayID =
(CGDirectDisplayID)[screen.deviceDescription[@"NSScreenNumber"] unsignedIntegerValue];
CVDisplayLinkSetCurrentCGDisplay(self.displayLink, viewDisplayID);
}
CVDisplayLinkStart(self.displayLink);
} else {
CVDisplayLinkStop(self.displayLink);
}
}
@end

View File

@ -19,19 +19,6 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
#endif
}
@interface FVPVideoPlayer : NSObject <FlutterStreamHandler>
@property(readonly, nonatomic) AVPlayer *player;
@property(readonly, nonatomic) AVPlayerLayer *playerLayer;
@property(readonly, nonatomic) int64_t position;
- (void)onTextureUnregistered:(NSObject<FlutterTexture> *)texture;
@end
@interface FVPVideoPlayerPlugin (Test) <FVPAVFoundationVideoPlayerApi>
@property(readonly, strong, nonatomic)
NSMutableDictionary<NSNumber *, FVPVideoPlayer *> *playersByTextureId;
@end
#if TARGET_OS_IOS
@interface FakeAVAssetTrack : AVAssetTrack
@property(readonly, nonatomic) CGAffineTransform preferredTransform;
@ -78,6 +65,7 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
@interface StubAVPlayer : AVPlayer
@property(readonly, nonatomic) NSNumber *beforeTolerance;
@property(readonly, nonatomic) NSNumber *afterTolerance;
@property(readonly, assign) CMTime lastSeekTime;
@end
@implementation StubAVPlayer
@ -88,33 +76,87 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
completionHandler:(void (^)(BOOL finished))completionHandler {
_beforeTolerance = [NSNumber numberWithLong:toleranceBefore.value];
_afterTolerance = [NSNumber numberWithLong:toleranceAfter.value];
completionHandler(YES);
_lastSeekTime = time;
[super seekToTime:time
toleranceBefore:toleranceBefore
toleranceAfter:toleranceAfter
completionHandler:completionHandler];
}
@end
@interface StubFVPPlayerFactory : NSObject <FVPPlayerFactory>
@interface StubFVPAVFactory : NSObject <FVPAVFactory>
@property(nonatomic, strong) StubAVPlayer *stubAVPlayer;
@property(nonatomic, strong) AVPlayerItemVideoOutput *output;
- (instancetype)initWithPlayer:(StubAVPlayer *)stubAVPlayer;
- (instancetype)initWithPlayer:(StubAVPlayer *)stubAVPlayer
output:(AVPlayerItemVideoOutput *)output;
@end
@implementation StubFVPPlayerFactory
@implementation StubFVPAVFactory
- (instancetype)initWithPlayer:(StubAVPlayer *)stubAVPlayer {
// Creates a factory that returns the given items. Any items that are nil will instead return
// a real object just as the non-test implementation would.
- (instancetype)initWithPlayer:(StubAVPlayer *)stubAVPlayer
output:(AVPlayerItemVideoOutput *)output {
self = [super init];
_stubAVPlayer = stubAVPlayer;
_output = output;
return self;
}
- (AVPlayer *)playerWithPlayerItem:(AVPlayerItem *)playerItem {
return _stubAVPlayer;
return _stubAVPlayer ?: [AVPlayer playerWithPlayerItem:playerItem];
}
- (AVPlayerItemVideoOutput *)videoOutputWithPixelBufferAttributes:
(NSDictionary<NSString *, id> *)attributes {
return _output ?: [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:attributes];
}
@end
#pragma mark -
/** Test implementation of FVPDisplayLinkFactory that returns a provided display link nstance. */
@interface StubFVPDisplayLinkFactory : NSObject <FVPDisplayLinkFactory>
/** This display link to return. */
@property(nonatomic, strong) FVPDisplayLink *displayLink;
- (instancetype)initWithDisplayLink:(FVPDisplayLink *)displayLink;
@end
@implementation StubFVPDisplayLinkFactory
- (instancetype)initWithDisplayLink:(FVPDisplayLink *)displayLink {
self = [super init];
_displayLink = displayLink;
return self;
}
- (FVPDisplayLink *)displayLinkWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback {
return self.displayLink;
}
@end
/** Non-test implementation of the diplay link factory. */
@interface FVPDefaultDisplayLinkFactory : NSObject <FVPDisplayLinkFactory>
@end
@implementation FVPDefaultDisplayLinkFactory
- (FVPDisplayLink *)displayLinkWithRegistrar:(id<FlutterPluginRegistrar>)registrar
callback:(void (^)(void))callback {
return [[FVPDisplayLink alloc] initWithRegistrar:registrar callback:callback];
}
@end
#pragma mark -
@implementation VideoPlayerTests
- (void)testBlankVideoBugWithEncryptedVideoStreamAndInvertedAspectRatioBugForSomeVideoStream {
@ -148,15 +190,24 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
XCTAssertNotNil(player.playerLayer.superlayer, @"AVPlayerLayer should be added on screen.");
}
- (void)testSeekToInvokesTextureFrameAvailableOnTextureRegistry {
- (void)testSeekToWhilePausedStartsDisplayLinkTemporarily {
NSObject<FlutterTextureRegistry> *mockTextureRegistry =
OCMProtocolMock(@protocol(FlutterTextureRegistry));
NSObject<FlutterPluginRegistrar> *registrar =
[GetPluginRegistry() registrarForPlugin:@"SeekToInvokestextureFrameAvailable"];
[GetPluginRegistry() registrarForPlugin:@"SeekToWhilePausedStartsDisplayLinkTemporarily"];
NSObject<FlutterPluginRegistrar> *partialRegistrar = OCMPartialMock(registrar);
OCMStub([partialRegistrar textures]).andReturn(mockTextureRegistry);
FVPVideoPlayerPlugin *videoPlayerPlugin =
(FVPVideoPlayerPlugin *)[[FVPVideoPlayerPlugin alloc] initWithRegistrar:partialRegistrar];
FVPDisplayLink *mockDisplayLink =
OCMPartialMock([[FVPDisplayLink alloc] initWithRegistrar:registrar
callback:^(){
}]);
StubFVPDisplayLinkFactory *stubDisplayLinkFactory =
[[StubFVPDisplayLinkFactory alloc] initWithDisplayLink:mockDisplayLink];
AVPlayerItemVideoOutput *mockVideoOutput = OCMPartialMock([[AVPlayerItemVideoOutput alloc] init]);
FVPVideoPlayerPlugin *videoPlayerPlugin = [[FVPVideoPlayerPlugin alloc]
initWithAVFactory:[[StubFVPAVFactory alloc] initWithPlayer:nil output:mockVideoOutput]
displayLinkFactory:stubDisplayLinkFactory
registrar:partialRegistrar];
FlutterError *initalizationError;
[videoPlayerPlugin initialize:&initalizationError];
@ -171,6 +222,10 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
FVPTextureMessage *textureMessage = [videoPlayerPlugin create:create error:&createError];
NSInteger textureId = textureMessage.textureId;
// Ensure that the video playback is paused before seeking.
FlutterError *pauseError;
[videoPlayerPlugin pause:textureMessage error:&pauseError];
XCTestExpectation *initializedExpectation = [self expectationWithDescription:@"seekTo completes"];
FVPPositionMessage *message = [FVPPositionMessage makeWithTextureId:textureId position:1234];
[videoPlayerPlugin seekTo:message
@ -178,10 +233,89 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
[initializedExpectation fulfill];
}];
[self waitForExpectationsWithTimeout:30.0 handler:nil];
OCMVerify([mockTextureRegistry textureFrameAvailable:message.textureId]);
// Seeking to a new position should start the display link temporarily.
OCMVerify([mockDisplayLink setRunning:YES]);
FVPVideoPlayer *player = videoPlayerPlugin.playersByTextureId[@(textureId)];
XCTAssertEqual([player position], 1234);
// Simulate a buffer being available.
OCMStub([mockVideoOutput hasNewPixelBufferForItemTime:kCMTimeZero])
.ignoringNonObjectArgs()
.andReturn(YES);
// Any non-zero value is fine here since it won't actually be used, just NULL-checked.
CVPixelBufferRef fakeBufferRef = (CVPixelBufferRef)1;
OCMStub([mockVideoOutput copyPixelBufferForItemTime:kCMTimeZero itemTimeForDisplay:NULL])
.ignoringNonObjectArgs()
.andReturn(fakeBufferRef);
// Simulate a callback from the engine to request a new frame.
[player copyPixelBuffer];
// Since a frame was found, and the video is paused, the display link should be paused again.
OCMVerify([mockDisplayLink setRunning:NO]);
}
- (void)testSeekToWhilePlayingDoesNotStopDisplayLink {
NSObject<FlutterTextureRegistry> *mockTextureRegistry =
OCMProtocolMock(@protocol(FlutterTextureRegistry));
NSObject<FlutterPluginRegistrar> *registrar =
[GetPluginRegistry() registrarForPlugin:@"SeekToWhilePlayingDoesNotStopDisplayLink"];
NSObject<FlutterPluginRegistrar> *partialRegistrar = OCMPartialMock(registrar);
OCMStub([partialRegistrar textures]).andReturn(mockTextureRegistry);
FVPDisplayLink *mockDisplayLink =
OCMPartialMock([[FVPDisplayLink alloc] initWithRegistrar:registrar
callback:^(){
}]);
StubFVPDisplayLinkFactory *stubDisplayLinkFactory =
[[StubFVPDisplayLinkFactory alloc] initWithDisplayLink:mockDisplayLink];
AVPlayerItemVideoOutput *mockVideoOutput = OCMPartialMock([[AVPlayerItemVideoOutput alloc] init]);
FVPVideoPlayerPlugin *videoPlayerPlugin = [[FVPVideoPlayerPlugin alloc]
initWithAVFactory:[[StubFVPAVFactory alloc] initWithPlayer:nil output:mockVideoOutput]
displayLinkFactory:stubDisplayLinkFactory
registrar:partialRegistrar];
FlutterError *initalizationError;
[videoPlayerPlugin initialize:&initalizationError];
XCTAssertNil(initalizationError);
FVPCreateMessage *create = [FVPCreateMessage
makeWithAsset:nil
uri:@"https://flutter.github.io/assets-for-api-docs/assets/videos/hls/bee.m3u8"
packageName:nil
formatHint:nil
httpHeaders:@{}];
FlutterError *createError;
FVPTextureMessage *textureMessage = [videoPlayerPlugin create:create error:&createError];
NSInteger textureId = textureMessage.textureId;
// Ensure that the video is playing before seeking.
FlutterError *pauseError;
[videoPlayerPlugin play:textureMessage error:&pauseError];
XCTestExpectation *initializedExpectation = [self expectationWithDescription:@"seekTo completes"];
FVPPositionMessage *message = [FVPPositionMessage makeWithTextureId:textureId position:1234];
[videoPlayerPlugin seekTo:message
completion:^(FlutterError *_Nullable error) {
[initializedExpectation fulfill];
}];
[self waitForExpectationsWithTimeout:30.0 handler:nil];
OCMVerify([mockDisplayLink setRunning:YES]);
FVPVideoPlayer *player = videoPlayerPlugin.playersByTextureId[@(textureId)];
XCTAssertEqual([player position], 1234);
// Simulate a buffer being available.
OCMStub([mockVideoOutput hasNewPixelBufferForItemTime:kCMTimeZero])
.ignoringNonObjectArgs()
.andReturn(YES);
// Any non-zero value is fine here since it won't actually be used, just NULL-checked.
CVPixelBufferRef fakeBufferRef = (CVPixelBufferRef)1;
OCMStub([mockVideoOutput copyPixelBufferForItemTime:kCMTimeZero itemTimeForDisplay:NULL])
.ignoringNonObjectArgs()
.andReturn(fakeBufferRef);
// Simulate a callback from the engine to request a new frame.
[player copyPixelBuffer];
// Since the video was playing, the display link should not be paused after getting a buffer.
OCMVerify(never(), [mockDisplayLink setRunning:NO]);
}
- (void)testDeregistersFromPlayer {
@ -323,10 +457,12 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
[GetPluginRegistry() registrarForPlugin:@"TestSeekTolerance"];
StubAVPlayer *stubAVPlayer = [[StubAVPlayer alloc] init];
StubFVPPlayerFactory *stubFVPPlayerFactory =
[[StubFVPPlayerFactory alloc] initWithPlayer:stubAVPlayer];
StubFVPAVFactory *stubAVFactory = [[StubFVPAVFactory alloc] initWithPlayer:stubAVPlayer
output:nil];
FVPVideoPlayerPlugin *pluginWithMockAVPlayer =
[[FVPVideoPlayerPlugin alloc] initWithPlayerFactory:stubFVPPlayerFactory registrar:registrar];
[[FVPVideoPlayerPlugin alloc] initWithAVFactory:stubAVFactory
displayLinkFactory:nil
registrar:registrar];
FlutterError *initializationError;
[pluginWithMockAVPlayer initialize:&initializationError];
@ -360,10 +496,12 @@ NSObject<FlutterPluginRegistry> *GetPluginRegistry(void) {
[GetPluginRegistry() registrarForPlugin:@"TestSeekToEndTolerance"];
StubAVPlayer *stubAVPlayer = [[StubAVPlayer alloc] init];
StubFVPPlayerFactory *stubFVPPlayerFactory =
[[StubFVPPlayerFactory alloc] initWithPlayer:stubAVPlayer];
StubFVPAVFactory *stubAVFactory = [[StubFVPAVFactory alloc] initWithPlayer:stubAVPlayer
output:nil];
FVPVideoPlayerPlugin *pluginWithMockAVPlayer =
[[FVPVideoPlayerPlugin alloc] initWithPlayerFactory:stubFVPPlayerFactory registrar:registrar];
[[FVPVideoPlayerPlugin alloc] initWithAVFactory:stubAVFactory
displayLinkFactory:nil
registrar:registrar];
FlutterError *initializationError;
[pluginWithMockAVPlayer initialize:&initializationError];

View File

@ -14,7 +14,9 @@ Downloaded by pub (not CocoaPods).
s.author = { 'Flutter Dev Team' => 'flutter-dev@googlegroups.com' }
s.source = { :http => 'https://github.com/flutter/packages/tree/main/packages/video_player/video_player_avfoundation' }
s.documentation_url = 'https://pub.dev/packages/video_player'
s.source_files = 'Classes/**/*'
s.source_files = 'Classes/*'
s.ios.source_files = 'Classes/ios/*'
s.osx.source_files = 'Classes/macos/*'
s.public_header_files = 'Classes/**/*.h'
s.ios.dependency 'Flutter'
s.osx.dependency 'FlutterMacOS'

View File

@ -31,7 +31,7 @@ target 'Runner' do
flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__))
target 'RunnerTests' do
inherit! :search_paths
pod 'OCMock', '3.5'
pod 'OCMock', '3.9.1'
end
end

View File

@ -67,7 +67,7 @@
331C80D5294CF71000263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
333000ED22D3DE5D00554162 /* Warnings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Warnings.xcconfig; sourceTree = "<group>"; };
335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GeneratedPluginRegistrant.swift; sourceTree = "<group>"; };
33683FF02ABCAC94007305E4 /* VideoPlayerTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = VideoPlayerTests.m; path = ../../../darwin/RunnerTests/VideoPlayerTests.m; sourceTree = "<group>"; };
33683FF02ABCAC94007305E4 /* VideoPlayerTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = VideoPlayerTests.m; path = ../../darwin/RunnerTests/VideoPlayerTests.m; sourceTree = SOURCE_ROOT; };
33CC10ED2044A3C60003C045 /* video_player_avfoundation_example.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = video_player_avfoundation_example.app; sourceTree = BUILT_PRODUCTS_DIR; };
33CC10F02044A3C60003C045 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
33CC10F22044A3C60003C045 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Assets.xcassets; path = Runner/Assets.xcassets; sourceTree = "<group>"; };

View File

@ -2,7 +2,7 @@ name: video_player_avfoundation
description: iOS and macOS implementation of the video_player plugin.
repository: https://github.com/flutter/packages/tree/main/packages/video_player/video_player_avfoundation
issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+video_player%22
version: 2.5.1
version: 2.5.2
environment:
sdk: ">=3.1.0 <4.0.0"