[camera_avfoundation] handle interruptions and use single offset (#8982)

Handle interruptions of capture sessions. Stop receiving and writing video and audio samples during such interruptions to avoid synchronization issues between video and audio.

Handle asynchronous error notifications posted about capture sessions. This can happen for example when there is an incoming phone call and then the camera starts recording with audio. The camera controller enters an error state and can be recovered by disposing and recreating.

Use single offset for both video and audio as separate offsets can cause unsynchronized video with audio. Seems audio timestamps tend to be ignored except for the first sample (probably by the video encoder, maybe it is inherent to the video format used) so prefer audio samples for calculating offset to avoid desynchronization. Avoid errors when trying to append samples with timestamp less or equal to the previous sample by not appending them. Fix a bug when after adjusting offset the last sample time was not updated which could cause adding past offsets multiple times (this could happen for example if `videoIsDisconnected` is set to true too early after it was previously set to true).

- fixes https://github.com/flutter/flutter/issues/149978
- fixes https://github.com/flutter/flutter/issues/151253

## Pre-Review Checklist

[^1]: Regular contributors who have demonstrated familiarity with the repository guidelines only need to comment if the PR is not auto-exempted by repo tooling.
This commit is contained in:
misos1
2026-02-03 16:47:22 +01:00
committed by GitHub
parent 2662b91ee9
commit 5b1bea87e8
11 changed files with 295 additions and 76 deletions

View File

@@ -1,5 +1,6 @@
## NEXT
## 0.11.3+1
* Fixes delivering errors from onCameraError.
* Updates README to reflect currently supported OS versions for the latest
versions of the endorsed platform implementations.
* Applications built with older versions of Flutter will continue to

View File

@@ -174,7 +174,7 @@ class CameraValue {
}) {
return CameraValue(
isInitialized: isInitialized ?? this.isInitialized,
errorDescription: errorDescription,
errorDescription: errorDescription ?? this.errorDescription,
previewSize: previewSize ?? this.previewSize,
isRecordingVideo: isRecordingVideo ?? this.isRecordingVideo,
isTakingPicture: isTakingPicture ?? this.isTakingPicture,
@@ -352,6 +352,14 @@ class CameraController extends ValueNotifier<CameraValue> {
}),
);
_unawaited(
CameraPlatform.instance.onCameraError(_cameraId).first.then((
CameraErrorEvent event,
) {
value = value.copyWith(errorDescription: event.description);
}),
);
await CameraPlatform.instance.initializeCamera(
_cameraId,
imageFormatGroup: imageFormatGroup ?? ImageFormatGroup.unknown,

View File

@@ -4,7 +4,7 @@ description: A Flutter plugin for controlling the camera. Supports previewing
Dart.
repository: https://github.com/flutter/packages/tree/main/packages/camera/camera
issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
version: 0.11.3
version: 0.11.3+1
environment:
sdk: ^3.9.0

View File

@@ -1801,6 +1801,24 @@ void main() {
);
},
);
test('error from onCameraError is received', () async {
final cameraController = CameraController(
const CameraDescription(
name: 'cam',
lensDirection: CameraLensDirection.back,
sensorOrientation: 90,
),
ResolutionPreset.max,
);
await cameraController.initialize();
expect(cameraController.value.hasError, isTrue);
expect(
cameraController.value.errorDescription,
mockOnCameraErrorEvent.description,
);
});
});
}

View File

@@ -1,3 +1,8 @@
## 0.9.22+10
* Handles video and audio interruptions and errors by "pausing" and reporting errors.
* Uses a single time offset for both video and audio to avoid desync.
## 0.9.22+9
* Migrates `FLTSavePhotoDelegate`, `FLTWritableData`, and `FLTImageStreamHandler` classes to Swift.

View File

@@ -100,7 +100,10 @@ enum CameraTestUtils {
/// Creates a test sample buffer.
/// @return a test sample buffer.
static func createTestSampleBuffer() -> CMSampleBuffer {
static func createTestSampleBuffer(
timestamp: CMTime = .zero,
duration: CMTime = CMTimeMake(value: 1, timescale: 44100)
) -> CMSampleBuffer {
var pixelBuffer: CVPixelBuffer?
CVPixelBufferCreate(kCFAllocatorDefault, 100, 100, kCVPixelFormatType_32BGRA, nil, &pixelBuffer)
@@ -111,9 +114,9 @@ enum CameraTestUtils {
formatDescriptionOut: &formatDescription)
var timingInfo = CMSampleTimingInfo(
duration: CMTimeMake(value: 1, timescale: 44100),
presentationTimeStamp: CMTime.zero,
decodeTimeStamp: CMTime.invalid)
duration: duration,
presentationTimeStamp: timestamp,
decodeTimeStamp: .invalid)
var sampleBuffer: CMSampleBuffer?
CMSampleBufferCreateReadyWithImageBuffer(
@@ -128,22 +131,25 @@ enum CameraTestUtils {
/// Creates a test audio sample buffer.
/// @return a test audio sample buffer.
static func createTestAudioSampleBuffer() -> CMSampleBuffer {
static func createTestAudioSampleBuffer(
timestamp: CMTime = .zero,
duration: CMTime = CMTimeMake(value: 1, timescale: 44100)
) -> CMSampleBuffer {
var blockBuffer: CMBlockBuffer?
CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault,
memoryBlock: nil,
blockLength: 100,
blockLength: Int(duration.value),
blockAllocator: kCFAllocatorDefault,
customBlockSource: nil,
offsetToData: 0,
dataLength: 100,
dataLength: Int(duration.value),
flags: kCMBlockBufferAssureMemoryNowFlag,
blockBufferOut: &blockBuffer)
var formatDescription: CMFormatDescription?
var basicDescription = AudioStreamBasicDescription(
mSampleRate: 44100,
mSampleRate: Float64(duration.timescale),
mFormatID: kAudioFormatLinearPCM,
mFormatFlags: 0,
mBytesPerPacket: 1,
@@ -168,8 +174,8 @@ enum CameraTestUtils {
allocator: kCFAllocatorDefault,
dataBuffer: blockBuffer!,
formatDescription: formatDescription!,
sampleCount: 1,
presentationTimeStamp: .zero,
sampleCount: CMItemCount(duration.value),
presentationTimeStamp: timestamp,
packetDescriptions: nil,
sampleBufferOut: &sampleBuffer)

View File

@@ -28,6 +28,7 @@ final class MockCaptureSession: NSObject, CaptureSession {
private(set) var addedAudioOutputCount: Int = 0
var automaticallyConfiguresApplicationAudioSession = false
var isRunning = true
var sessionPreset: AVCaptureSession.Preset {
get {

View File

@@ -359,4 +359,156 @@ final class CameraSampleBufferTests: XCTestCase {
AVAudioSession.sharedInstance().category == .playAndRecord,
"Category should be PlayAndRecord.")
}
func testDidOutputSampleBufferMustUseSingleOffsetForVideoAndAudio() {
let (camera, writerMock, adaptorMock, inputMock) = createCamera()
let testVideoConnection = CameraTestUtils.createTestConnection(
camera.captureVideoOutput.avOutput)
let testAudioOutput = CameraTestUtils.createTestAudioOutput()
let testAudioConnection = CameraTestUtils.createTestConnection(testAudioOutput)
var status = AVAssetWriter.Status.unknown
writerMock.startWritingStub = {
status = .writing
return true
}
writerMock.statusStub = {
return status
}
var appendedTime = CMTime.invalid
adaptorMock.appendStub = { buffer, time in
appendedTime = time
return true
}
inputMock.isReadyForMoreMediaData = true
inputMock.appendStub = { buffer in
appendedTime = CMSampleBufferGetPresentationTimeStamp(buffer)
return true
}
camera.startVideoRecording(completion: { error in }, messengerForStreaming: nil)
let appendVideoSample = { (time: Int64) in
camera.captureOutput(
camera.captureVideoOutput.avOutput,
didOutput: CameraTestUtils.createTestSampleBuffer(
timestamp: CMTimeMake(value: time, timescale: 1),
duration: .invalid),
from: testVideoConnection)
}
let appendAudioSample = { (time: Int64, duration: Int64) in
camera.captureOutput(
testAudioOutput,
didOutput: CameraTestUtils.createTestAudioSampleBuffer(
timestamp: CMTimeMake(value: time, timescale: 1),
duration: CMTimeMake(value: duration, timescale: 1)),
from: testAudioConnection)
}
appendedTime = .invalid
camera.pauseVideoRecording()
camera.resumeVideoRecording()
appendVideoSample(1)
XCTAssertEqual(appendedTime, CMTimeMake(value: 1, timescale: 1))
appendedTime = .invalid
camera.pauseVideoRecording()
camera.resumeVideoRecording()
appendVideoSample(11)
XCTAssertEqual(appendedTime, .invalid)
appendVideoSample(12)
XCTAssertEqual(appendedTime, CMTimeMake(value: 2, timescale: 1))
appendedTime = .invalid
camera.pauseVideoRecording()
camera.resumeVideoRecording()
appendAudioSample(20, 2)
XCTAssertEqual(appendedTime, .invalid)
appendVideoSample(23)
XCTAssertEqual(appendedTime, CMTimeMake(value: 3, timescale: 1))
appendedTime = .invalid
camera.pauseVideoRecording()
camera.resumeVideoRecording()
appendVideoSample(28)
XCTAssertEqual(appendedTime, .invalid)
appendAudioSample(30, 2)
XCTAssertEqual(appendedTime, .invalid)
appendVideoSample(33)
XCTAssertEqual(appendedTime, .invalid)
appendAudioSample(32, 2)
XCTAssertEqual(appendedTime, CMTimeMake(value: 2, timescale: 1))
}
func testDidOutputSampleBufferMustConnectVideoAfterSessionInterruption() {
let (camera, writerMock, adaptorMock, inputMock) = createCamera()
let testVideoConnection = CameraTestUtils.createTestConnection(
camera.captureVideoOutput.avOutput)
let testAudioOutput = CameraTestUtils.createTestAudioOutput()
let testAudioConnection = CameraTestUtils.createTestConnection(testAudioOutput)
var status = AVAssetWriter.Status.unknown
writerMock.startWritingStub = {
status = .writing
return true
}
writerMock.statusStub = {
return status
}
var appendedTime = CMTime.invalid
adaptorMock.appendStub = { buffer, time in
appendedTime = time
return true
}
inputMock.isReadyForMoreMediaData = true
inputMock.appendStub = { buffer in
appendedTime = CMSampleBufferGetPresentationTimeStamp(buffer)
return true
}
camera.startVideoRecording(completion: { error in }, messengerForStreaming: nil)
let appendVideoSample = { (time: Int64) in
camera.captureOutput(
camera.captureVideoOutput.avOutput,
didOutput: CameraTestUtils.createTestSampleBuffer(
timestamp: CMTimeMake(value: time, timescale: 1),
duration: .invalid),
from: testVideoConnection)
}
let appendAudioSample = { (time: Int64, duration: Int64) in
camera.captureOutput(
testAudioOutput,
didOutput: CameraTestUtils.createTestAudioSampleBuffer(
timestamp: CMTimeMake(value: time, timescale: 1),
duration: CMTimeMake(value: duration, timescale: 1)),
from: testAudioConnection)
}
appendVideoSample(1)
appendAudioSample(1, 1)
NotificationCenter.default.post(
name: AVCaptureSession.wasInterruptedNotification,
object: camera.audioCaptureSession)
appendedTime = .invalid
appendAudioSample(11, 1)
XCTAssertEqual(appendedTime, .invalid)
appendVideoSample(12)
XCTAssertEqual(appendedTime, CMTimeMake(value: 2, timescale: 1))
appendedTime = .invalid
appendAudioSample(12, 1)
XCTAssertEqual(appendedTime, CMTimeMake(value: 2, timescale: 1))
}
}

View File

@@ -16,6 +16,7 @@ protocol CaptureSession: NSObjectProtocol {
var inputs: [AVCaptureInput] { get }
var outputs: [AVCaptureOutput] { get }
var automaticallyConfiguresApplicationAudioSession: Bool { get set }
var isRunning: Bool { get }
func beginConfiguration()
func commitConfiguration()

View File

@@ -45,8 +45,8 @@ final class DefaultCamera: NSObject, Camera {
private let mediaSettings: FCPPlatformMediaSettings
private let mediaSettingsAVWrapper: FLTCamMediaSettingsAVWrapper
private let videoCaptureSession: CaptureSession
private let audioCaptureSession: CaptureSession
let videoCaptureSession: CaptureSession
let audioCaptureSession: CaptureSession
/// A wrapper for AVCaptureDevice creation to allow for dependency injection in tests.
private let videoCaptureDeviceFactory: VideoCaptureDeviceFactory
@@ -100,13 +100,17 @@ final class DefaultCamera: NSObject, Camera {
private var isRecording = false
private var isRecordingPaused = false
private var isFirstVideoSample = false
private var videoIsDisconnected = false
private var audioIsDisconnected = false
private var isAudioSetup = false
private var lastVideoSampleTime = CMTime.zero
private var lastAudioSampleTime = CMTime.zero
private var videoTimeOffset = CMTime.zero
private var audioTimeOffset = CMTime.zero
/// Time of the end of the last sample.
private var lastSampleEndTime = CMTime.invalid
/// Whether the recording is disconnected.
private var isRecordingDisconnected = false
/// Represents sum of all pauses/interruptions during recording.
private var recordingTimeOffset = CMTime.invalid
/// Output to use for adjusting of recording time offset.
private var outputForOffsetAdjusting: AVCaptureOutput?
/// Time of the last appended video sample.
private var lastAppendedVideoSampleTime = CMTime.invalid
/// True when images from the camera are being streamed.
private(set) var isStreamingImages = false
@@ -233,6 +237,33 @@ final class DefaultCamera: NSObject, Camera {
}
updateOrientation()
// Handle video and audio interruptions and errors. Interruption can happen for example by
// an incoming call during video recording. Error can happen for example when recording starts
// during an incoming call.
// https://github.com/flutter/flutter/issues/151253
for session in [videoCaptureSession, audioCaptureSession] {
NotificationCenter.default.addObserver(
self,
selector: #selector(captureSessionWasInterrupted),
name: AVCaptureSession.wasInterruptedNotification,
object: session)
NotificationCenter.default.addObserver(
self,
selector: #selector(captureSessionRuntimeError),
name: AVCaptureSession.runtimeErrorNotification,
object: session)
}
}
@objc private func captureSessionWasInterrupted(notification: NSNotification) {
isRecordingDisconnected = true
}
@objc private func captureSessionRuntimeError(notification: NSNotification) {
reportErrorMessage(
"\(String(describing: notification.userInfo?[AVCaptureSessionErrorKey] as? Error))")
}
// Possible values for presets are hard-coded in FLT interface having
@@ -516,10 +547,10 @@ final class DefaultCamera: NSObject, Camera {
isFirstVideoSample = true
isRecording = true
isRecordingPaused = false
videoTimeOffset = CMTime.zero
audioTimeOffset = CMTime.zero
videoIsDisconnected = false
audioIsDisconnected = false
isRecordingDisconnected = false
recordingTimeOffset = CMTime.zero
outputForOffsetAdjusting = captureVideoOutput.avOutput
lastAppendedVideoSampleTime = CMTime.negativeInfinity
completion(nil)
}
@@ -608,8 +639,7 @@ final class DefaultCamera: NSObject, Camera {
func pauseVideoRecording() {
isRecordingPaused = true
videoIsDisconnected = true
audioIsDisconnected = true
isRecordingDisconnected = true
}
func resumeVideoRecording() {
@@ -1178,75 +1208,74 @@ final class DefaultCamera: NSObject, Camera {
handleSampleBufferStreaming(sampleBuffer)
if isRecording && !isRecordingPaused {
if isRecording && !isRecordingPaused && videoCaptureSession.isRunning
&& audioCaptureSession.isRunning
{
if videoWriter?.status == .failed, let error = videoWriter?.error {
reportErrorMessage("\(error)")
return
}
// ignore audio samples until the first video sample arrives to avoid black frames
// https://github.com/flutter/flutter/issues/57831
if isFirstVideoSample && output != captureVideoOutput.avOutput {
return
// do not append sample buffer when readyForMoreMediaData is NO to avoid crash
// https://github.com/flutter/flutter/issues/132073
if output == captureVideoOutput.avOutput {
if !(videoWriterInput?.isReadyForMoreMediaData ?? false) {
return
}
} else {
// ignore audio samples until the first video sample arrives to avoid black frames
// https://github.com/flutter/flutter/issues/57831
if isFirstVideoSample || !(audioWriterInput?.isReadyForMoreMediaData ?? false) {
return
}
outputForOffsetAdjusting = output
}
var currentSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
let sampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
if isFirstVideoSample {
videoWriter?.startSession(atSourceTime: currentSampleTime)
videoWriter?.startSession(atSourceTime: sampleTime)
// fix sample times not being numeric when pause/resume happens before first sample buffer
// arrives
// https://github.com/flutter/flutter/issues/132014
lastVideoSampleTime = currentSampleTime
lastAudioSampleTime = currentSampleTime
isRecordingDisconnected = false
isFirstVideoSample = false
}
if output == captureVideoOutput.avOutput {
if videoIsDisconnected {
videoIsDisconnected = false
var currentSampleEndTime = sampleTime
let duration = CMSampleBufferGetDuration(sampleBuffer)
if CMTIME_IS_NUMERIC(duration) {
currentSampleEndTime = CMTimeAdd(currentSampleEndTime, duration)
}
videoTimeOffset =
videoTimeOffset.value == 0
? CMTimeSubtract(currentSampleTime, lastVideoSampleTime)
: CMTimeAdd(videoTimeOffset, CMTimeSubtract(currentSampleTime, lastVideoSampleTime))
return
// Use a single time offset for both video and audio to avoid desync.
// https://github.com/flutter/flutter/issues/149978
if isRecordingDisconnected {
if output == outputForOffsetAdjusting {
let offset = CMTimeSubtract(currentSampleEndTime, lastSampleEndTime)
recordingTimeOffset = CMTimeAdd(recordingTimeOffset, offset)
lastSampleEndTime = currentSampleEndTime
isRecordingDisconnected = false
}
return
}
lastVideoSampleTime = currentSampleTime
if output == outputForOffsetAdjusting {
lastSampleEndTime = currentSampleEndTime
}
if output == captureVideoOutput.avOutput {
let nextBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
let nextSampleTime = CMTimeSubtract(lastVideoSampleTime, videoTimeOffset)
// do not append sample buffer when readyForMoreMediaData is NO to avoid crash
// https://github.com/flutter/flutter/issues/132073
if videoWriterInput?.isReadyForMoreMediaData ?? false {
let nextSampleTime = CMTimeSubtract(sampleTime, recordingTimeOffset)
if nextSampleTime > lastAppendedVideoSampleTime {
let _ = videoAdaptor?.append(nextBuffer!, withPresentationTime: nextSampleTime)
lastAppendedVideoSampleTime = nextSampleTime
}
} else {
let dur = CMSampleBufferGetDuration(sampleBuffer)
if dur.value > 0 {
currentSampleTime = CMTimeAdd(currentSampleTime, dur)
}
if audioIsDisconnected {
audioIsDisconnected = false
audioTimeOffset =
audioTimeOffset.value == 0
? CMTimeSubtract(currentSampleTime, lastAudioSampleTime)
: CMTimeAdd(audioTimeOffset, CMTimeSubtract(currentSampleTime, lastAudioSampleTime))
return
}
lastAudioSampleTime = currentSampleTime
if audioTimeOffset.value != 0 {
if recordingTimeOffset.value != 0 {
if let adjustedSampleBuffer = copySampleBufferWithAdjustedTime(
sampleBuffer,
by: audioTimeOffset)
by: recordingTimeOffset)
{
newAudioSample(adjustedSampleBuffer)
}
@@ -1370,10 +1399,8 @@ final class DefaultCamera: NSObject, Camera {
}
return
}
if audioWriterInput?.isReadyForMoreMediaData ?? false {
if !(audioWriterInput?.append(sampleBuffer) ?? false) {
reportErrorMessage("Unable to write to audio input")
}
if !(audioWriterInput?.append(sampleBuffer) ?? false) {
reportErrorMessage("Unable to write to audio input")
}
}

View File

@@ -2,7 +2,7 @@ name: camera_avfoundation
description: iOS implementation of the camera plugin.
repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
version: 0.9.22+9
version: 0.9.22+10
environment:
sdk: ^3.9.0