unity/MediaPipe.NET
1
0
Fork 0
mirror of https://github.com/cosyneco/MediaPipe.NET.git synced 2025-05-17 07:26:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix critical memory leak (#22)

Browse Source 
* fix examples (or at least try to)

* stuff

* Better dispose that

* Add video input options because I'm selfish

* THIS ONE MOTHERFUCKING FOR LOOP

* Remove unnecessary using

* Fix osu framework example

* Use stable `Google.Protobuf` package

* Add options to osu framework example

* Use same options on all examples

* Add framerate option

* Use framerate option

Co-authored-by: Speykious <speykious@gmail.com>
This commit is contained in:
Adwyzz (OLED Edition)
2022-02-21 16:07:15 +00:00
committed by GitHub
parent d0bbbe4895
commit cc7ed1ebd0
17 changed files with 217 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Mediapipe.Net.Examples.FaceMesh/Mediapipe.Net.Examples.FaceMesh.csproj
View File

@ -11,7 +11,7 @@
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
<PackageReference Include="Mediapipe.Net.Runtime.CPU" Version="0.8.9" />
<PackageReference Include="SeeShark" Version="2.2.0" />
<PackageReference Include="SeeShark" Version="3.0.0" />
</ItemGroup>
<ItemGroup>

12
Mediapipe.Net.Examples.FaceMesh/Options.cs
View File

@ -10,5 +10,17 @@ namespace Mediapipe.Net.Examples.FaceMesh
{
[Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")]
public int CameraIndex { get; set; }
[Option('f', "input-format", Default = null, HelpText = "The format of the camera input")]
public string? InputFormat { get; set; }
[Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")]
public int? Framerate { get; set; }
[Option('w', "width", Default = null, HelpText = "The width of the camera input")]
public int? Width { get; set; }
[Option('h', "height", Default = null, HelpText = "The height of the camera input")]
public int? Height { get; set; }
}
}

60
Mediapipe.Net.Examples.FaceMesh/Program.cs
View File

@ -5,11 +5,13 @@
using System;
using System.Collections.Generic;
using CommandLine;
using FFmpeg.AutoGen;
using Mediapipe.Net.Calculators;
using Mediapipe.Net.External;
using Mediapipe.Net.Framework.Format;
using Mediapipe.Net.Framework.Protobuf;
using SeeShark;
using SeeShark.Device;
using SeeShark.FFmpeg;
namespace Mediapipe.Net.Examples.FaceMesh
@ -24,6 +26,15 @@ namespace Mediapipe.Net.Examples.FaceMesh
{
// Get and parse command line arguments
Options parsed = Parser.Default.ParseArguments<Options>(args).Value;
(int, int)? videoSize = null;
if (parsed.Width != null && parsed.Height != null)
videoSize = ((int)parsed.Width, (int)parsed.Height);
else if (parsed.Width != null && parsed.Height == null)
Console.Error.WriteLine("Specifying width requires to specify height");
else if (parsed.Width == null && parsed.Height != null)
Console.Error.WriteLine("Specifying height requires to specify width");
FFmpegManager.SetupFFmpeg("/usr/lib");
Glog.Initialize("stuff");
@ -32,7 +43,17 @@ namespace Mediapipe.Net.Examples.FaceMesh
{
try
{
camera = manager.GetCamera(parsed.CameraIndex);
camera = manager.GetDevice(parsed.CameraIndex,
new VideoInputOptions
{
InputFormat = parsed.InputFormat,
Framerate = parsed.Framerate == null ? null : new AVRational
{
num = (int)parsed.Framerate,
den = 1,
},
VideoSize = videoSize,
});
Console.WriteLine($"Using camera {camera.Info}");
}
catch (Exception)
@ -41,15 +62,25 @@ namespace Mediapipe.Net.Examples.FaceMesh
return;
}
}
camera.OnFrame += onFrame;
calculator = new FaceMeshCpuCalculator();
calculator.OnResult += handleLandmarks;
calculator.Run();
camera.StartCapture();
Console.CancelKeyPress += (sender, eventArgs) => exit();
Console.ReadLine();
while (true)
{
var frame = camera.GetFrame();
converter ??= new FrameConverter(frame, PixelFormat.Rgba);
Frame cFrame = converter.Convert(frame);
using ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame img = calculator.Send(imgframe);
}
}
private static void handleLandmarks(object? sender, List<NormalizedLandmarkList> landmarks)
@ -57,27 +88,6 @@ namespace Mediapipe.Net.Examples.FaceMesh
Console.WriteLine($"Got a list of {landmarks[0].Landmark.Count} landmarks at frame {calculator?.CurrentFrame}");
}
private static unsafe void onFrame(object? sender, FrameEventArgs e)
{
if (calculator == null)
return;
var frame = e.Frame;
converter ??= new FrameConverter(frame, PixelFormat.Rgba);
// Don't use a frame if it's not new
if (e.Status != DecodeStatus.NewFrame)
return;
Frame cFrame = converter.Convert(frame);
ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame img = calculator.Send(imgframe);
imgframe.Dispose();
}
// Dispose everything on exit
private static void exit()
{

1
Mediapipe.Net.Examples.FaceMesh/mediapipe Symbolic link
View File

@ -0,0 +1 @@
../mediapipe

2
Mediapipe.Net.Examples.FaceMeshGpu/Mediapipe.Net.Examples.FaceMeshGpu.csproj
View File

@ -11,7 +11,7 @@
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
<PackageReference Include="Mediapipe.Net.Runtime.GPU" Version="0.8.9" />
<PackageReference Include="SeeShark" Version="2.2.0" />
<PackageReference Include="SeeShark" Version="3.0.0" />
</ItemGroup>
<ItemGroup>

12
Mediapipe.Net.Examples.FaceMeshGpu/Options.cs
View File

@ -10,5 +10,17 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
{
[Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")]
public int CameraIndex { get; set; }
[Option('f', "input-format", Default = null, HelpText = "The format of the camera input")]
public string? InputFormat { get; set; }
[Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")]
public int? Framerate { get; set; }
[Option('w', "width", Default = null, HelpText = "The width of the camera input")]
public int? Width { get; set; }
[Option('h', "height", Default = null, HelpText = "The height of the camera input")]
public int? Height { get; set; }
}
}

61
Mediapipe.Net.Examples.FaceMeshGpu/Program.cs
View File

@ -6,11 +6,13 @@ using System;
using System.Collections.Generic;
using System.Runtime.Versioning;
using CommandLine;
using FFmpeg.AutoGen;
using Mediapipe.Net.Calculators;
using Mediapipe.Net.External;
using Mediapipe.Net.Framework.Format;
using Mediapipe.Net.Framework.Protobuf;
using SeeShark;
using SeeShark.Device;
using SeeShark.FFmpeg;
namespace Mediapipe.Net.Examples.FaceMeshGpu
@ -26,6 +28,15 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
{
// Get and parse command line arguments
Options parsed = Parser.Default.ParseArguments<Options>(args).Value;
(int, int)? videoSize = null;
if (parsed.Width != null && parsed.Height != null)
videoSize = ((int)parsed.Width, (int)parsed.Height);
else if (parsed.Width != null && parsed.Height == null)
Console.Error.WriteLine("Specifying width requires to specify height");
else if (parsed.Width == null && parsed.Height != null)
Console.Error.WriteLine("Specifying height requires to specify width");
FFmpegManager.SetupFFmpeg("/usr/lib");
Glog.Initialize("stuff");
@ -34,7 +45,17 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
{
try
{
camera = manager.GetCamera(parsed.CameraIndex);
camera = manager.GetDevice(parsed.CameraIndex,
new VideoInputOptions
{
InputFormat = parsed.InputFormat,
Framerate = parsed.Framerate == null ? null : new AVRational
{
num = (int)parsed.Framerate,
den = 1,
},
VideoSize = videoSize,
});
Console.WriteLine($"Using camera {camera.Info}");
}
catch (Exception)
@ -43,15 +64,26 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
return;
}
}
camera.OnFrame += onFrame;
calculator = new FaceMeshGpuCalculator();
calculator.OnResult += handleLandmarks;
calculator.Run();
camera.StartCapture();
Console.CancelKeyPress += (sender, eventArgs) => exit();
Console.ReadLine();
while (true)
{
var frame = camera.GetFrame();
converter ??= new FrameConverter(frame, PixelFormat.Rgba);
Frame cFrame = converter.Convert(frame);
ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame img = calculator.Send(imgframe);
imgframe.Dispose();
}
}
private static void handleLandmarks(object? sender, List<NormalizedLandmarkList> landmarks)
@ -59,27 +91,6 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
Console.WriteLine($"Got a list of {landmarks[0].Landmark.Count} landmarks at frame {calculator?.CurrentFrame}");
}
private static unsafe void onFrame(object? sender, FrameEventArgs e)
{
if (calculator == null)
return;
var frame = e.Frame;
converter ??= new FrameConverter(frame, PixelFormat.Rgba);
// Don't use a frame if it's not new
if (e.Status != DecodeStatus.NewFrame)
return;
Frame cFrame = converter.Convert(frame);
ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame img = calculator.Send(imgframe);
imgframe.Dispose();
}
// Dispose everything on exit
private static void exit()
{

11
Mediapipe.Net.Examples.OsuFrameworkVisualTests/Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj
View File

@ -10,15 +10,16 @@
</PropertyGroup>
<ItemGroup Label="Package References">
<PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
<PackageReference Include="Mediapipe.Net.Framework.Protobuf" Version="0.8.9" />
<PackageReference Include="Mediapipe.Net.Runtime.CPU" Version="0.8.9" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
<PackageReference Include="NUnit3TestAdapter" Version="4.0.0" />
<PackageReference Include="ppy.osu.Framework" Version="2022.118.0" />
<PackageReference Include="SeeShark" Version="2.2.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
<PackageReference Include="ppy.osu.Framework" Version="2022.217.0" />
<PackageReference Include="SeeShark" Version="3.0.0" />
</ItemGroup>
<ItemGroup>
<ItemGroup Label="Project References">
<ProjectReference Include="..\Mediapipe.Net\Mediapipe.Net.csproj" />
</ItemGroup>

64
Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs
View File

@ -5,37 +5,32 @@
using System;
using Mediapipe.Net.Calculators;
using Mediapipe.Net.Framework.Format;
using osu.Framework.Allocation;
using osu.Framework.Graphics;
using osu.Framework.Graphics.Containers;
using osu.Framework.Graphics.Sprites;
using osu.Framework.Graphics.Textures;
using SeeShark;
using SeeShark.FFmpeg;
using SeeShark.Decode;
using SeeShark.Device;
using SixLabors.ImageSharp.PixelFormats;
using Image = SixLabors.ImageSharp.Image;
namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
{
public class MediapipeDrawable : CompositeDrawable
{
public readonly Camera Camera;
private Camera? camera;
private FrameConverter? converter;
private readonly FaceMeshCpuCalculator calculator;
private FaceMeshCpuCalculator? calculator;
private readonly Sprite sprite;
private Texture? texture;
public MediapipeDrawable(int cameraIndex = 0)
public MediapipeDrawable()
{
var manager = new CameraManager();
Camera = manager.GetCamera(cameraIndex);
manager.Dispose();
Camera.OnFrame += onFrame;
calculator = new FaceMeshCpuCalculator();
Masking = true;
CornerRadius = 10;
AddInternal(sprite = new Sprite
{
Anchor = Anchor.Centre,
@ -46,53 +41,38 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
});
}
public void Start()
#pragma warning disable IDE0051
[BackgroundDependencyLoader]
private void load(Camera camera, FrameConverter converter, FaceMeshCpuCalculator calculator)
{
calculator.Run();
Camera.StartCapture();
this.camera = camera;
this.converter = converter;
this.calculator = calculator;
}
#pragma warning restore IDE0051
private unsafe void onFrame(object? sender, FrameEventArgs e)
protected override unsafe void Update()
{
var frame = e.Frame;
if (converter == null)
converter = new FrameConverter(frame, PixelFormat.Rgba);
base.Update();
if (camera == null || converter == null || calculator == null)
return;
// Don't use a frame if it's not new
if (e.Status != DecodeStatus.NewFrame)
if (camera.TryGetFrame(out Frame frame) != DecodeStatus.NewFrame)
return;
Frame cFrame = converter.Convert(frame);
ImageFrame imgFrame;
fixed (byte* rawDataPtr = cFrame.RawData)
{
imgFrame = new ImageFrame(ImageFormat.Srgba, cFrame.Width, cFrame.Height, cFrame.WidthStep,
rawDataPtr);
}
ImageFrame imgFrame = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame outImgFrame = calculator.Send(imgFrame);
imgFrame.Dispose();
var span = new ReadOnlySpan<byte>(outImgFrame.MutablePixelData, outImgFrame.Height * outImgFrame.WidthStep);
var pixelData = SixLabors.ImageSharp.Image.LoadPixelData<Rgba32>(span, cFrame.Width, cFrame.Height);
var pixelData = Image.LoadPixelData<Rgba32>(span, cFrame.Width, cFrame.Height);
texture ??= new Texture(cFrame.Width, cFrame.Height);
texture.SetData(new TextureUpload(pixelData));
sprite.FillAspectRatio = (float)cFrame.Width / cFrame.Height;
sprite.Texture = texture;
}
public new void Dispose()
{
if (IsDisposed)
return;
Camera.StopCapture();
Camera.Dispose();
converter?.Dispose();
calculator.Dispose();
base.Dispose();
}
}
}

26
Mediapipe.Net.Examples.OsuFrameworkVisualTests/Options.cs Normal file
View File

@ -0,0 +1,26 @@
// Copyright (c) homuler and The Vignette Authors
// This file is part of MediaPipe.NET.
// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
using CommandLine;
namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
{
public class Options
{
[Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")]
public int CameraIndex { get; set; }
[Option('f', "input-format", Default = null, HelpText = "The format of the camera input")]
public string? InputFormat { get; set; }
[Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")]
public int? Framerate { get; set; }
[Option('w', "width", Default = null, HelpText = "The width of the camera input")]
public int? Width { get; set; }
[Option('h', "height", Default = null, HelpText = "The height of the camera input")]
public int? Height { get; set; }
}
}

56
Mediapipe.Net.Examples.OsuFrameworkVisualTests/OsuFrameworkVisualTestsGameBase.cs
View File

@ -2,17 +2,26 @@
// This file is part of MediaPipe.NET.
// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
using System;
using CommandLine;
using FFmpeg.AutoGen;
using Mediapipe.Net.Calculators;
using osu.Framework;
using osu.Framework.Allocation;
using osu.Framework.Graphics;
using osu.Framework.Graphics.Containers;
using osuTK;
using SeeShark;
using SeeShark.Device;
namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
{
public class OsuFrameworkVisualTestsGameBase : Game
{
private MediapipeDrawable? mediapipeDrawable;
private Camera? camera;
private FrameConverter? converter;
private FaceMeshCpuCalculator? calculator;
protected override Container<Drawable> Content { get; }
private DependencyContainer? dependencies;
@ -31,20 +40,47 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
[BackgroundDependencyLoader]
private void load()
{
mediapipeDrawable = new MediapipeDrawable
{
Anchor = Anchor.Centre,
Origin = Anchor.Centre,
Size = new Vector2(1280, 720),
FillMode = FillMode.Fit
};
dependencies?.Cache(mediapipeDrawable);
string[] args = Environment.GetCommandLineArgs();
Options parsed = Parser.Default.ParseArguments<Options>(args).Value;
(int, int)? videoSize = null;
if (parsed.Width != null && parsed.Height != null)
videoSize = ((int)parsed.Width, (int)parsed.Height);
else if (parsed.Width != null && parsed.Height == null)
Console.Error.WriteLine("Specifying width requires to specify height");
else if (parsed.Width == null && parsed.Height != null)
Console.Error.WriteLine("Specifying height requires to specify width");
var manager = new CameraManager();
camera = manager.GetDevice(parsed.CameraIndex,
new VideoInputOptions
{
InputFormat = parsed.InputFormat,
Framerate = parsed.Framerate == null ? null : new AVRational
{
num = (int)parsed.Framerate,
den = 1,
},
VideoSize = videoSize,
});
dependencies?.Cache(camera);
manager.Dispose();
var dummyFrame = camera.GetFrame();
converter = new FrameConverter(dummyFrame, PixelFormat.Rgba);
dependencies?.Cache(converter);
calculator = new FaceMeshCpuCalculator();
calculator.Run();
dependencies?.Cache(calculator);
}
#pragma warning restore IDE0051
protected override bool OnExiting()
{
mediapipeDrawable?.Dispose();
calculator?.Dispose();
converter?.Dispose();
camera?.Dispose();
return base.OnExiting();
}
}

2
Mediapipe.Net.Examples.OsuFrameworkVisualTests/Program.cs
View File

@ -13,7 +13,7 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
public static void Main()
{
FFmpegManager.SetupFFmpeg("/usr/lib");
using GameHost host = Host.GetSuitableHost("visual-tests");
using GameHost host = Host.GetSuitableDesktopHost("visual-tests");
using var game = new OsuFrameworkVisualTestsTestBrowser();
host.Run(game);
}

12
Mediapipe.Net.Examples.OsuFrameworkVisualTests/Visual/TestSceneFaceMeshCpu.cs
View File

@ -7,6 +7,7 @@ using osu.Framework.Extensions.Color4Extensions;
using osu.Framework.Graphics;
using osu.Framework.Graphics.Containers;
using osu.Framework.Graphics.Shapes;
using osuTK;
namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests.Visual
{
@ -14,7 +15,7 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests.Visual
{
#pragma warning disable IDE0051
[BackgroundDependencyLoader]
private void load(MediapipeDrawable mediapipeDrawable)
private void load()
{
Add(new Container
{
@ -26,10 +27,15 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests.Visual
RelativeSizeAxes = Axes.Both,
Colour = Color4Extensions.FromHex(@"272727"),
},
mediapipeDrawable,
new MediapipeDrawable
{
Anchor = Anchor.Centre,
Origin = Anchor.Centre,
Size = new Vector2(1280, 720),
FillMode = FillMode.Fit,
},
},
});
mediapipeDrawable.Start();
}
#pragma warning restore IDE0051
}

1
Mediapipe.Net.Examples.OsuFrameworkVisualTests/mediapipe Symbolic link
View File

@ -0,0 +1 @@
../mediapipe

7
Mediapipe.Net/External/SerializedProtoVector.cs vendored
View File

@ -17,7 +17,12 @@ namespace Mediapipe.Net.External
// TODO: This is looking just as sus as SerializedProto.Dispose().
// Should be investigated in the same way.
public void Dispose() => UnsafeNativeMethods.mp_api_SerializedProtoArray__delete(Data);
public void Dispose()
{
for (int i = 0; i < Size; i++)
Data[i].Dispose();
UnsafeNativeMethods.mp_api_SerializedProtoArray__delete(Data);
}
public List<T> Deserialize<T>(MessageParser<T> parser) where T : IMessage<T>
{

1
Mediapipe.Net/Framework/CalculatorGraph.cs
View File

@ -89,6 +89,7 @@ namespace Mediapipe.Net.Framework
{
var packet = (TPacket?)Activator.CreateInstance(typeof(TPacket), (IntPtr)packetPtr, false);
status = packetCallback(packet);
packet?.Dispose();
}
catch (Exception e)
{

2
Mediapipe.Net/Mediapipe.Net.csproj
View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup Label="Package References">
<PackageReference Include="Google.Protobuf" Version="3.19.3" />
<PackageReference Include="Google.Protobuf" Version="3.19.4" />
<PackageReference Include="Mediapipe.Net.Framework.Protobuf" Version="0.8.9" />
</ItemGroup>