unity/MediaPipe.NET
1
0
Fork 0
mirror of https://github.com/cosyneco/MediaPipe.NET.git synced 2025-05-17 07:26:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Working BlazePose example (#28)

Browse Source 
* Make ImageFrame nullable, because BlazePose can return empty frames

* Add BlazePose example

Co-authored-by: Darren <darren@velogicfit.com>
This commit is contained in:
Darren Bruning
2022-03-15 18:59:57 +13:00
committed by GitHub
parent 0f116b41bb
commit a95e179d80
11 changed files with 209 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
MediaPipe.NET.sln
View File

@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mediapipe.Net.Examples.Face
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mediapipe.Net.Examples.OsuFrameworkVisualTests", "Mediapipe.Net.Examples.OsuFrameworkVisualTests\Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj", "{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mediapipe.Net.Examples.BlazePose", "Mediapipe.Net.Examples.BlazePose\Mediapipe.Net.Examples.BlazePose.csproj", "{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -42,5 +44,9 @@ Global
{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}.Release|Any CPU.Build.0 = Release|Any CPU
{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

26
Mediapipe.Net.Examples.BlazePose/DummyResourceManager.cs Normal file
View File

@ -0,0 +1,26 @@
// Copyright (c) homuler and The Vignette Authors
// This file is part of MediaPipe.NET.
// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
using System;
using System.IO;
using Mediapipe.Net.Util;
namespace Mediapipe.Net.Examples.BlazePose
{
public class DummyResourceManager : ResourceManager
{
public override PathResolver ResolvePath => (path) =>
{
Console.WriteLine($"PathResolver: (not) resolving path '{path}'");
return path;
};
public unsafe override ResourceProvider ProvideResource => (path) =>
{
Console.WriteLine($"ResourceProvider: providing resource '{path}'");
byte[] bytes = File.ReadAllBytes(path);
return bytes;
};
}
}

21
Mediapipe.Net.Examples.BlazePose/Mediapipe.Net.Examples.BlazePose.csproj Normal file
View File

@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<ImplicitUsings>disable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
<PackageReference Include="Mediapipe.Net.Runtime.CPU" Version="0.8.9" />
<PackageReference Include="SeeShark" Version="3.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Mediapipe.Net\Mediapipe.Net.csproj" />
</ItemGroup>
</Project>

29
Mediapipe.Net.Examples.BlazePose/Options.cs Normal file
View File

@ -0,0 +1,29 @@
// Copyright (c) homuler and The Vignette Authors
// This file is part of MediaPipe.NET.
// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
using CommandLine;
namespace Mediapipe.Net.Examples.BlazePose
{
public class Options
{
[Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")]
public int CameraIndex { get; set; }
[Option('f', "input-format", Default = null, HelpText = "The format of the camera input")]
public string? InputFormat { get; set; }
[Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")]
public int? Framerate { get; set; }
[Option('w', "width", Default = null, HelpText = "The width of the camera input")]
public int? Width { get; set; }
[Option('h', "height", Default = null, HelpText = "The height of the camera input")]
public int? Height { get; set; }
[Option("use-resource-manager", Default = false, HelpText = "Whether to use a resource manager.")]
public bool UseResourceManager { get; set; }
}
}

104
Mediapipe.Net.Examples.BlazePose/Program.cs Normal file
View File

@ -0,0 +1,104 @@
// Copyright (c) homuler and The Vignette Authors
// This file is part of MediaPipe.NET.
// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
using System;
using CommandLine;
using FFmpeg.AutoGen;
using Mediapipe.Net.Calculators;
using Mediapipe.Net.External;
using Mediapipe.Net.Framework.Format;
using Mediapipe.Net.Framework.Protobuf;
using Mediapipe.Net.Util;
using SeeShark;
using SeeShark.Device;
using SeeShark.FFmpeg;
namespace Mediapipe.Net.Examples.BlazePose
{
public static class Program
{
private static Camera? camera;
private static FrameConverter? converter;
private static BlazePoseCpuCalculator? calculator;
private static ResourceManager? resourceManager;
public static void Main(string[] args)
{
// Get and parse command line arguments
Options parsed = Parser.Default.ParseArguments<Options>(args).Value;
(int, int)? videoSize = null;
if (parsed.Width != null && parsed.Height != null)
videoSize = ((int)parsed.Width, (int)parsed.Height);
else if (parsed.Width != null && parsed.Height == null)
Console.Error.WriteLine("Specifying width requires to specify height");
else if (parsed.Width == null && parsed.Height != null)
Console.Error.WriteLine("Specifying height requires to specify width");
// FFmpegManager.SetupFFmpeg("/usr/lib");
FFmpegManager.SetupFFmpeg(@"C:\ffmpeg\v5.0_x64\");
Glog.Initialize("stuff");
if (parsed.UseResourceManager)
resourceManager = new DummyResourceManager();
// Get a camera device
using (CameraManager manager = new CameraManager())
{
try
{
camera = manager.GetDevice(parsed.CameraIndex,
new VideoInputOptions
{
InputFormat = parsed.InputFormat,
Framerate = parsed.Framerate == null ? null : new AVRational
{
num = (int)parsed.Framerate,
den = 1,
},
VideoSize = videoSize,
});
Console.WriteLine($"Using camera {camera.Info}");
}
catch (Exception)
{
Console.Error.WriteLine($"No camera exists at index {parsed.CameraIndex}.");
return;
}
}
calculator = new BlazePoseCpuCalculator();
calculator.OnResult += handleLandmarks;
calculator.Run();
Console.CancelKeyPress += (sender, eventArgs) => exit();
while (true)
{
var frame = camera.GetFrame();
converter ??= new FrameConverter(frame, PixelFormat.Rgb24);
Frame cFrame = converter.Convert(frame);
using ImageFrame imgframe = new ImageFrame(ImageFormat.Srgb,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame? img = calculator.Send(imgframe);
}
}
private static void handleLandmarks(object? sender, NormalizedLandmarkList landmarks)
{
Console.WriteLine($"Got a list of {landmarks.Landmark.Count} landmarks at frame {calculator?.CurrentFrame}");
}
// Dispose everything on exit
private static void exit()
{
Console.WriteLine("Exiting...");
camera?.Dispose();
converter?.Dispose();
calculator?.Dispose();
}
}
}

1
Mediapipe.Net.Examples.BlazePose/mediapipe Normal file
View File

@ -0,0 +1 @@
../mediapipe

5
Mediapipe.Net.Examples.FaceMesh/Program.cs
View File

@ -37,7 +37,8 @@ namespace Mediapipe.Net.Examples.FaceMesh
else if (parsed.Width == null && parsed.Height != null)
Console.Error.WriteLine("Specifying height requires to specify width");
FFmpegManager.SetupFFmpeg("/usr/lib");
FFmpegManager.SetupFFmpeg(@"C:\ffmpeg\v5.0_x64\");
// FFmpegManager.SetupFFmpeg("/usr/lib");
Glog.Initialize("stuff");
if (parsed.UseResourceManager)
resourceManager = new DummyResourceManager();
@ -83,7 +84,7 @@ namespace Mediapipe.Net.Examples.FaceMesh
using ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame img = calculator.Send(imgframe);
using ImageFrame? img = calculator.Send(imgframe);
}
}

2
Mediapipe.Net.Examples.FaceMeshGpu/Program.cs
View File

@ -81,7 +81,7 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame img = calculator.Send(imgframe);
using ImageFrame? img = calculator.Send(imgframe);
}
}

3
Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs
View File

@ -63,7 +63,8 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
Frame cFrame = converter.Convert(frame);
ImageFrame imgFrame = new ImageFrame(ImageFormat.Srgba,
cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
using ImageFrame outImgFrame = calculator.Send(imgFrame);
using ImageFrame? outImgFrame = calculator.Send(imgFrame);
if (outImgFrame == null) return;
var span = new ReadOnlySpan<byte>(outImgFrame.MutablePixelData, outImgFrame.Height * outImgFrame.WidthStep);
var pixelData = Image.LoadPixelData<Rgba32>(span, cFrame.Width, cFrame.Height);

6
Mediapipe.Net/Calculators/Calculator.cs
View File

@ -64,7 +64,7 @@ namespace Mediapipe.Net.Calculators
/// <remarks>You need to call this method before sending frames to it.</remarks>
public void Run() => Graph.StartRun().AssertOk();
protected abstract ImageFrame SendFrame(ImageFrame frame);
protected abstract ImageFrame? SendFrame(ImageFrame frame);
/// <summary>
/// Sends an <see cref="ImageFrame"/> for the calculator to process.
@ -73,11 +73,11 @@ namespace Mediapipe.Net.Calculators
/// <param name="frame">The frame that MediaPipe should process.</param>
/// <param name="disposeSourceFrame">Whether or not to dispose the source frame.</param>
/// <returns>An <see cref="ImageFrame"/> with the contents of the source <see cref="ImageFrame"/> and the MediaPipe solution drawn.</returns>
public ImageFrame Send(ImageFrame frame, bool disposeSourceFrame = true)
public ImageFrame? Send(ImageFrame frame, bool disposeSourceFrame = true)
{
lock (frame)
{
ImageFrame outFrame = SendFrame(frame);
ImageFrame? outFrame = SendFrame(frame);
CurrentFrame++;
if (disposeSourceFrame)
frame.Dispose();

17
Mediapipe.Net/Calculators/CpuCalculator.cs
View File

@ -2,6 +2,7 @@
// This file is part of MediaPipe.NET.
// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
using System;
using Mediapipe.Net.Framework;
using Mediapipe.Net.Framework.Format;
using Mediapipe.Net.Framework.Packet;
@ -24,16 +25,24 @@ namespace Mediapipe.Net.Calculators
framePoller = Graph.AddOutputStreamPoller<ImageFrame>(OUTPUT_VIDEO_STREAM).Value();
}
protected override ImageFrame SendFrame(ImageFrame frame)
protected override ImageFrame? SendFrame(ImageFrame frame)
{
using ImageFramePacket packet = new ImageFramePacket(frame, new Timestamp(CurrentFrame));
Graph.AddPacketToInputStream(INPUT_VIDEO_STREAM, packet).AssertOk();
ImageFramePacket outPacket = new ImageFramePacket();
framePoller.Next(outPacket);
ImageFrame outFrame = outPacket.Get();
if (framePoller.Next(outPacket))
{
ImageFrame outFrame = outPacket.Get();
return outFrame;
}
else
{
// Dispose the unused outPacket
outPacket.Dispose();
}
return outFrame;
return null;
}
protected override void DisposeManaged()