Working BlazePose example (#28)

* Make ImageFrame nullable, because BlazePose can return empty frames

* Add BlazePose example

Co-authored-by: Darren <darren@velogicfit.com>

MediaPipe.NET.sln

@@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mediapipe.Net.Examples.Face
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mediapipe.Net.Examples.OsuFrameworkVisualTests", "Mediapipe.Net.Examples.OsuFrameworkVisualTests\Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj", "{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mediapipe.Net.Examples.BlazePose", "Mediapipe.Net.Examples.BlazePose\Mediapipe.Net.Examples.BlazePose.csproj", "{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -42,5 +44,9 @@ Global
 		{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2422B1E2-7C28-4D7E-B29C-4349FFB65EE1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A0B5C78E-D278-4C2D-8A5D-815B8C9F3B60}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

Mediapipe.Net.Examples.BlazePose/DummyResourceManager.cs

@@ -0,0 +1,26 @@
+// Copyright (c) homuler and The Vignette Authors
+// This file is part of MediaPipe.NET.
+// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
+
+using System;
+using System.IO;
+using Mediapipe.Net.Util;
+
+namespace Mediapipe.Net.Examples.BlazePose
+{
+    public class DummyResourceManager : ResourceManager
+    {
+        public override PathResolver ResolvePath => (path) =>
+        {
+            Console.WriteLine($"PathResolver: (not) resolving path '{path}'");
+            return path;
+        };
+
+        public unsafe override ResourceProvider ProvideResource => (path) =>
+        {
+            Console.WriteLine($"ResourceProvider: providing resource '{path}'");
+            byte[] bytes = File.ReadAllBytes(path);
+            return bytes;
+        };
+    }
+}

Mediapipe.Net.Examples.BlazePose/Mediapipe.Net.Examples.BlazePose.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <ImplicitUsings>disable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
+    <PackageReference Include="Mediapipe.Net.Runtime.CPU" Version="0.8.9" />
+    <PackageReference Include="SeeShark" Version="3.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Mediapipe.Net\Mediapipe.Net.csproj" />
+  </ItemGroup>
+
+</Project>

Mediapipe.Net.Examples.BlazePose/Options.cs

@@ -0,0 +1,29 @@
+// Copyright (c) homuler and The Vignette Authors
+// This file is part of MediaPipe.NET.
+// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
+
+using CommandLine;
+
+namespace Mediapipe.Net.Examples.BlazePose
+{
+    public class Options
+    {
+        [Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")]
+        public int CameraIndex { get; set; }
+
+        [Option('f', "input-format", Default = null, HelpText = "The format of the camera input")]
+        public string? InputFormat { get; set; }
+
+        [Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")]
+        public int? Framerate { get; set; }
+
+        [Option('w', "width", Default = null, HelpText = "The width of the camera input")]
+        public int? Width { get; set; }
+
+        [Option('h', "height", Default = null, HelpText = "The height of the camera input")]
+        public int? Height { get; set; }
+
+        [Option("use-resource-manager", Default = false, HelpText = "Whether to use a resource manager.")]
+        public bool UseResourceManager { get; set; }
+    }
+}

Mediapipe.Net.Examples.BlazePose/Program.cs

@@ -0,0 +1,104 @@
+// Copyright (c) homuler and The Vignette Authors
+// This file is part of MediaPipe.NET.
+// MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
+
+using System;
+using CommandLine;
+using FFmpeg.AutoGen;
+using Mediapipe.Net.Calculators;
+using Mediapipe.Net.External;
+using Mediapipe.Net.Framework.Format;
+using Mediapipe.Net.Framework.Protobuf;
+using Mediapipe.Net.Util;
+using SeeShark;
+using SeeShark.Device;
+using SeeShark.FFmpeg;
+
+namespace Mediapipe.Net.Examples.BlazePose
+{
+    public static class Program
+    {
+        private static Camera? camera;
+        private static FrameConverter? converter;
+        private static BlazePoseCpuCalculator? calculator;
+        private static ResourceManager? resourceManager;
+
+        public static void Main(string[] args)
+        {
+            // Get and parse command line arguments
+            Options parsed = Parser.Default.ParseArguments<Options>(args).Value;
+
+            (int, int)? videoSize = null;
+            if (parsed.Width != null && parsed.Height != null)
+                videoSize = ((int)parsed.Width, (int)parsed.Height);
+            else if (parsed.Width != null && parsed.Height == null)
+                Console.Error.WriteLine("Specifying width requires to specify height");
+            else if (parsed.Width == null && parsed.Height != null)
+                Console.Error.WriteLine("Specifying height requires to specify width");
+
+            // FFmpegManager.SetupFFmpeg("/usr/lib");
+            FFmpegManager.SetupFFmpeg(@"C:\ffmpeg\v5.0_x64\");
+            Glog.Initialize("stuff");
+            if (parsed.UseResourceManager)
+                resourceManager = new DummyResourceManager();
+
+            // Get a camera device
+            using (CameraManager manager = new CameraManager())
+            {
+                try
+                {
+                    camera = manager.GetDevice(parsed.CameraIndex,
+                        new VideoInputOptions
+                        {
+                            InputFormat = parsed.InputFormat,
+                            Framerate = parsed.Framerate == null ? null : new AVRational
+                            {
+                                num = (int)parsed.Framerate,
+                                den = 1,
+                            },
+                            VideoSize = videoSize,
+                        });
+                    Console.WriteLine($"Using camera {camera.Info}");
+                }
+                catch (Exception)
+                {
+                    Console.Error.WriteLine($"No camera exists at index {parsed.CameraIndex}.");
+                    return;
+                }
+            }
+
+            calculator = new BlazePoseCpuCalculator();
+            calculator.OnResult += handleLandmarks;
+            calculator.Run();
+
+            Console.CancelKeyPress += (sender, eventArgs) => exit();
+            while (true)
+            {
+                var frame = camera.GetFrame();
+
+                converter ??= new FrameConverter(frame, PixelFormat.Rgb24);
+
+                Frame cFrame = converter.Convert(frame);
+
+                using ImageFrame imgframe = new ImageFrame(ImageFormat.Srgb,
+                    cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
+
+                using ImageFrame? img = calculator.Send(imgframe);
+            }
+        }
+
+        private static void handleLandmarks(object? sender, NormalizedLandmarkList landmarks)
+        {
+            Console.WriteLine($"Got a list of {landmarks.Landmark.Count} landmarks at frame {calculator?.CurrentFrame}");
+        }
+
+        // Dispose everything on exit
+        private static void exit()
+        {
+            Console.WriteLine("Exiting...");
+            camera?.Dispose();
+            converter?.Dispose();
+            calculator?.Dispose();
+        }
+    }
+}

Mediapipe.Net.Examples.BlazePose/mediapipe

@@ -0,0 +1 @@
+../mediapipe

Mediapipe.Net.Examples.FaceMesh/Program.cs

@@ -37,7 +37,8 @@ namespace Mediapipe.Net.Examples.FaceMesh
             else if (parsed.Width == null && parsed.Height != null)
                 Console.Error.WriteLine("Specifying height requires to specify width");
 
-            FFmpegManager.SetupFFmpeg("/usr/lib");
+            FFmpegManager.SetupFFmpeg(@"C:\ffmpeg\v5.0_x64\");
+            // FFmpegManager.SetupFFmpeg("/usr/lib");
             Glog.Initialize("stuff");
             if (parsed.UseResourceManager)
                 resourceManager = new DummyResourceManager();
@@ -83,7 +84,7 @@ namespace Mediapipe.Net.Examples.FaceMesh
                 using ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
                     cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
 
-                using ImageFrame img = calculator.Send(imgframe);
+                using ImageFrame? img = calculator.Send(imgframe);
             }
         }
 

Mediapipe.Net.Examples.FaceMeshGpu/Program.cs

@@ -81,7 +81,7 @@ namespace Mediapipe.Net.Examples.FaceMeshGpu
                 ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba,
                     cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
 
-                using ImageFrame img = calculator.Send(imgframe);
+                using ImageFrame? img = calculator.Send(imgframe);
             }
         }
 

Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs

@@ -63,7 +63,8 @@ namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests
             Frame cFrame = converter.Convert(frame);
             ImageFrame imgFrame = new ImageFrame(ImageFormat.Srgba,
                 cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData);
-            using ImageFrame outImgFrame = calculator.Send(imgFrame);
+            using ImageFrame? outImgFrame = calculator.Send(imgFrame);
+            if (outImgFrame == null) return;
 
             var span = new ReadOnlySpan<byte>(outImgFrame.MutablePixelData, outImgFrame.Height * outImgFrame.WidthStep);
             var pixelData = Image.LoadPixelData<Rgba32>(span, cFrame.Width, cFrame.Height);

Mediapipe.Net/Calculators/Calculator.cs

@@ -64,7 +64,7 @@ namespace Mediapipe.Net.Calculators
         /// <remarks>You need to call this method before sending frames to it.</remarks>
         public void Run() => Graph.StartRun().AssertOk();
 
-        protected abstract ImageFrame SendFrame(ImageFrame frame);
+        protected abstract ImageFrame? SendFrame(ImageFrame frame);
 
         /// <summary>
         /// Sends an <see cref="ImageFrame"/> for the calculator to process.
@@ -73,11 +73,11 @@ namespace Mediapipe.Net.Calculators
         /// <param name="frame">The frame that MediaPipe should process.</param>
         /// <param name="disposeSourceFrame">Whether or not to dispose the source frame.</param>
         /// <returns>An <see cref="ImageFrame"/> with the contents of the source <see cref="ImageFrame"/> and the MediaPipe solution drawn.</returns>
-        public ImageFrame Send(ImageFrame frame, bool disposeSourceFrame = true)
+        public ImageFrame? Send(ImageFrame frame, bool disposeSourceFrame = true)
         {
             lock (frame)
             {
-                ImageFrame outFrame = SendFrame(frame);
+                ImageFrame? outFrame = SendFrame(frame);
                 CurrentFrame++;
                 if (disposeSourceFrame)
                     frame.Dispose();

Mediapipe.Net/Calculators/CpuCalculator.cs

@@ -2,6 +2,7 @@
 // This file is part of MediaPipe.NET.
 // MediaPipe.NET is licensed under the MIT License. See LICENSE for details.
 
+using System;
 using Mediapipe.Net.Framework;
 using Mediapipe.Net.Framework.Format;
 using Mediapipe.Net.Framework.Packet;
@@ -24,17 +25,25 @@ namespace Mediapipe.Net.Calculators
             framePoller = Graph.AddOutputStreamPoller<ImageFrame>(OUTPUT_VIDEO_STREAM).Value();
         }
 
-        protected override ImageFrame SendFrame(ImageFrame frame)
+        protected override ImageFrame? SendFrame(ImageFrame frame)
         {
             using ImageFramePacket packet = new ImageFramePacket(frame, new Timestamp(CurrentFrame));
             Graph.AddPacketToInputStream(INPUT_VIDEO_STREAM, packet).AssertOk();
 
             ImageFramePacket outPacket = new ImageFramePacket();
-            framePoller.Next(outPacket);
+            if (framePoller.Next(outPacket))
+            {
                 ImageFrame outFrame = outPacket.Get();
-
                 return outFrame;
             }
+            else
+            {
+                // Dispose the unused outPacket
+                outPacket.Dispose();
+            }
+
+            return null;
+        }
 
         protected override void DisposeManaged()
         {