Merge pull request #109 from facebookresearch/feature-human-object

Add posed SAM 3D Objects and SAM 3D Body alignment

demo.py

@@ -1,3 +1,4 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 import sys
 
 # import inference code

notebook/demo_3db_mesh_alignment.ipynb

@@ -44,6 +44,37 @@
     "os.makedirs(output_dir, exist_ok=True)\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 0. Inference and Save SAM 3D Objects"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Please inference SAM 3D Objects Repo with https://github.com/facebookresearch/sam-3d-objects/blob/main/notebook/demo_multi_object.ipynb\n",
+    "# The above notebook will apply the generated layout to the generated objects, and same them as ply. \n",
+    "# Then, this cell will load the posed SAM 3D Objects and transform them into the OpenGL coordinate system, which is the same system as SAM 3D Body. \n",
+    "import numpy as np\n",
+    "import open3d as o3d\n",
+    "\n",
+    "# Load PLY file\n",
+    "input_path = 'gaussians/human_object_posed.ply'\n",
+    "output_path = 'meshes/human_object/3Dfy_results/0.ply'\n",
+    "mesh = o3d.io.read_point_cloud(input_path)\n",
+    "points = np.asarray(mesh.points)\n",
+    "\n",
+    "# Transform to OpenGL coordinate system. \n",
+    "points[:, [0, 2]] *= -1  # flip x and z\n",
+    "mesh.points = o3d.utility.Vector3dVector(points)\n",
+    "o3d.io.write_point_cloud(output_path, mesh)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -115,7 +146,7 @@
     "from mesh_alignment import visualize_meshes_interactive\n",
     "\n",
     "aligned_mesh_path = f\"{PATH}/meshes/human_object/aligned_meshes/human_aligned.ply\"\n",
-    "dfy_mesh_path = f\"{PATH}/meshes/human_object/3Dfy_results/0.glb\"\n",
+    "dfy_mesh_path = f\"{PATH}/meshes/human_object/3Dfy_results/0.ply\"\n",
     "\n",
     "demo, combined_glb_path = visualize_meshes_interactive(\n",
     "    aligned_mesh_path=aligned_mesh_path,\n",
@@ -127,7 +158,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "sam3d_objects-3dfy",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },

notebook/demo_multi_object.ipynb

@@ -95,8 +95,10 @@
             "outputs": [],
             "source": [
                 "scene_gs = make_scene(*outputs)\n",
-                "scene_gs = ready_gaussian_for_video_rendering(scene_gs)\n",
+                "# export posed gaussian splatting (as point cloud)\n",
+                "scene_gs.save_ply(f\"{PATH}/gaussians/{IMAGE_NAME}_posed.ply\")\n",
                 "\n",
+                "scene_gs = ready_gaussian_for_video_rendering(scene_gs)\n",
                 "# export gaussian splatting (as point cloud)\n",
                 "scene_gs.save_ply(f\"{PATH}/gaussians/multi/{IMAGE_NAME}.ply\")\n",
                 "\n",
@@ -140,7 +142,7 @@
     ],
     "metadata": {
         "kernelspec": {
-            "display_name": "sam3d-objects",
+            "display_name": "Python 3",
             "language": "python",
             "name": "python3"
         },

notebook/mesh_alignment.py

@@ -298,7 +298,7 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No
 
     Args:
         aligned_mesh_path: Path to aligned mesh PLY file
-        dfy_mesh_path: Path to 3Dfy GLB file
+        dfy_mesh_path: Path to 3Dfy Object file
         output_dir: Directory to save combined GLB file (defaults to same dir as aligned_mesh_path)
         share: Whether to create a public shareable link (default: True)
         height: Height of the 3D viewer in pixels (default: 600)
@@ -307,6 +307,7 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No
         tuple: (demo, combined_glb_path) - Gradio demo object and path to combined GLB file
     """
     import gradio as gr
+    import numpy as np
 
     print("Loading meshes for interactive visualization...")
 
@@ -315,17 +316,17 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No
         aligned_mesh = trimesh.load(aligned_mesh_path)
         print(f"Loaded aligned mesh: {len(aligned_mesh.vertices)} vertices")
 
-        # Load 3Dfy mesh (GLB - handle scene structure)
+        # Load 3Dfy mesh (PLY)
         dfy_scene = trimesh.load(dfy_mesh_path)
 
-        if hasattr(dfy_scene, 'dump'):  # It's a scene
+        if hasattr(dfy_scene, 'dump'):
             dfy_meshes = [geom for geom in dfy_scene.geometry.values() if hasattr(geom, 'vertices')]
             if len(dfy_meshes) == 1:
                 dfy_mesh = dfy_meshes[0]
             elif len(dfy_meshes) > 1:
                 dfy_mesh = trimesh.util.concatenate(dfy_meshes)
             else:
-                raise ValueError("No valid meshes in GLB file")
+                raise ValueError("No valid meshes in PLY file")
         else:
             dfy_mesh = dfy_scene
 
@@ -348,14 +349,42 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No
             output_dir = os.path.dirname(aligned_mesh_path)
         os.makedirs(output_dir, exist_ok=True)
 
+        # Save combined PLY by concatenating both meshes
+        combined_ply_path = os.path.join(output_dir, 'combined_scene.ply')
+        
+        # Ccombine the geometries for PLY output
+        if isinstance(dfy_mesh, trimesh.points.PointCloud):
+            # Convert point cloud to vertices-only mesh for combination
+            dfy_vertices = dfy_mesh.vertices
+            human_vertices = aligned_mesh.vertices
+            
+            # Combine vertices from both
+            all_vertices = np.vstack([human_vertices, dfy_vertices])
+            
+            # Create colors: red for human, blue for object
+            human_colors = np.array([[255, 0, 0, 200]] * len(human_vertices))
+            object_colors = np.array([[0, 0, 255, 200]] * len(dfy_vertices))
+            all_colors = np.vstack([human_colors, object_colors])
+            
+            # Create combined point cloud
+            combined_cloud = trimesh.points.PointCloud(vertices=all_vertices, colors=all_colors)
+            combined_cloud.export(combined_ply_path)
+        else:
+            # Both are meshes, use scene export
+            scene.export(combined_ply_path)
+        
+        print(f"Exported combined scene to: {combined_ply_path}")
+
+        # Also save GLB for Gradio viewer (NOTE: GLB may not show point cloud object properly)
         combined_glb_path = os.path.join(output_dir, 'combined_scene.glb')
         scene.export(combined_glb_path)
-        print(f"Exported combined scene to: {combined_glb_path}")
+        print(f"Exported GLB for Gradio viewer to: {combined_glb_path}")
+        print("NOTE: Use PLY for complete data, GLB is for Gradio visualization only")
 
         # Create interactive Gradio viewer
         with gr.Blocks() as demo:
             gr.Markdown("# 3D Mesh Alignment Visualization")
-            gr.Markdown("**Red**: SAM 3D Body Aligned Human | **Blue**: 3Dfy Object")
+            gr.Markdown("**Red**: SAM 3D Body Aligned Human | **Blue**: SAM 3D Object")
             gr.Model3D(
                 value=combined_glb_path,
                 label="Combined 3D Scene (Interactive)",

sam3d_objects/pipeline/inference_pipeline.py

@@ -11,6 +11,8 @@ from torch.utils._pytree import tree_map_only
 def set_attention_backend():
     if torch.cuda.is_available():
         gpu_name = torch.cuda.get_device_name(0)
+    else:
+        gpu_name = "CPU"
 
     logger.info(f"GPU name is {gpu_name}")
     if "A100" in gpu_name or "H100" in gpu_name or "H200" in gpu_name: