diff --git a/demo.py b/demo.py index befdd84..ac52037 100644 --- a/demo.py +++ b/demo.py @@ -1,3 +1,4 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. import sys # import inference code diff --git a/notebook/demo_3db_mesh_alignment.ipynb b/notebook/demo_3db_mesh_alignment.ipynb index e99c69f..899f868 100644 --- a/notebook/demo_3db_mesh_alignment.ipynb +++ b/notebook/demo_3db_mesh_alignment.ipynb @@ -44,6 +44,37 @@ "os.makedirs(output_dir, exist_ok=True)\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Inference and Save SAM 3D Objects" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Please inference SAM 3D Objects Repo with https://github.com/facebookresearch/sam-3d-objects/blob/main/notebook/demo_multi_object.ipynb\n", + "# The above notebook will apply the generated layout to the generated objects, and same them as ply. \n", + "# Then, this cell will load the posed SAM 3D Objects and transform them into the OpenGL coordinate system, which is the same system as SAM 3D Body. \n", + "import numpy as np\n", + "import open3d as o3d\n", + "\n", + "# Load PLY file\n", + "input_path = 'gaussians/human_object_posed.ply'\n", + "output_path = 'meshes/human_object/3Dfy_results/0.ply'\n", + "mesh = o3d.io.read_point_cloud(input_path)\n", + "points = np.asarray(mesh.points)\n", + "\n", + "# Transform to OpenGL coordinate system. \n", + "points[:, [0, 2]] *= -1 # flip x and z\n", + "mesh.points = o3d.utility.Vector3dVector(points)\n", + "o3d.io.write_point_cloud(output_path, mesh)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -115,7 +146,7 @@ "from mesh_alignment import visualize_meshes_interactive\n", "\n", "aligned_mesh_path = f\"{PATH}/meshes/human_object/aligned_meshes/human_aligned.ply\"\n", - "dfy_mesh_path = f\"{PATH}/meshes/human_object/3Dfy_results/0.glb\"\n", + "dfy_mesh_path = f\"{PATH}/meshes/human_object/3Dfy_results/0.ply\"\n", "\n", "demo, combined_glb_path = visualize_meshes_interactive(\n", " aligned_mesh_path=aligned_mesh_path,\n", @@ -127,7 +158,7 @@ ], "metadata": { "kernelspec": { - "display_name": "sam3d_objects-3dfy", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/notebook/demo_multi_object.ipynb b/notebook/demo_multi_object.ipynb index ecd817f..a6d8a22 100644 --- a/notebook/demo_multi_object.ipynb +++ b/notebook/demo_multi_object.ipynb @@ -95,8 +95,10 @@ "outputs": [], "source": [ "scene_gs = make_scene(*outputs)\n", - "scene_gs = ready_gaussian_for_video_rendering(scene_gs)\n", + "# export posed gaussian splatting (as point cloud)\n", + "scene_gs.save_ply(f\"{PATH}/gaussians/{IMAGE_NAME}_posed.ply\")\n", "\n", + "scene_gs = ready_gaussian_for_video_rendering(scene_gs)\n", "# export gaussian splatting (as point cloud)\n", "scene_gs.save_ply(f\"{PATH}/gaussians/multi/{IMAGE_NAME}.ply\")\n", "\n", @@ -140,7 +142,7 @@ ], "metadata": { "kernelspec": { - "display_name": "sam3d-objects", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/notebook/images/human_object/image.png b/notebook/images/human_object/image.png index e9d6739..b18eeab 100644 Binary files a/notebook/images/human_object/image.png and b/notebook/images/human_object/image.png differ diff --git a/notebook/mesh_alignment.py b/notebook/mesh_alignment.py index d211e39..9c6da5a 100644 --- a/notebook/mesh_alignment.py +++ b/notebook/mesh_alignment.py @@ -298,7 +298,7 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No Args: aligned_mesh_path: Path to aligned mesh PLY file - dfy_mesh_path: Path to 3Dfy GLB file + dfy_mesh_path: Path to 3Dfy Object file output_dir: Directory to save combined GLB file (defaults to same dir as aligned_mesh_path) share: Whether to create a public shareable link (default: True) height: Height of the 3D viewer in pixels (default: 600) @@ -307,6 +307,7 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No tuple: (demo, combined_glb_path) - Gradio demo object and path to combined GLB file """ import gradio as gr + import numpy as np print("Loading meshes for interactive visualization...") @@ -315,17 +316,17 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No aligned_mesh = trimesh.load(aligned_mesh_path) print(f"Loaded aligned mesh: {len(aligned_mesh.vertices)} vertices") - # Load 3Dfy mesh (GLB - handle scene structure) + # Load 3Dfy mesh (PLY) dfy_scene = trimesh.load(dfy_mesh_path) - if hasattr(dfy_scene, 'dump'): # It's a scene + if hasattr(dfy_scene, 'dump'): dfy_meshes = [geom for geom in dfy_scene.geometry.values() if hasattr(geom, 'vertices')] if len(dfy_meshes) == 1: dfy_mesh = dfy_meshes[0] elif len(dfy_meshes) > 1: dfy_mesh = trimesh.util.concatenate(dfy_meshes) else: - raise ValueError("No valid meshes in GLB file") + raise ValueError("No valid meshes in PLY file") else: dfy_mesh = dfy_scene @@ -348,14 +349,42 @@ def visualize_meshes_interactive(aligned_mesh_path, dfy_mesh_path, output_dir=No output_dir = os.path.dirname(aligned_mesh_path) os.makedirs(output_dir, exist_ok=True) + # Save combined PLY by concatenating both meshes + combined_ply_path = os.path.join(output_dir, 'combined_scene.ply') + + # Ccombine the geometries for PLY output + if isinstance(dfy_mesh, trimesh.points.PointCloud): + # Convert point cloud to vertices-only mesh for combination + dfy_vertices = dfy_mesh.vertices + human_vertices = aligned_mesh.vertices + + # Combine vertices from both + all_vertices = np.vstack([human_vertices, dfy_vertices]) + + # Create colors: red for human, blue for object + human_colors = np.array([[255, 0, 0, 200]] * len(human_vertices)) + object_colors = np.array([[0, 0, 255, 200]] * len(dfy_vertices)) + all_colors = np.vstack([human_colors, object_colors]) + + # Create combined point cloud + combined_cloud = trimesh.points.PointCloud(vertices=all_vertices, colors=all_colors) + combined_cloud.export(combined_ply_path) + else: + # Both are meshes, use scene export + scene.export(combined_ply_path) + + print(f"Exported combined scene to: {combined_ply_path}") + + # Also save GLB for Gradio viewer (NOTE: GLB may not show point cloud object properly) combined_glb_path = os.path.join(output_dir, 'combined_scene.glb') scene.export(combined_glb_path) - print(f"Exported combined scene to: {combined_glb_path}") + print(f"Exported GLB for Gradio viewer to: {combined_glb_path}") + print("NOTE: Use PLY for complete data, GLB is for Gradio visualization only") # Create interactive Gradio viewer with gr.Blocks() as demo: gr.Markdown("# 3D Mesh Alignment Visualization") - gr.Markdown("**Red**: SAM 3D Body Aligned Human | **Blue**: 3Dfy Object") + gr.Markdown("**Red**: SAM 3D Body Aligned Human | **Blue**: SAM 3D Object") gr.Model3D( value=combined_glb_path, label="Combined 3D Scene (Interactive)", diff --git a/sam3d_objects/pipeline/inference_pipeline.py b/sam3d_objects/pipeline/inference_pipeline.py index e4b81fe..d34d5c2 100644 --- a/sam3d_objects/pipeline/inference_pipeline.py +++ b/sam3d_objects/pipeline/inference_pipeline.py @@ -11,6 +11,8 @@ from torch.utils._pytree import tree_map_only def set_attention_backend(): if torch.cuda.is_available(): gpu_name = torch.cuda.get_device_name(0) + else: + gpu_name = "CPU" logger.info(f"GPU name is {gpu_name}") if "A100" in gpu_name or "H100" in gpu_name or "H200" in gpu_name: