Task/manipulation tests fixes (#1522)

* pyrealsense2 added as manipulation dependency

* renamed manipulation_blueprint to blueprint for name standardization

* added back manipulation client for users to interface without using agent cli

* updated blueprints to run everything in 1 dimos run command

* deleted the unitree-sdk submodule

* added Jointstate type for manipulation client methods

* fix mypy type checking for manipulation client

* fix mymy test

* missed blueprint added

---------

Co-authored-by: stash <pomichterstash@gmail.com>

Author: mustafab0

dimos/hardware/sensors/camera/realsense/camera.py

@@ -44,7 +44,7 @@
 from dimos.utils.reactive import backpressure
 
 if TYPE_CHECKING:
-    import pyrealsense2 as rs  # type: ignore[import-not-found]
+    import pyrealsense2 as rs  # type: ignore[import-untyped,import-not-found]
 
 
 def default_base_transform() -> Transform:
@@ -119,7 +119,7 @@ def __init__(self, *args, **kwargs) -> None:  # type: ignore[no-untyped-def]
 
     @rpc
     def start(self) -> None:
-        import pyrealsense2 as rs  # type: ignore[import-not-found]
+        import pyrealsense2 as rs  # type: ignore[import-untyped,import-not-found]
 
         self._pipeline = rs.pipeline()
         config = rs.config()
@@ -187,7 +187,7 @@ def _publish_camera_info(self) -> None:
             self.depth_camera_info.publish(self._depth_camera_info)
 
     def _build_camera_info(self) -> None:
-        import pyrealsense2 as rs  # type: ignore[import-not-found]
+        import pyrealsense2 as rs  # type: ignore[import-untyped,import-not-found]
 
         if self._profile is None:
             return
@@ -214,7 +214,7 @@ def _build_camera_info(self) -> None:
                 )
 
     def _intrinsics_to_camera_info(self, intrinsics: rs.intrinsics, frame_id: str) -> CameraInfo:
-        import pyrealsense2 as rs  # type: ignore[import-not-found]
+        import pyrealsense2 as rs  # type: ignore[import-untyped,import-not-found]
 
         fx, fy = intrinsics.fx, intrinsics.fy
         cx, cy = intrinsics.ppx, intrinsics.ppy
@@ -243,7 +243,7 @@ def _intrinsics_to_camera_info(self, intrinsics: rs.intrinsics, frame_id: str) -
         )
 
     def _get_extrinsics(self) -> None:
-        import pyrealsense2 as rs  # type: ignore[import-not-found]
+        import pyrealsense2 as rs  # type: ignore[import-untyped,import-not-found]
 
         if self._profile is None or not self.config.enable_depth:
             return

…/manipulation/manipulation_blueprints.py dimos/manipulation/blueprints.pydimos/manipulation/manipulation_blueprints.py renamed to dimos/manipulation/blueprints.py dimos/manipulation/manipulation_blueprints.py renamed to dimos/manipulation/blueprints.py

@@ -13,22 +13,26 @@
 # limitations under the License.
 
 """
-Blueprints for manipulation module integration with ControlCoordinator.
+Manipulation blueprints.
 
-Usage:
-    # Non-agentic (manual RPC):
-    dimos run coordinator-mock
-    dimos run xarm-perception
+Quick start:
+    # 1. Verify manipulation deps load correctly (standalone, no hardware):
+    dimos run xarm6-planner-only
 
-    # Agentic (LLM agent with skills):
-    dimos run coordinator-mock
-    dimos run xarm-perception-agent
+    # 2. Keyboard teleop with mock arm:
+    dimos run keyboard-teleop-xarm7
+
+    # 3. Interactive RPC client (plan, preview, execute from Python):
+    dimos run xarm7-planner-coordinator
+    python -i -m dimos.manipulation.planning.examples.manipulation_client
 """
 
 import math
 from pathlib import Path
 
 from dimos.agents.agent import Agent
+from dimos.control.components import HardwareComponent, HardwareType, make_joints
+from dimos.control.coordinator import TaskConfig, control_coordinator
 from dimos.core.blueprints import autoconnect
 from dimos.core.transport import LCMTransport
 from dimos.hardware.sensors.camera.realsense import realsense_camera
@@ -316,12 +320,35 @@ def _make_piper_config(
 )
 
 
-# Single XArm7 planner for coordinator-mock
-# Usage: dimos run coordinator-mock, then dimos run xarm7-planner-coordinator
-xarm7_planner_coordinator = manipulation_module(
-    robots=[_make_xarm7_config("arm", joint_prefix="arm_", coordinator_task="traj_arm")],
-    planning_timeout=10.0,
-    enable_viz=True,
+# Single XArm7 planner + mock coordinator (standalone, no external coordinator needed)
+# Usage: dimos run xarm7-planner-coordinator
+xarm7_planner_coordinator = autoconnect(
+    manipulation_module(
+        robots=[_make_xarm7_config("arm", joint_prefix="arm_", coordinator_task="traj_arm")],
+        planning_timeout=10.0,
+        enable_viz=True,
+    ),
+    control_coordinator(
+        tick_rate=100.0,
+        publish_joint_state=True,
+        joint_state_frame_id="coordinator",
+        hardware=[
+            HardwareComponent(
+                hardware_id="arm",
+                hardware_type=HardwareType.MANIPULATOR,
+                joints=make_joints("arm", 7),
+                adapter_type="mock",
+            ),
+        ],
+        tasks=[
+            TaskConfig(
+                name="traj_arm",
+                type="trajectory",
+                joint_names=[f"arm_joint{i + 1}" for i in range(7)],
+                priority=10,
+            ),
+        ],
+    ),
 ).transports(
     {
         ("joint_state", JointState): LCMTransport("/coordinator/joint_state", JointState),

dimos/manipulation/planning/README.md

@@ -5,19 +5,21 @@ Motion planning for robotic manipulators. Backend-agnostic design with Drake imp
 ## Quick Start
 
 ```bash
-# Terminal 1: Mock coordinator
-dimos run coordinator-mock
+# 1. Verify manipulation dependencies load correctly (standalone, no hardware):
+dimos run xarm6-planner-only
 
-# Terminal 2: Manipulation planner
-dimos run xarm7-planner-coordinator
+# 2. Keyboard teleop with mock arm (single command):
+dimos run keyboard-teleop-xarm7
 
-# Terminal 3: IPython client
-python -m dimos.manipulation.planning.examples.manipulation_client
+# 3. Interactive RPC client (plan, preview, execute from Python):
+dimos run xarm7-planner-coordinator                                    # terminal 1
+python -i -m dimos.manipulation.planning.examples.manipulation_client  # terminal 2
 ```
 
-In IPython:
+In the interactive client:
 ```python
-joints()                # Get current joints
+commands()              # List available commands
+joints()                # Get current joint positions
 plan([0.1] * 7)         # Plan to target
 preview()               # Preview in Meshcat (url() for link)
 execute()               # Execute via coordinator
@@ -146,8 +148,7 @@ planning/
 ├── monitor/                 # WorldMonitor (live state sync)
 ├── trajectory_generator/    # Time-parameterized trajectories
 └── examples/
-    ├── planning_tester.py        # Standalone CLI tester
-    └── manipulation_client.py    # IPython RPC client
+    └── manipulation_client.py    # Interactive RPC client (python -i)
 ```
 
 ## Obstacle Types

dimos/manipulation/planning/examples/manipulation_client.py

@@ -0,0 +1,187 @@
+# Copyright 2025-2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+RPC client for interacting with a running ManipulationModule.
+
+Usage:
+    # Start a manipulation blueprint in another terminal first:
+    #   dimos run xarm7-planner-coordinator
+    #
+    # Then run this client:
+    python -i -m dimos.manipulation.planning.examples.manipulation_client
+
+Available functions:
+    joints()              Get current joint positions
+    ee()                  Get end-effector pose
+    state()               Get module state (IDLE, PLANNING, EXECUTING, ...)
+    plan(joints)          Plan to joint configuration, e.g. plan([0.1]*7)
+    plan_pose(x,y,z)      Plan to Cartesian pose
+    preview(duration)     Preview planned path in Meshcat
+    execute()             Execute planned trajectory via coordinator
+    home()                Move to home position
+    url()                 Get Meshcat visualization URL
+    robots()              List configured robots
+    info(robot)           Get robot config details
+    gripper(pos)          Set gripper position (0.0=closed, 0.85=open)
+    add_box(name,x,y,z)   Add box obstacle
+    add_sphere(name,x,y,z) Add sphere obstacle
+    add_cylinder(name,x,y,z) Add cylinder obstacle
+    remove(id)            Remove obstacle by ID
+    collision_free(joints) Check if config is collision-free
+"""
+
+# mypy: disable-error-code=no-any-return
+from __future__ import annotations
+
+from typing import Any
+
+from dimos.core.rpc_client import RPCClient
+from dimos.manipulation.manipulation_module import ManipulationModule
+from dimos.msgs.geometry_msgs import Pose, Quaternion, Vector3
+
+_client = RPCClient(None, ManipulationModule)
+
+
+def joints(robot_name: str | None = None) -> list[float] | None:
+    """Get current joint positions."""
+    return _client.get_current_joints(robot_name)
+
+
+def ee(robot_name: str | None = None) -> Pose | None:
+    """Get end-effector pose."""
+    return _client.get_ee_pose(robot_name)
+
+
+def state() -> str:
+    """Get module state."""
+    return _client.get_state()
+
+
+def plan(target_joints: list[float], robot_name: str | None = None) -> bool:
+    """Plan to joint configuration. e.g. plan([0.1]*7)"""
+    from dimos.msgs.sensor_msgs import JointState
+
+    js = JointState(position=target_joints)
+    return _client.plan_to_joints(js, robot_name)
+
+
+def plan_pose(
+    x: float,
+    y: float,
+    z: float,
+    roll: float | None = None,
+    pitch: float | None = None,
+    yaw: float | None = None,
+    robot_name: str | None = None,
+) -> bool:
+    """Plan to Cartesian pose. Preserves current orientation if rpy not given."""
+    orientation = Quaternion(0, 0, 0, 1)
+    if roll is not None or pitch is not None or yaw is not None:
+        orientation = Quaternion.from_euler(Vector3(x=roll or 0, y=pitch or 0, z=yaw or 0))
+    target = Pose(position=Vector3(x=x, y=y, z=z), orientation=orientation)
+    return _client.plan_to_pose(target, robot_name)
+
+
+def preview(duration: float = 3.0, robot_name: str | None = None) -> bool:
+    """Preview planned path in Meshcat."""
+    return _client.preview_path(duration, robot_name)
+
+
+def execute(robot_name: str | None = None) -> bool:
+    """Execute planned trajectory via coordinator."""
+    return _client.execute(robot_name)
+
+
+def home(robot_name: str | None = None) -> bool:
+    """Plan and execute move to home position."""
+    from dimos.msgs.sensor_msgs import JointState
+
+    home_joints = _client.get_robot_info(robot_name).get("home_joints", [0.0] * 7)
+    success = _client.plan_to_joints(JointState(position=home_joints), robot_name)
+    if success:
+        return _client.execute(robot_name)
+    return False
+
+
+def url() -> str | None:
+    """Get Meshcat visualization URL."""
+    return _client.get_visualization_url()
+
+
+def robots() -> list[str]:
+    """List configured robots."""
+    return _client.list_robots()
+
+
+def info(robot_name: str | None = None) -> dict[str, Any] | None:
+    """Get robot config details."""
+    return _client.get_robot_info(robot_name)
+
+
+def gripper(position: float, robot_name: str | None = None) -> str:
+    """Set gripper position (0.0=closed, 0.85=open)."""
+    return _client.set_gripper(position, robot_name)
+
+
+def add_box(
+    name: str, x: float, y: float, z: float, w: float = 0.05, h: float = 0.05, d: float = 0.05
+) -> str | None:
+    """Add a box obstacle. e.g. add_box("cube", 0.3, 0, 0.2)"""
+    pose = Pose(position=Vector3(x=x, y=y, z=z), orientation=Quaternion(0, 0, 0, 1))
+    return _client.add_obstacle(name, pose, "box", [w, h, d], None)
+
+
+def add_sphere(name: str, x: float, y: float, z: float, radius: float = 0.05) -> str | None:
+    """Add a sphere obstacle. e.g. add_sphere("ball", 0.3, 0, 0.2)"""
+    pose = Pose(position=Vector3(x=x, y=y, z=z), orientation=Quaternion(0, 0, 0, 1))
+    return _client.add_obstacle(name, pose, "sphere", [radius], None)
+
+
+def add_cylinder(
+    name: str, x: float, y: float, z: float, radius: float = 0.03, height: float = 0.1
+) -> str | None:
+    """Add a cylinder obstacle. e.g. add_cylinder("can", 0.3, 0, 0.2)"""
+    pose = Pose(position=Vector3(x=x, y=y, z=z), orientation=Quaternion(0, 0, 0, 1))
+    return _client.add_obstacle(name, pose, "cylinder", [radius, height], None)
+
+
+def remove(obstacle_id: str) -> bool:
+    """Remove an obstacle by ID (returned from add_*)."""
+    return _client.remove_obstacle(obstacle_id)
+
+
+def collision_free(target_joints: list[float], robot_name: str | None = None) -> bool:
+    """Check if a joint configuration is collision-free."""
+    return _client.is_collision_free(target_joints, robot_name)
+
+
+def commands() -> None:
+    """Print available functions and raw RPC methods."""
+    print("=== Client Functions ===")
+    for name, obj in sorted(globals().items()):
+        if callable(obj) and not name.startswith("_") and obj.__module__ == __name__:
+            doc = (obj.__doc__ or "").split("\n")[0]
+            print(f"  {name:25s} {doc}")
+
+
+def stop() -> None:
+    """Stop the RPC client."""
+    _client.stop_rpc_client()
+
+
+if __name__ == "__main__":
+    print("Manipulation RPC client ready.")
+    print("Type commands() for available functions.")
+    print("Try: joints(), plan([0.1]*7), preview(), execute()")

dimos/robot/all_blueprints.py

@@ -52,7 +52,7 @@
     "demo-skill": "dimos.agents.skills.demo_skill:demo_skill",
     "drone-agentic": "dimos.robot.drone.blueprints.agentic.drone_agentic:drone_agentic",
     "drone-basic": "dimos.robot.drone.blueprints.basic.drone_basic:drone_basic",
-    "dual-xarm6-planner": "dimos.manipulation.manipulation_blueprints:dual_xarm6_planner",
+    "dual-xarm6-planner": "dimos.manipulation.blueprints:dual_xarm6_planner",
     "keyboard-teleop-piper": "dimos.robot.manipulators.piper.blueprints:keyboard_teleop_piper",
     "keyboard-teleop-xarm6": "dimos.robot.manipulators.xarm.blueprints:keyboard_teleop_xarm6",
     "keyboard-teleop-xarm7": "dimos.robot.manipulators.xarm.blueprints:keyboard_teleop_xarm7",
@@ -86,11 +86,11 @@
     "unitree-go2-spatial": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial:unitree_go2_spatial",
     "unitree-go2-temporal-memory": "dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_temporal_memory:unitree_go2_temporal_memory",
     "unitree-go2-vlm-stream-test": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2_vlm_stream_test:unitree_go2_vlm_stream_test",
-    "xarm-perception": "dimos.manipulation.manipulation_blueprints:xarm_perception",
-    "xarm-perception-agent": "dimos.manipulation.manipulation_blueprints:xarm_perception_agent",
-    "xarm6-planner-only": "dimos.manipulation.manipulation_blueprints:xarm6_planner_only",
-    "xarm7-planner-coordinator": "dimos.manipulation.manipulation_blueprints:xarm7_planner_coordinator",
-    "xarm7-planner-coordinator-agent": "dimos.manipulation.manipulation_blueprints:xarm7_planner_coordinator_agent",
+    "xarm-perception": "dimos.manipulation.blueprints:xarm_perception",
+    "xarm-perception-agent": "dimos.manipulation.blueprints:xarm_perception_agent",
+    "xarm6-planner-only": "dimos.manipulation.blueprints:xarm6_planner_only",
+    "xarm7-planner-coordinator": "dimos.manipulation.blueprints:xarm7_planner_coordinator",
+    "xarm7-planner-coordinator-agent": "dimos.manipulation.blueprints:xarm7_planner_coordinator_agent",
     "xarm7-trajectory-sim": "dimos.simulation.sim_blueprints:xarm7_trajectory_sim",
 }
 

dimos/robot/manipulators/xarm/blueprints.py

@@ -27,7 +27,7 @@
 from dimos.control.coordinator import TaskConfig, control_coordinator
 from dimos.core.blueprints import autoconnect
 from dimos.core.transport import LCMTransport
-from dimos.manipulation.manipulation_blueprints import (
+from dimos.manipulation.blueprints import (
     _make_xarm6_config,
     _make_xarm7_config,
 )

docs/capabilities/manipulation/readme.md

@@ -108,7 +108,7 @@ KeyboardTeleopModule ──→ ControlCoordinator ──→ ManipulationModule
 | File | Description |
 |------|-------------|
 | [`manipulation_module.py`](/dimos/manipulation/manipulation_module.py) | Main module (RPC interface, state machine) |
-| [`manipulation_blueprints.py`](/dimos/manipulation/manipulation_blueprints.py) | Planner and perception blueprints |
+| [`manipulation/blueprints.py`](/dimos/manipulation/blueprints.py) | Planner and perception blueprints |
 | [`robot/manipulators/piper/blueprints.py`](/dimos/robot/manipulators/piper/blueprints.py) | Piper keyboard teleop blueprint |
 | [`robot/manipulators/xarm/blueprints.py`](/dimos/robot/manipulators/xarm/blueprints.py) | XArm keyboard teleop blueprints |
 | [`teleop/keyboard/keyboard_teleop_module.py`](/dimos/teleop/keyboard/keyboard_teleop_module.py) | Keyboard teleop module |