From 4adc0889ba05877c371014860643248b7fbeee26 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 29 Apr 2024 17:24:29 +0200 Subject: [PATCH 01/13] Adding python IDE typing --- apis/python/node/README.md | 9 + apis/python/node/dora/__init__.py | 32 -- apis/python/node/dora/__init__.pyi | 316 +++++++++++ apis/python/node/dora/generate_stubs.py | 512 ++++++++++++++++++ apis/python/node/src/lib.rs | 78 ++- apis/python/operator/src/lib.rs | 12 +- binaries/runtime/src/operator/python.rs | 5 +- libraries/core/src/descriptor/mod.rs | 1 + .../extensions/ros2-bridge/python/src/lib.rs | 108 ++++ .../extensions/ros2-bridge/python/src/qos.rs | 22 +- 10 files changed, 1048 insertions(+), 47 deletions(-) delete mode 100644 apis/python/node/dora/__init__.py create mode 100644 apis/python/node/dora/__init__.pyi create mode 100644 apis/python/node/dora/generate_stubs.py diff --git a/apis/python/node/README.md b/apis/python/node/README.md index 5909eebe..fd1d844f 100644 --- a/apis/python/node/README.md +++ b/apis/python/node/README.md @@ -10,3 +10,12 @@ source .env/bin/activate pip install maturin maturin develop ``` + +## Type hinting + +Type hinting requires to run a second step + +```bash +python dora/generate_stubs.py dora dora/__init__.pyi +maturin develop +``` diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py deleted file mode 100644 index be8c5016..00000000 --- a/apis/python/node/dora/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -# dora-rs - -This is the dora python client for interacting with dora dataflow. - -You can install it via: - -```bash -pip install dora-rs -``` -""" - -from enum import Enum - -from .dora import * - -__author__ = "Dora-rs Authors" -__version__ = "0.3.3" - - -class DoraStatus(Enum): - """Dora status to indicate if operator `on_input` loop - should be stopped. - - Args: - Enum (u8): Status signaling to dora operator to - stop or continue the operator. - """ - - CONTINUE = 0 - STOP = 1 - STOP_ALL = 2 diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi new file mode 100644 index 00000000..3014cc35 --- /dev/null +++ b/apis/python/node/dora/__init__.pyi @@ -0,0 +1,316 @@ +import pyarrow +import typing + +@typing.final +class DoraStatus: + + def __eq__(self, value: typing.Any, /) -> bool: + """Return self==value.""" + + def __ge__(self, value: typing.Any, /) -> bool: + """Return self>=value.""" + + def __gt__(self, value: typing.Any, /) -> bool: + """Return self>value.""" + + def __int__(self, /) -> None: + """int(self)""" + + def __le__(self, value: typing.Any, /) -> bool: + """Return self<=value.""" + + def __lt__(self, value: typing.Any, /) -> bool: + """Return self bool: + """Return self!=value.""" + + def __repr__(self, /) -> str: + """Return repr(self).""" + CONTINUE: DoraStatus = ... + STOP: DoraStatus = ... + STOP_ALL: DoraStatus = ... + +@typing.final +class Node: + """The custom node API lets you integrate `dora` into your application. +It allows you to retrieve input and send output in any fashion you want. + +Use with: + +```python +from dora import Node + +node = Node() +```""" + + def __init__(self, /) -> None: + """The custom node API lets you integrate `dora` into your application. +It allows you to retrieve input and send output in any fashion you want. + +Use with: + +```python +from dora import Node + +node = Node() +```""" + + def dataflow_descriptor(self, /) -> dict:... + + def merge_external_events(self, /, subscription: Ros2Subscription) -> None:... + + def next(self, /, timeout: float=None) -> PyEvent: + """`.next()` gives you the next input that the node has received. +It blocks until the next event becomes available. +You can use timeout in seconds to return if no input is available. +It will return `None` when all senders has been dropped. + +```python +event = node.next() +``` + +You can also iterate over the event stream with a loop + +```python +for event in node: +match event["type"]: +case "INPUT": +match event["id"]: +case "image": +```""" + + def send_output(self, /, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None: + """`send_output` send data from the node. + +```python +Args: +output_id: str, +data: Bytes|Arrow, +metadata: Option[Dict], +``` + +```python +node.send_output("string", b"string", {"open_telemetry_context": "7632e76"}) +```""" + + def __iter__(self, /) -> typing.Any: + """Implement iter(self).""" + + def __next__(self, /) -> typing.Any: + """Implement next(self).""" + +@typing.final +class PyEvent: + + def inner(self, /):... + + def __getitem__(self, key: typing.Any, /) -> typing.Any: + """Return self[key].""" + +@typing.final +class Ros2Context: + """ROS2 Context holding all messages definition for receiving and sending messages to ROS2. + +By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition. + +AMENT_PREFIX_PATH folder structure should be the following: + +- For messages: /msg/.msg +- For services: /srv/.srv + +You can also use `ros_paths` if you don't want to use env variable. + +```python +context = Ros2Context() +``` + +list of paths to search for ROS2 message types defintion""" + + def __init__(self, /, ros_paths: List[str]=None) -> None: + """ROS2 Context holding all messages definition for receiving and sending messages to ROS2. + +By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition. + +AMENT_PREFIX_PATH folder structure should be the following: + +- For messages: /msg/.msg +- For services: /srv/.srv + +You can also use `ros_paths` if you don't want to use env variable. + +```python +context = Ros2Context() +``` + +list of paths to search for ROS2 message types defintion""" + + def new_node(self, /, name: str, namespace: str, options: Ros2NodeOptions) -> Ros2Node: + """Create a new ROS2 node + +```python +ros2_node = ros2_context.new_node( +"turtle_teleop", +"/ros2_demo", +dora.experimental.ros2_bridge.Ros2NodeOptions(rosout=True), +) +``` + +name of the node +name of the namespace +options for the node""" + +@typing.final +class Ros2Durability: + """DDS 2.2.3.4 DURABILITY""" + + def __eq__(self, value: typing.Any, /) -> bool: + """Return self==value.""" + + def __ge__(self, value: typing.Any, /) -> bool: + """Return self>=value.""" + + def __gt__(self, value: typing.Any, /) -> bool: + """Return self>value.""" + + def __int__(self, /) -> None: + """int(self)""" + + def __le__(self, value: typing.Any, /) -> bool: + """Return self<=value.""" + + def __lt__(self, value: typing.Any, /) -> bool: + """Return self bool: + """Return self!=value.""" + + def __repr__(self, /) -> str: + """Return repr(self).""" + Persistent: Ros2Durability = ... + Transient: Ros2Durability = ... + TransientLocal: Ros2Durability = ... + Volatile: Ros2Durability = ... + +@typing.final +class Ros2Liveliness: + """DDS 2.2.3.11 LIVELINESS""" + + def __eq__(self, value: typing.Any, /) -> bool: + """Return self==value.""" + + def __ge__(self, value: typing.Any, /) -> bool: + """Return self>=value.""" + + def __gt__(self, value: typing.Any, /) -> bool: + """Return self>value.""" + + def __int__(self, /) -> None: + """int(self)""" + + def __le__(self, value: typing.Any, /) -> bool: + """Return self<=value.""" + + def __lt__(self, value: typing.Any, /) -> bool: + """Return self bool: + """Return self!=value.""" + + def __repr__(self, /) -> str: + """Return repr(self).""" + Automatic: Ros2Liveliness = ... + ManualByParticipant: Ros2Liveliness = ... + ManualByTopic: Ros2Liveliness = ... + +@typing.final +class Ros2Node: + """ROS2 Node + +Warnings: +- There's a known issue about ROS2 nodes not being discoverable by ROS2 +See: https://github.com/jhelovuo/ros2-client/issues/4""" + + def create_publisher(self, /, topic: Ros2Topic, qos: Ros2QosPolicies=None) -> Ros2Publisher: + """Create a ROS2 publisher + +```python +pose_publisher = ros2_node.create_publisher(turtle_pose_topic) +``` + +QoS policies for the topic""" + + def create_subscription(self, /, topic: Ros2Topic, qos: Ros2QosPolicies=None) -> Ros2Subscription: + """Create a ROS2 subscription + +```python +pose_reader = ros2_node.create_subscription(turtle_pose_topic) +``` + +QoS policies for the topic""" + + def create_topic(self, /, name: str, message_type: str, qos: Ros2QosPolicies) -> Ros2Topic: + """Create a ROS2 topic to connect to. + +```python +turtle_twist_topic = ros2_node.create_topic( +"/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos +) +``` + +name of the topic. e.g. "pose" +message type of the topic. e.g. "std_msgs::UInt8MultiArray" +QoS policies for the topic""" + +@typing.final +class Ros2NodeOptions: + """ROS2 Node Options +enable rosout logging""" + + def __init__(self, /, rosout: bool=None) -> None: + """ROS2 Node Options +enable rosout logging""" + +@typing.final +class Ros2Publisher: + """ROS2 Publisher""" + + def publish(self, /, data: pyarrow.Array) -> None: + """Publish a message into ROS2 topic. + +Remember that the data format should respect the structure of the ROS2 message usinng an arrow Structure. + +ex: +```python +gripper_command.publish( +pa.array( +[ +{ +"name": "gripper", +"cmd": np.float32(5), +} +] +), +) +```""" + +@typing.final +class Ros2QosPolicies: + """ROS2 QoS Policy""" + + def __init__(self, /, durability: Ros2Durability=None, liveliness: Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> Ros2QoSPolicies: + """ROS2 QoS Policy""" + +@typing.final +class Ros2Subscription: + """ROS2 Subscription""" + + def next(self, /):... + +@typing.final +class Ros2Topic: + """ROS2 Topic +enable rosout logging""" + +def start_runtime() -> None: + """Start a runtime for Operators""" \ No newline at end of file diff --git a/apis/python/node/dora/generate_stubs.py b/apis/python/node/dora/generate_stubs.py new file mode 100644 index 00000000..47723489 --- /dev/null +++ b/apis/python/node/dora/generate_stubs.py @@ -0,0 +1,512 @@ +import argparse +import ast +import importlib +import inspect +import logging +import re +import subprocess +from functools import reduce +from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union + + +def path_to_type(*elements: str) -> ast.AST: + base: ast.AST = ast.Name(id=elements[0], ctx=ast.Load()) + for e in elements[1:]: + base = ast.Attribute(value=base, attr=e, ctx=ast.Load()) + return base + + +OBJECT_MEMBERS = dict(inspect.getmembers(object)) +BUILTINS: Dict[str, Union[None, Tuple[List[ast.AST], ast.AST]]] = { + "__annotations__": None, + "__bool__": ([], path_to_type("bool")), + "__bytes__": ([], path_to_type("bytes")), + "__class__": None, + "__contains__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__del__": None, + "__delattr__": ([path_to_type("str")], path_to_type("None")), + "__delitem__": ([path_to_type("typing", "Any")], path_to_type("typing", "Any")), + "__dict__": None, + "__dir__": None, + "__doc__": None, + "__eq__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__format__": ([path_to_type("str")], path_to_type("str")), + "__ge__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__getattribute__": ([path_to_type("str")], path_to_type("typing", "Any")), + "__getitem__": ([path_to_type("typing", "Any")], path_to_type("typing", "Any")), + "__gt__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__hash__": ([], path_to_type("int")), + "__init__": ([], path_to_type("None")), + "__init_subclass__": None, + "__iter__": ([], path_to_type("typing", "Any")), + "__le__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__len__": ([], path_to_type("int")), + "__lt__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__module__": None, + "__ne__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__new__": None, + "__next__": ([], path_to_type("typing", "Any")), + "__int__": ([], path_to_type("None")), + "__reduce__": None, + "__reduce_ex__": None, + "__repr__": ([], path_to_type("str")), + "__setattr__": ( + [path_to_type("str"), path_to_type("typing", "Any")], + path_to_type("None"), + ), + "__setitem__": ( + [path_to_type("typing", "Any"), path_to_type("typing", "Any")], + path_to_type("typing", "Any"), + ), + "__sizeof__": None, + "__str__": ([], path_to_type("str")), + "__subclasshook__": None, +} + + +def module_stubs(module: Any) -> ast.Module: + types_to_import = {"typing"} + classes = [] + functions = [] + for member_name, member_value in inspect.getmembers(module): + element_path = [module.__name__, member_name] + if member_name.startswith("__"): + pass + elif inspect.isclass(member_value): + classes.append( + class_stubs(member_name, member_value, element_path, types_to_import) + ) + elif inspect.isbuiltin(member_value): + functions.append( + function_stub( + member_name, + member_value, + element_path, + types_to_import, + in_class=False, + ) + ) + else: + logging.warning(f"Unsupported root construction {member_name}") + return ast.Module( + body=[ast.Import(names=[ast.alias(name=t)]) for t in sorted(types_to_import)] + + classes + + functions, + type_ignores=[], + ) + + +def class_stubs( + cls_name: str, cls_def: Any, element_path: List[str], types_to_import: Set[str] +) -> ast.ClassDef: + attributes: List[ast.AST] = [] + methods: List[ast.AST] = [] + magic_methods: List[ast.AST] = [] + constants: List[ast.AST] = [] + for member_name, member_value in inspect.getmembers(cls_def): + current_element_path = [*element_path, member_name] + if member_name == "__init__": + try: + inspect.signature(cls_def) # we check it actually exists + methods = [ + function_stub( + member_name, + cls_def, + current_element_path, + types_to_import, + in_class=True, + ), + *methods, + ] + except ValueError as e: + if "no signature found" not in str(e): + raise ValueError( + f"Error while parsing signature of {cls_name}.__init_" + ) from e + elif ( + member_value == OBJECT_MEMBERS.get(member_name) + or BUILTINS.get(member_name, ()) is None + ): + pass + elif inspect.isdatadescriptor(member_value): + attributes.extend( + data_descriptor_stub( + member_name, member_value, current_element_path, types_to_import + ) + ) + elif inspect.isroutine(member_value): + (magic_methods if member_name.startswith("__") else methods).append( + function_stub( + member_name, + member_value, + current_element_path, + types_to_import, + in_class=True, + ) + ) + elif member_name == "__match_args__": + constants.append( + ast.AnnAssign( + target=ast.Name(id=member_name, ctx=ast.Store()), + annotation=ast.Subscript( + value=path_to_type("tuple"), + slice=ast.Tuple( + elts=[path_to_type("str"), ast.Ellipsis()], ctx=ast.Load() + ), + ctx=ast.Load(), + ), + value=ast.Constant(member_value), + simple=1, + ) + ) + elif member_value is not None: + constants.append( + ast.AnnAssign( + target=ast.Name(id=member_name, ctx=ast.Store()), + annotation=concatenated_path_to_type( + member_value.__class__.__name__, element_path, types_to_import + ), + value=ast.Ellipsis(), + simple=1, + ) + ) + else: + logging.warning( + f"Unsupported member {member_name} of class {'.'.join(element_path)}" + ) + + doc = inspect.getdoc(cls_def) + doc_comment = build_doc_comment(doc) if doc else None + return ast.ClassDef( + cls_name, + bases=[], + keywords=[], + body=( + ([doc_comment] if doc_comment else []) + + attributes + + methods + + magic_methods + + constants + ) + or [ast.Ellipsis()], + decorator_list=[path_to_type("typing", "final")], + ) + + +def data_descriptor_stub( + data_desc_name: str, + data_desc_def: Any, + element_path: List[str], + types_to_import: Set[str], +) -> Union[Tuple[ast.AnnAssign, ast.Expr], Tuple[ast.AnnAssign]]: + annotation = None + doc_comment = None + + doc = inspect.getdoc(data_desc_def) + if doc is not None: + annotation = returns_stub(data_desc_name, doc, element_path, types_to_import) + m = re.findall(r"^ *:return: *(.*) *$", doc, re.MULTILINE) + if len(m) == 1: + doc_comment = m[0] + elif len(m) > 1: + raise ValueError( + f"Multiple return annotations found with :return: in {'.'.join(element_path)} documentation" + ) + + assign = ast.AnnAssign( + target=ast.Name(id=data_desc_name, ctx=ast.Store()), + annotation=annotation or path_to_type("typing", "Any"), + simple=1, + ) + doc_comment = build_doc_comment(doc_comment) if doc_comment else None + return (assign, doc_comment) if doc_comment else (assign,) + + +def function_stub( + fn_name: str, + fn_def: Any, + element_path: List[str], + types_to_import: Set[str], + *, + in_class: bool, +) -> ast.FunctionDef: + body: List[ast.AST] = [] + doc = inspect.getdoc(fn_def) + if doc is not None: + doc_comment = build_doc_comment(doc) + if doc_comment is not None: + body.append(doc_comment) + + decorator_list = [] + if in_class and hasattr(fn_def, "__self__"): + decorator_list.append(ast.Name("staticmethod")) + + return ast.FunctionDef( + fn_name, + arguments_stub(fn_name, fn_def, doc or "", element_path, types_to_import), + body or [ast.Ellipsis()], + decorator_list=decorator_list, + returns=( + returns_stub(fn_name, doc, element_path, types_to_import) if doc else None + ), + lineno=0, + ) + + +def arguments_stub( + callable_name: str, + callable_def: Any, + doc: str, + element_path: List[str], + types_to_import: Set[str], +) -> ast.arguments: + real_parameters: Mapping[str, inspect.Parameter] = inspect.signature( + callable_def + ).parameters + if callable_name == "__init__": + real_parameters = { + "self": inspect.Parameter("self", inspect.Parameter.POSITIONAL_ONLY), + **real_parameters, + } + + parsed_param_types = {} + optional_params = set() + + # Types for magic functions types + builtin = BUILTINS.get(callable_name) + if isinstance(builtin, tuple): + param_names = list(real_parameters.keys()) + if param_names and param_names[0] == "self": + del param_names[0] + for name, t in zip(param_names, builtin[0]): + parsed_param_types[name] = t + + # Types from comment + for match in re.findall( + r"^ *:type *([a-zA-Z0-9_]+): ([^\n]*) *$", doc, re.MULTILINE + ): + if match[0] not in real_parameters: + raise ValueError( + f"The parameter {match[0]} of {'.'.join(element_path)} " + "is defined in the documentation but not in the function signature" + ) + type = match[1] + if type.endswith(", optional"): + optional_params.add(match[0]) + type = type[:-10] + parsed_param_types[match[0]] = convert_type_from_doc( + type, element_path, types_to_import + ) + + # we parse the parameters + posonlyargs = [] + args = [] + vararg = None + kwonlyargs = [] + kw_defaults = [] + kwarg = None + defaults = [] + for param in real_parameters.values(): + if param.name != "self" and param.name not in parsed_param_types: + raise ValueError( + f"The parameter {param.name} of {'.'.join(element_path)} " + "has no type definition in the function documentation" + ) + param_ast = ast.arg( + arg=param.name, annotation=parsed_param_types.get(param.name) + ) + + default_ast = None + if param.default != param.empty: + default_ast = ast.Constant(param.default) + if param.name not in optional_params: + raise ValueError( + f"Parameter {param.name} of {'.'.join(element_path)} " + "is optional according to the type but not flagged as such in the doc" + ) + elif param.name in optional_params: + raise ValueError( + f"Parameter {param.name} of {'.'.join(element_path)} " + "is optional according to the documentation but has no default value" + ) + + if param.kind == param.POSITIONAL_ONLY: + posonlyargs.append(param_ast) + defaults.append(default_ast) + elif param.kind == param.POSITIONAL_OR_KEYWORD: + args.append(param_ast) + defaults.append(default_ast) + elif param.kind == param.VAR_POSITIONAL: + vararg = param_ast + elif param.kind == param.KEYWORD_ONLY: + kwonlyargs.append(param_ast) + kw_defaults.append(default_ast) + elif param.kind == param.VAR_KEYWORD: + kwarg = param_ast + + return ast.arguments( + posonlyargs=posonlyargs, + args=args, + vararg=vararg, + kwonlyargs=kwonlyargs, + kw_defaults=kw_defaults, + defaults=defaults, + kwarg=kwarg, + ) + + +def returns_stub( + callable_name: str, doc: str, element_path: List[str], types_to_import: Set[str] +) -> Optional[ast.AST]: + m = re.findall(r"^ *:rtype: *([^\n]*) *$", doc, re.MULTILINE) + if len(m) == 0: + builtin = BUILTINS.get(callable_name) + if isinstance(builtin, tuple) and builtin[1] is not None: + return builtin[1] + raise ValueError( + f"The return type of {'.'.join(element_path)} " + "has no type definition using :rtype: in the function documentation" + ) + if len(m) > 1: + raise ValueError( + f"Multiple return type annotations found with :rtype: for {'.'.join(element_path)}" + ) + return convert_type_from_doc(m[0], element_path, types_to_import) + + +def convert_type_from_doc( + type_str: str, element_path: List[str], types_to_import: Set[str] +) -> ast.AST: + type_str = type_str.strip() + return parse_type_to_ast(type_str, element_path, types_to_import) + + +def parse_type_to_ast( + type_str: str, element_path: List[str], types_to_import: Set[str] +) -> ast.AST: + # let's tokenize + tokens = [] + current_token = "" + for c in type_str: + if "a" <= c <= "z" or "A" <= c <= "Z" or c == ".": + current_token += c + else: + if current_token: + tokens.append(current_token) + current_token = "" + if c != " ": + tokens.append(c) + if current_token: + tokens.append(current_token) + + # let's first parse nested parenthesis + stack: List[List[Any]] = [[]] + for token in tokens: + if token == "[": + children: List[str] = [] + stack[-1].append(children) + stack.append(children) + elif token == "]": + stack.pop() + else: + stack[-1].append(token) + + # then it's easy + def parse_sequence(sequence: List[Any]) -> ast.AST: + # we split based on "or" + or_groups: List[List[str]] = [[]] + print(sequence) + # TODO: Fix sequence + if "Ros" in sequence and "2" in sequence: + sequence = ["".join(sequence)] + + for e in sequence: + if e == "or": + or_groups.append([]) + else: + or_groups[-1].append(e) + if any(not g for g in or_groups): + raise ValueError( + f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}" + ) + + new_elements: List[ast.AST] = [] + for group in or_groups: + if len(group) == 1 and isinstance(group[0], str): + new_elements.append( + concatenated_path_to_type(group[0], element_path, types_to_import) + ) + elif ( + len(group) == 2 + and isinstance(group[0], str) + and isinstance(group[1], list) + ): + new_elements.append( + ast.Subscript( + value=concatenated_path_to_type( + group[0], element_path, types_to_import + ), + slice=parse_sequence(group[1]), + ctx=ast.Load(), + ) + ) + else: + raise ValueError( + f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}" + ) + return reduce( + lambda left, right: ast.BinOp(left=left, op=ast.BitOr(), right=right), + new_elements, + ) + + return parse_sequence(stack[0]) + + +def concatenated_path_to_type( + path: str, element_path: List[str], types_to_import: Set[str] +) -> ast.AST: + parts = path.split(".") + if any(not p for p in parts): + raise ValueError( + f"Not able to parse type '{path}' used by {'.'.join(element_path)}" + ) + if len(parts) > 1: + types_to_import.add(".".join(parts[:-1])) + return path_to_type(*parts) + + +def build_doc_comment(doc: str) -> Optional[ast.Expr]: + lines = [line.strip() for line in doc.split("\n")] + clean_lines = [] + for line in lines: + if line.startswith((":type", ":rtype")): + continue + clean_lines.append(line) + text = "\n".join(clean_lines).strip() + return ast.Expr(value=ast.Constant(text)) if text else None + + +def format_with_ruff(file: str) -> None: + subprocess.check_call(["python", "-m", "ruff", "format", file]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Extract Python type stub from a python module." + ) + parser.add_argument( + "module_name", help="Name of the Python module for which generate stubs" + ) + parser.add_argument( + "out", + help="Name of the Python stub file to write to", + type=argparse.FileType("wt"), + ) + parser.add_argument( + "--ruff", help="Formats the generated stubs using Ruff", action="store_true" + ) + args = parser.parse_args() + stub_content = ast.unparse(module_stubs(importlib.import_module(args.module_name))) + args.out.write(stub_content) + if args.ruff: + format_with_ruff(args.out.name) diff --git a/apis/python/node/src/lib.rs b/apis/python/node/src/lib.rs index 0d6714f7..3bc03ebc 100644 --- a/apis/python/node/src/lib.rs +++ b/apis/python/node/src/lib.rs @@ -43,6 +43,7 @@ impl Node { /// `.next()` gives you the next input that the node has received. /// It blocks until the next event becomes available. + /// You can use timeout in seconds to return if no input is available. /// It will return `None` when all senders has been dropped. /// /// ```python @@ -58,17 +59,41 @@ impl Node { /// match event["id"]: /// case "image": /// ``` + /// :type timeout: float, optional + /// :rtype: PyEvent #[allow(clippy::should_implement_trait)] pub fn next(&mut self, py: Python, timeout: Option) -> PyResult> { let event = py.allow_threads(|| self.events.recv(timeout.map(Duration::from_secs_f32))); Ok(event) } + /// You can iterate over the event stream with a loop + /// + /// ```python + /// for event in node: + /// match event["type"]: + /// case "INPUT": + /// match event["id"]: + /// case "image": + /// ``` + /// + /// :rtype: PyEvent pub fn __next__(&mut self, py: Python) -> PyResult> { let event = py.allow_threads(|| self.events.recv(None)); Ok(event) } + /// You can iterate over the event stream with a loop + /// + /// ```python + /// for event in node: + /// match event["type"]: + /// case "INPUT": + /// match event["id"]: + /// case "image": + /// ``` + /// + /// :rtype: PyEvent fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -85,7 +110,10 @@ impl Node { /// ```python /// node.send_output("string", b"string", {"open_telemetry_context": "7632e76"}) /// ``` - /// + /// :type output_id: str + /// :type data: pyarrow.Array + /// :type metadata: dict, optional + /// :rtype: None pub fn send_output( &mut self, output_id: String, @@ -113,13 +141,20 @@ impl Node { Ok(()) } - /// Returns the full dataflow descriptor that this node is part of. + // Returns the full dataflow descriptor that this node is part of. + // + // This method returns the parsed dataflow YAML file. /// - /// This method returns the parsed dataflow YAML file. + /// :rtype: dict pub fn dataflow_descriptor(&self, py: Python) -> pythonize::Result { pythonize::pythonize(py, self.node.dataflow_descriptor()) } + // Merge an external event stream with dora main loop. + // This currently only work with ROS2. + /// + /// :type subscription: Ros2Subscription + /// :rtype: None pub fn merge_external_events( &mut self, subscription: &mut Ros2Subscription, @@ -197,21 +232,42 @@ impl Node { } /// Start a runtime for Operators +/// +/// :rtype: None #[pyfunction] pub fn start_runtime() -> eyre::Result<()> { dora_runtime::main().wrap_err("Dora Runtime raised an error.") } +/// :rtype: DoraStatus +#[derive(Copy, Clone, Debug, PartialEq)] +#[pyclass] +pub enum DoraStatus { + CONTINUE, + STOP, + STOP_ALL, +} + +impl DoraStatus { + /// :rtype: int + pub fn value(self) -> i32 { + match self { + DoraStatus::CONTINUE => 0, + DoraStatus::STOP => 1, + DoraStatus::STOP_ALL => 2, + } + } +} + #[pymodule] -fn dora(py: Python, m: &PyModule) -> PyResult<()> { +fn dora(_py: Python, m: &PyModule) -> PyResult<()> { + dora_ros2_bridge_python::create_dora_ros2_bridge_module(m)?; m.add_function(wrap_pyfunction!(start_runtime, m)?)?; - m.add_class::().unwrap(); - - let ros2_bridge = PyModule::new(py, "ros2_bridge")?; - dora_ros2_bridge_python::create_dora_ros2_bridge_module(ros2_bridge)?; - let experimental = PyModule::new(py, "experimental")?; - experimental.add_submodule(ros2_bridge)?; - m.add_submodule(experimental)?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.setattr("__version__", env!("CARGO_PKG_VERSION"))?; + m.setattr("__author__", "Dora-rs Authors")?; Ok(()) } diff --git a/apis/python/operator/src/lib.rs b/apis/python/operator/src/lib.rs index f4712e92..d4c3e475 100644 --- a/apis/python/operator/src/lib.rs +++ b/apis/python/operator/src/lib.rs @@ -3,6 +3,7 @@ use dora_node_api::{merged::MergedEvent, Event, Metadata, MetadataParameters}; use eyre::{Context, Result}; use pyo3::{exceptions::PyLookupError, prelude::*, types::PyDict}; +// Dora Event #[pyclass] pub struct PyEvent { event: MergedEvent, @@ -11,6 +12,8 @@ pub struct PyEvent { #[pymethods] impl PyEvent { + /// + /// :rtype: PyObject pub fn __getitem__(&self, key: &str, py: Python<'_>) -> PyResult> { if key == "kind" { let kind = match &self.event { @@ -35,7 +38,14 @@ impl PyEvent { }; Ok(value) } - MergedEvent::External(event) => event.call_method1(py, "__getitem__", (key,)).map(Some), + MergedEvent::External(event) => { + let value = match key { + "value" => event, + _ => todo!(), + }; + + Ok(Some(value.clone())) + } } } diff --git a/binaries/runtime/src/operator/python.rs b/binaries/runtime/src/operator/python.rs index 8ebbbfc2..fefc92a4 100644 --- a/binaries/runtime/src/operator/python.rs +++ b/binaries/runtime/src/operator/python.rs @@ -224,8 +224,9 @@ pub fn run( .map_err(traceback); match status_enum { Ok(status_enum) => { - let status_val = Python::with_gil(|py| status_enum.getattr(py, "value")) - .wrap_err("on_event must have enum return value")?; + let status_val = + Python::with_gil(|py| status_enum.call_method0(py, "value")) + .wrap_err("on_event must have enum return value")?; Python::with_gil(|py| status_val.extract(py)) .wrap_err("on_event has invalid return value") } diff --git a/libraries/core/src/descriptor/mod.rs b/libraries/core/src/descriptor/mod.rs index ad4b7757..d42a3be8 100644 --- a/libraries/core/src/descriptor/mod.rs +++ b/libraries/core/src/descriptor/mod.rs @@ -17,6 +17,7 @@ mod validate; mod visualize; pub const SHELL_SOURCE: &str = "shell"; +/// Dataflow description #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Descriptor { diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index a07a7fbe..a0107b58 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -23,6 +23,24 @@ use typed::{deserialize::StructDeserializer, TypeInfo, TypedValue}; pub mod qos; pub mod typed; +/// ROS2 Context holding all messages definition for receiving and sending messages to ROS2. +/// +/// By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition. +/// +/// AMENT_PREFIX_PATH folder structure should be the following: +/// +/// - For messages: /msg/.msg +/// - For services: /srv/.srv +/// +/// You can also use `ros_paths` if you don't want to use env variable. +/// +/// ```python +/// context = Ros2Context() +/// ``` +/// +/// :type ros_paths: List[str], optional +/// list of paths to search for ROS2 message types defintion +/// #[pyclass] pub struct Ros2Context { context: ros2_client::Context, @@ -31,6 +49,7 @@ pub struct Ros2Context { #[pymethods] impl Ros2Context { + /// Create a new context #[new] pub fn new(ros_paths: Option>) -> eyre::Result { let ament_prefix_path = std::env::var("AMENT_PREFIX_PATH"); @@ -72,6 +91,22 @@ impl Ros2Context { } /// Create a new ROS2 node + /// + /// ```python + /// ros2_node = ros2_context.new_node( + /// "turtle_teleop", + /// "/ros2_demo", + /// dora.experimental.ros2_bridge.Ros2NodeOptions(rosout=True), + /// ) + /// ``` + /// + /// :type name: str + /// name of the node + /// :type namespace: str + /// name of the namespace + /// :type options: Ros2NodeOptions + /// options for the node + /// :rtype: Ros2Node pub fn new_node( &self, name: &str, @@ -90,6 +125,12 @@ impl Ros2Context { } } +/// ROS2 Node +/// +/// Warnings: +/// - There's a known issue about ROS2 nodes not being discoverable by ROS2 +/// See: https://github.com/jhelovuo/ros2-client/issues/4 +/// #[pyclass] pub struct Ros2Node { node: ros2_client::Node, @@ -98,6 +139,21 @@ pub struct Ros2Node { #[pymethods] impl Ros2Node { + /// Create a ROS2 topic to connect to. + /// + /// ```python + /// turtle_twist_topic = ros2_node.create_topic( + /// "/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos + /// ) + /// ``` + /// + /// :type name: str + /// name of the topic. e.g. "pose" + /// :type message_type: str + /// message type of the topic. e.g. "std_msgs::UInt8MultiArray" + /// :type qos: Ros2QosPolicies + /// QoS policies for the topic + /// :rtype: Ros2Topic pub fn create_topic( &self, name: &str, @@ -126,6 +182,16 @@ impl Ros2Node { Ok(Ros2Topic { topic, type_info }) } + /// Create a ROS2 publisher + /// + /// ```python + /// pose_publisher = ros2_node.create_publisher(turtle_pose_topic) + /// ``` + /// + /// :type topic: Ros2Topic + /// :type qos: Ros2QosPolicies, optional + /// QoS policies for the topic + /// :rtype: Ros2Publisher pub fn create_publisher( &mut self, topic: &Ros2Topic, @@ -140,6 +206,16 @@ impl Ros2Node { }) } + /// Create a ROS2 subscription + /// + /// ```python + /// pose_reader = ros2_node.create_subscription(turtle_pose_topic) + /// ``` + /// + /// :type topic: Ros2Topic + /// :type qos: Ros2QosPolicies, optional + /// QoS policies for the topic + /// :rtype: Ros2Subscription pub fn create_subscription( &mut self, topic: &Ros2Topic, @@ -155,6 +231,10 @@ impl Ros2Node { } } +/// ROS2 Node Options +/// :type rosout: bool, optional +/// enable rosout logging +/// #[derive(Debug, Clone, Default)] #[pyclass] #[non_exhaustive] @@ -178,6 +258,10 @@ impl From for ros2_client::NodeOptions { } } +/// ROS2 Topic +/// :type rosout: bool, optional +/// enable rosout logging +/// #[pyclass] #[non_exhaustive] pub struct Ros2Topic { @@ -185,6 +269,7 @@ pub struct Ros2Topic { type_info: TypeInfo<'static>, } +/// ROS2 Publisher #[pyclass] #[non_exhaustive] pub struct Ros2Publisher { @@ -194,6 +279,27 @@ pub struct Ros2Publisher { #[pymethods] impl Ros2Publisher { + /// Publish a message into ROS2 topic. + /// + /// Remember that the data format should respect the structure of the ROS2 message usinng an arrow Structure. + /// + /// ex: + /// ```python + /// gripper_command.publish( + /// pa.array( + /// [ + /// { + /// "name": "gripper", + /// "cmd": np.float32(5), + /// } + /// ] + /// ), + /// ) + /// ``` + /// + /// :type data: pyarrow.Array + /// :rtype: None + /// pub fn publish(&self, data: &PyAny) -> eyre::Result<()> { let pyarrow = PyModule::import(data.py(), "pyarrow")?; @@ -228,6 +334,8 @@ impl Ros2Publisher { } } +/// ROS2 Subscription +/// #[pyclass] #[non_exhaustive] pub struct Ros2Subscription { diff --git a/libraries/extensions/ros2-bridge/python/src/qos.rs b/libraries/extensions/ros2-bridge/python/src/qos.rs index 398cfe5f..1421dfb5 100644 --- a/libraries/extensions/ros2-bridge/python/src/qos.rs +++ b/libraries/extensions/ros2-bridge/python/src/qos.rs @@ -1,8 +1,19 @@ use ::dora_ros2_bridge::rustdds::{self, policy}; use pyo3::prelude::{pyclass, pymethods}; +/// ROS2 QoS Policy +/// +/// :type durability: Ros2Durability, optional +/// :type liveliness: Ros2Liveliness, optional +/// :type reliable: bool, optional +/// :type keep_all: bool, optional +/// :type lease_duration: float, optional +/// :type max_blocking_time: float, optional +/// :type keep_last: int, optional +/// :rtype: Ros2QoSPolicies +/// #[derive(Debug, Clone)] -#[pyclass(get_all, set_all)] +#[pyclass] #[non_exhaustive] pub struct Ros2QosPolicies { pub durability: Ros2Durability, @@ -64,6 +75,8 @@ impl From for rustdds::QosPolicies { } /// DDS 2.2.3.4 DURABILITY +/// +/// :rtype: Ros2Durability #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[pyclass] pub enum Ros2Durability { @@ -73,7 +86,11 @@ pub enum Ros2Durability { Persistent, } +/// :type value: Ros2Durability +/// :rtype: Ros2Durability impl From for policy::Durability { + /// :type value: Ros2Durability + /// :rtype: Ros2Durability fn from(value: Ros2Durability) -> Self { match value { Ros2Durability::Volatile => policy::Durability::Volatile, @@ -85,6 +102,7 @@ impl From for policy::Durability { } /// DDS 2.2.3.11 LIVELINESS +/// :rtype: Ros2Liveliness #[derive(Copy, Clone, Debug, PartialEq)] #[pyclass] pub enum Ros2Liveliness { @@ -94,6 +112,8 @@ pub enum Ros2Liveliness { } impl Ros2Liveliness { + /// :type lease_duration: float + /// :rtype: Ros2Liveliness fn convert(self, lease_duration: f64) -> policy::Liveliness { let lease_duration = if lease_duration.is_infinite() { rustdds::Duration::INFINITE From e790af385ebc59b65248641da13dd4a38155eb0f Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 29 Apr 2024 20:12:56 +0200 Subject: [PATCH 02/13] Using `__init__.py` as without using it --- apis/python/node/README.md | 2 +- apis/python/node/dora/__init__.py | 28 ++++++++++ apis/python/node/dora/__init__.pyi | 55 +++++++++---------- apis/python/node/{dora => }/generate_stubs.py | 0 apis/python/node/src/lib.rs | 12 ++-- apis/python/operator/src/lib.rs | 3 +- .../extensions/ros2-bridge/python/src/lib.rs | 11 ---- test_typing.py | 17 ++++++ 8 files changed, 80 insertions(+), 48 deletions(-) create mode 100644 apis/python/node/dora/__init__.py rename apis/python/node/{dora => }/generate_stubs.py (100%) create mode 100644 test_typing.py diff --git a/apis/python/node/README.md b/apis/python/node/README.md index fd1d844f..1faffc77 100644 --- a/apis/python/node/README.md +++ b/apis/python/node/README.md @@ -16,6 +16,6 @@ maturin develop Type hinting requires to run a second step ```bash -python dora/generate_stubs.py dora dora/__init__.pyi +python generate_stubs.py dora dora/__init__.pyi maturin develop ``` diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py new file mode 100644 index 00000000..e145cc22 --- /dev/null +++ b/apis/python/node/dora/__init__.py @@ -0,0 +1,28 @@ +""" +# dora-rs +This is the dora python client for interacting with dora dataflow. +You can install it via: +```bash +pip install dora-rs +``` +""" + +from enum import Enum + +from .dora import ( + Node, + PyEvent, + DoraStatus, + Ros2Context, + Ros2Node, + Ros2NodeOptions, + Ros2Topic, + Ros2Publisher, + Ros2Subscription, + start_runtime, + __version__, + __author__, + Ros2QosPolicies, + Ros2Durability, + Ros2Liveliness, +) diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi index 3014cc35..5d99353a 100644 --- a/apis/python/node/dora/__init__.pyi +++ b/apis/python/node/dora/__init__.pyi @@ -3,6 +3,7 @@ import typing @typing.final class DoraStatus: + """Dora Status for dora python operators.""" def __eq__(self, value: typing.Any, /) -> bool: """Return self==value.""" @@ -31,6 +32,13 @@ class DoraStatus: STOP: DoraStatus = ... STOP_ALL: DoraStatus = ... +@typing.final +class Enum: + """Generic enumeration. + +Derive from this class to define new enumerations.""" + __members__: mappingproxy = ... + @typing.final class Node: """The custom node API lets you integrate `dora` into your application. @@ -56,9 +64,14 @@ from dora import Node node = Node() ```""" - def dataflow_descriptor(self, /) -> dict:... + def dataflow_descriptor(self, /) -> dict: + """Returns the full dataflow descriptor that this node is part of. - def merge_external_events(self, /, subscription: Ros2Subscription) -> None:... +This method returns the parsed dataflow YAML file.""" + + def merge_external_events(self, /, subscription: Ros2Subscription) -> None: + """Merge an external event stream with dora main loop. +This currently only work with ROS2.""" def next(self, /, timeout: float=None) -> PyEvent: """`.next()` gives you the next input that the node has received. @@ -102,6 +115,7 @@ node.send_output("string", b"string", {"open_telemetry_context": "7632e76"}) @typing.final class PyEvent: + """Dora Event""" def inner(self, /):... @@ -123,9 +137,7 @@ You can also use `ros_paths` if you don't want to use env variable. ```python context = Ros2Context() -``` - -list of paths to search for ROS2 message types defintion""" +```""" def __init__(self, /, ros_paths: List[str]=None) -> None: """ROS2 Context holding all messages definition for receiving and sending messages to ROS2. @@ -141,9 +153,7 @@ You can also use `ros_paths` if you don't want to use env variable. ```python context = Ros2Context() -``` - -list of paths to search for ROS2 message types defintion""" +```""" def new_node(self, /, name: str, namespace: str, options: Ros2NodeOptions) -> Ros2Node: """Create a new ROS2 node @@ -154,11 +164,7 @@ ros2_node = ros2_context.new_node( "/ros2_demo", dora.experimental.ros2_bridge.Ros2NodeOptions(rosout=True), ) -``` - -name of the node -name of the namespace -options for the node""" +```""" @typing.final class Ros2Durability: @@ -236,18 +242,14 @@ See: https://github.com/jhelovuo/ros2-client/issues/4""" ```python pose_publisher = ros2_node.create_publisher(turtle_pose_topic) -``` - -QoS policies for the topic""" +```""" def create_subscription(self, /, topic: Ros2Topic, qos: Ros2QosPolicies=None) -> Ros2Subscription: """Create a ROS2 subscription ```python pose_reader = ros2_node.create_subscription(turtle_pose_topic) -``` - -QoS policies for the topic""" +```""" def create_topic(self, /, name: str, message_type: str, qos: Ros2QosPolicies) -> Ros2Topic: """Create a ROS2 topic to connect to. @@ -256,20 +258,14 @@ QoS policies for the topic""" turtle_twist_topic = ros2_node.create_topic( "/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos ) -``` - -name of the topic. e.g. "pose" -message type of the topic. e.g. "std_msgs::UInt8MultiArray" -QoS policies for the topic""" +```""" @typing.final class Ros2NodeOptions: - """ROS2 Node Options -enable rosout logging""" + """ROS2 Node Options""" def __init__(self, /, rosout: bool=None) -> None: - """ROS2 Node Options -enable rosout logging""" + """ROS2 Node Options""" @typing.final class Ros2Publisher: @@ -309,8 +305,7 @@ class Ros2Subscription: @typing.final class Ros2Topic: - """ROS2 Topic -enable rosout logging""" + """ROS2 Topic""" def start_runtime() -> None: """Start a runtime for Operators""" \ No newline at end of file diff --git a/apis/python/node/dora/generate_stubs.py b/apis/python/node/generate_stubs.py similarity index 100% rename from apis/python/node/dora/generate_stubs.py rename to apis/python/node/generate_stubs.py diff --git a/apis/python/node/src/lib.rs b/apis/python/node/src/lib.rs index 3bc03ebc..9ed27f29 100644 --- a/apis/python/node/src/lib.rs +++ b/apis/python/node/src/lib.rs @@ -59,6 +59,7 @@ impl Node { /// match event["id"]: /// case "image": /// ``` + /// /// :type timeout: float, optional /// :rtype: PyEvent #[allow(clippy::should_implement_trait)] @@ -141,17 +142,17 @@ impl Node { Ok(()) } - // Returns the full dataflow descriptor that this node is part of. - // - // This method returns the parsed dataflow YAML file. + /// Returns the full dataflow descriptor that this node is part of. + /// + /// This method returns the parsed dataflow YAML file. /// /// :rtype: dict pub fn dataflow_descriptor(&self, py: Python) -> pythonize::Result { pythonize::pythonize(py, self.node.dataflow_descriptor()) } - // Merge an external event stream with dora main loop. - // This currently only work with ROS2. + /// Merge an external event stream with dora main loop. + /// This currently only work with ROS2. /// /// :type subscription: Ros2Subscription /// :rtype: None @@ -239,6 +240,7 @@ pub fn start_runtime() -> eyre::Result<()> { dora_runtime::main().wrap_err("Dora Runtime raised an error.") } +/// Dora Status for dora python operators. /// :rtype: DoraStatus #[derive(Copy, Clone, Debug, PartialEq)] #[pyclass] diff --git a/apis/python/operator/src/lib.rs b/apis/python/operator/src/lib.rs index d4c3e475..13e5e671 100644 --- a/apis/python/operator/src/lib.rs +++ b/apis/python/operator/src/lib.rs @@ -3,13 +3,14 @@ use dora_node_api::{merged::MergedEvent, Event, Metadata, MetadataParameters}; use eyre::{Context, Result}; use pyo3::{exceptions::PyLookupError, prelude::*, types::PyDict}; -// Dora Event +/// Dora Event #[pyclass] pub struct PyEvent { event: MergedEvent, data: Option, } +// Dora Event #[pymethods] impl PyEvent { /// diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index a0107b58..3030856d 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -39,7 +39,6 @@ pub mod typed; /// ``` /// /// :type ros_paths: List[str], optional -/// list of paths to search for ROS2 message types defintion /// #[pyclass] pub struct Ros2Context { @@ -101,11 +100,8 @@ impl Ros2Context { /// ``` /// /// :type name: str - /// name of the node /// :type namespace: str - /// name of the namespace /// :type options: Ros2NodeOptions - /// options for the node /// :rtype: Ros2Node pub fn new_node( &self, @@ -148,11 +144,8 @@ impl Ros2Node { /// ``` /// /// :type name: str - /// name of the topic. e.g. "pose" /// :type message_type: str - /// message type of the topic. e.g. "std_msgs::UInt8MultiArray" /// :type qos: Ros2QosPolicies - /// QoS policies for the topic /// :rtype: Ros2Topic pub fn create_topic( &self, @@ -190,7 +183,6 @@ impl Ros2Node { /// /// :type topic: Ros2Topic /// :type qos: Ros2QosPolicies, optional - /// QoS policies for the topic /// :rtype: Ros2Publisher pub fn create_publisher( &mut self, @@ -214,7 +206,6 @@ impl Ros2Node { /// /// :type topic: Ros2Topic /// :type qos: Ros2QosPolicies, optional - /// QoS policies for the topic /// :rtype: Ros2Subscription pub fn create_subscription( &mut self, @@ -233,7 +224,6 @@ impl Ros2Node { /// ROS2 Node Options /// :type rosout: bool, optional -/// enable rosout logging /// #[derive(Debug, Clone, Default)] #[pyclass] @@ -260,7 +250,6 @@ impl From for ros2_client::NodeOptions { /// ROS2 Topic /// :type rosout: bool, optional -/// enable rosout logging /// #[pyclass] #[non_exhaustive] diff --git a/test_typing.py b/test_typing.py new file mode 100644 index 00000000..3c14e313 --- /dev/null +++ b/test_typing.py @@ -0,0 +1,17 @@ +from dora import ( + Node, + PyEvent, + DoraStatus, + Ros2Context, + Ros2Node, + Ros2NodeOptions, + Ros2Topic, + Ros2Publisher, + Ros2Subscription, + start_runtime, + __version__, + __author__, + Ros2QosPolicies, + Ros2Durability, + Ros2Liveliness, +) From 97c2dabdd7e371efee4b54726c7bab4e935fd9f5 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 29 Apr 2024 20:27:03 +0200 Subject: [PATCH 03/13] reintroduce `DoraStatus` --- apis/python/node/dora/__init__.py | 13 +++++++++++++ binaries/runtime/src/operator/python.rs | 5 ++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index e145cc22..58e7778d 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -26,3 +26,16 @@ from .dora import ( Ros2Durability, Ros2Liveliness, ) + + +class DoraStatus(Enum): + """Dora status to indicate if operator `on_input` loop + should be stopped. + Args: + Enum (u8): Status signaling to dora operator to + stop or continue the operator. + """ + + CONTINUE = 0 + STOP = 1 + STOP_ALL = 2 diff --git a/binaries/runtime/src/operator/python.rs b/binaries/runtime/src/operator/python.rs index fefc92a4..8ebbbfc2 100644 --- a/binaries/runtime/src/operator/python.rs +++ b/binaries/runtime/src/operator/python.rs @@ -224,9 +224,8 @@ pub fn run( .map_err(traceback); match status_enum { Ok(status_enum) => { - let status_val = - Python::with_gil(|py| status_enum.call_method0(py, "value")) - .wrap_err("on_event must have enum return value")?; + let status_val = Python::with_gil(|py| status_enum.getattr(py, "value")) + .wrap_err("on_event must have enum return value")?; Python::with_gil(|py| status_val.extract(py)) .wrap_err("on_event has invalid return value") } From d044cf87be4fa798d6f43eca0db94c1d8c2c20de Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 29 Apr 2024 21:21:32 +0200 Subject: [PATCH 04/13] Fixing DoraStatus return error --- apis/python/node/dora/__init__.py | 1 - apis/python/node/dora/__init__.pyi | 33 +------------------ apis/python/node/generate_stubs.py | 2 ++ .../python-ros2-dataflow/random_turtle.py | 12 +++---- .../extensions/ros2-bridge/python/src/lib.rs | 2 +- 5 files changed, 9 insertions(+), 41 deletions(-) diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 58e7778d..43e2ca2e 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -12,7 +12,6 @@ from enum import Enum from .dora import ( Node, PyEvent, - DoraStatus, Ros2Context, Ros2Node, Ros2NodeOptions, diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi index 5d99353a..e6bc35c7 100644 --- a/apis/python/node/dora/__init__.pyi +++ b/apis/python/node/dora/__init__.pyi @@ -1,37 +1,6 @@ import pyarrow import typing -@typing.final -class DoraStatus: - """Dora Status for dora python operators.""" - - def __eq__(self, value: typing.Any, /) -> bool: - """Return self==value.""" - - def __ge__(self, value: typing.Any, /) -> bool: - """Return self>=value.""" - - def __gt__(self, value: typing.Any, /) -> bool: - """Return self>value.""" - - def __int__(self, /) -> None: - """int(self)""" - - def __le__(self, value: typing.Any, /) -> bool: - """Return self<=value.""" - - def __lt__(self, value: typing.Any, /) -> bool: - """Return self bool: - """Return self!=value.""" - - def __repr__(self, /) -> str: - """Return repr(self).""" - CONTINUE: DoraStatus = ... - STOP: DoraStatus = ... - STOP_ALL: DoraStatus = ... - @typing.final class Enum: """Generic enumeration. @@ -162,7 +131,7 @@ context = Ros2Context() ros2_node = ros2_context.new_node( "turtle_teleop", "/ros2_demo", -dora.experimental.ros2_bridge.Ros2NodeOptions(rosout=True), +Ros2NodeOptions(rosout=True), ) ```""" diff --git a/apis/python/node/generate_stubs.py b/apis/python/node/generate_stubs.py index 47723489..d8b739b2 100644 --- a/apis/python/node/generate_stubs.py +++ b/apis/python/node/generate_stubs.py @@ -72,6 +72,8 @@ def module_stubs(module: Any) -> ast.Module: element_path = [module.__name__, member_name] if member_name.startswith("__"): pass + elif member_name.startswith("DoraStatus"): + pass elif inspect.isclass(member_value): classes.append( class_stubs(member_name, member_value, element_path, types_to_import) diff --git a/examples/python-ros2-dataflow/random_turtle.py b/examples/python-ros2-dataflow/random_turtle.py index e6fa2c75..20301963 100755 --- a/examples/python-ros2-dataflow/random_turtle.py +++ b/examples/python-ros2-dataflow/random_turtle.py @@ -1,23 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import dora -from dora import Node +import dora as dora +from dora import Node, Ros2Context, Ros2NodeOptions, Ros2QosPolicies CHECK_TICK = 50 # Create a ROS2 Context -ros2_context = dora.experimental.ros2_bridge.Ros2Context() +ros2_context = Ros2Context() ros2_node = ros2_context.new_node( "turtle_teleop", "/ros2_demo", - dora.experimental.ros2_bridge.Ros2NodeOptions(rosout=True), + Ros2NodeOptions(rosout=True), ) # Define a ROS2 QOS -topic_qos = dora.experimental.ros2_bridge.Ros2QosPolicies( - reliable=True, max_blocking_time=0.1 -) +topic_qos = Ros2QosPolicies(reliable=True, max_blocking_time=0.1) # Create a publisher to cmd_vel topic turtle_twist_topic = ros2_node.create_topic( diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index 3030856d..ac77d2a4 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -95,7 +95,7 @@ impl Ros2Context { /// ros2_node = ros2_context.new_node( /// "turtle_teleop", /// "/ros2_demo", - /// dora.experimental.ros2_bridge.Ros2NodeOptions(rosout=True), + /// Ros2NodeOptions(rosout=True), /// ) /// ``` /// From 102198a25cbf35142788901945b3e3492d8f17cc Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 30 Apr 2024 10:13:38 +0200 Subject: [PATCH 05/13] Add wildcard import in case an import is missing --- apis/python/node/dora/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 43e2ca2e..7269bda1 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -9,6 +9,8 @@ pip install dora-rs from enum import Enum +from .dora import * + from .dora import ( Node, PyEvent, From 597ee4142873cfe9dcfae05f3b09b8436ff38c60 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 30 Apr 2024 17:21:56 +0200 Subject: [PATCH 06/13] Improving typing by adding dora prefix --- apis/python/node/dora/__init__.pyi | 23 ++++++------ apis/python/node/generate_stubs.py | 2 ++ apis/python/node/src/lib.rs | 35 +++++-------------- apis/python/operator/src/lib.rs | 2 +- .../extensions/ros2-bridge/python/src/lib.rs | 22 ++++++------ .../extensions/ros2-bridge/python/src/qos.rs | 20 +++++------ 6 files changed, 45 insertions(+), 59 deletions(-) diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi index e6bc35c7..0d66a030 100644 --- a/apis/python/node/dora/__init__.pyi +++ b/apis/python/node/dora/__init__.pyi @@ -1,3 +1,4 @@ +import dora import pyarrow import typing @@ -38,11 +39,11 @@ node = Node() This method returns the parsed dataflow YAML file.""" - def merge_external_events(self, /, subscription: Ros2Subscription) -> None: + def merge_external_events(self, /, subscription: dora.Ros2Subscription) -> None: """Merge an external event stream with dora main loop. This currently only work with ROS2.""" - def next(self, /, timeout: float=None) -> PyEvent: + def next(self, /, timeout: float=None) -> dora.PyEvent: """`.next()` gives you the next input that the node has received. It blocks until the next event becomes available. You can use timeout in seconds to return if no input is available. @@ -62,16 +63,18 @@ match event["id"]: case "image": ```""" - def send_output(self, /, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None: + def send_output(self, /, output_id: str, data: pyarrow.Array, metadata: typing.Dict[str | str]=None) -> None: """`send_output` send data from the node. ```python Args: output_id: str, -data: Bytes|Arrow, +data: pyarrow.Array, metadata: Option[Dict], ``` +ex: + ```python node.send_output("string", b"string", {"open_telemetry_context": "7632e76"}) ```""" @@ -108,7 +111,7 @@ You can also use `ros_paths` if you don't want to use env variable. context = Ros2Context() ```""" - def __init__(self, /, ros_paths: List[str]=None) -> None: + def __init__(self, /, ros_paths: typing.List[str]=None) -> None: """ROS2 Context holding all messages definition for receiving and sending messages to ROS2. By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition. @@ -124,7 +127,7 @@ You can also use `ros_paths` if you don't want to use env variable. context = Ros2Context() ```""" - def new_node(self, /, name: str, namespace: str, options: Ros2NodeOptions) -> Ros2Node: + def new_node(self, /, name: str, namespace: str, options: dora.Ros2NodeOptions) -> dora.Ros2Node: """Create a new ROS2 node ```python @@ -206,21 +209,21 @@ Warnings: - There's a known issue about ROS2 nodes not being discoverable by ROS2 See: https://github.com/jhelovuo/ros2-client/issues/4""" - def create_publisher(self, /, topic: Ros2Topic, qos: Ros2QosPolicies=None) -> Ros2Publisher: + def create_publisher(self, /, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Publisher: """Create a ROS2 publisher ```python pose_publisher = ros2_node.create_publisher(turtle_pose_topic) ```""" - def create_subscription(self, /, topic: Ros2Topic, qos: Ros2QosPolicies=None) -> Ros2Subscription: + def create_subscription(self, /, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Subscription: """Create a ROS2 subscription ```python pose_reader = ros2_node.create_subscription(turtle_pose_topic) ```""" - def create_topic(self, /, name: str, message_type: str, qos: Ros2QosPolicies) -> Ros2Topic: + def create_topic(self, /, name: str, message_type: str, qos: dora.Ros2QosPolicies) -> dora.Ros2Topic: """Create a ROS2 topic to connect to. ```python @@ -263,7 +266,7 @@ pa.array( class Ros2QosPolicies: """ROS2 QoS Policy""" - def __init__(self, /, durability: Ros2Durability=None, liveliness: Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> Ros2QoSPolicies: + def __init__(self, /, durability: dora.Ros2Durability=None, liveliness: dora.Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> dora.Ros2QoSPolicies: """ROS2 QoS Policy""" @typing.final diff --git a/apis/python/node/generate_stubs.py b/apis/python/node/generate_stubs.py index d8b739b2..7144e0ed 100644 --- a/apis/python/node/generate_stubs.py +++ b/apis/python/node/generate_stubs.py @@ -421,6 +421,8 @@ def parse_type_to_ast( # TODO: Fix sequence if "Ros" in sequence and "2" in sequence: sequence = ["".join(sequence)] + elif "dora.Ros" in sequence and "2" in sequence: + sequence = ["".join(sequence)] for e in sequence: if e == "or": diff --git a/apis/python/node/src/lib.rs b/apis/python/node/src/lib.rs index 9ed27f29..f6f47ed9 100644 --- a/apis/python/node/src/lib.rs +++ b/apis/python/node/src/lib.rs @@ -61,7 +61,7 @@ impl Node { /// ``` /// /// :type timeout: float, optional - /// :rtype: PyEvent + /// :rtype: dora.PyEvent #[allow(clippy::should_implement_trait)] pub fn next(&mut self, py: Python, timeout: Option) -> PyResult> { let event = py.allow_threads(|| self.events.recv(timeout.map(Duration::from_secs_f32))); @@ -78,7 +78,7 @@ impl Node { /// case "image": /// ``` /// - /// :rtype: PyEvent + /// :rtype: dora.PyEvent pub fn __next__(&mut self, py: Python) -> PyResult> { let event = py.allow_threads(|| self.events.recv(None)); Ok(event) @@ -94,7 +94,7 @@ impl Node { /// case "image": /// ``` /// - /// :rtype: PyEvent + /// :rtype: dora.PyEvent fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -104,13 +104,16 @@ impl Node { /// ```python /// Args: /// output_id: str, - /// data: Bytes|Arrow, + /// data: pyarrow.Array, /// metadata: Option[Dict], /// ``` /// + /// ex: + /// /// ```python /// node.send_output("string", b"string", {"open_telemetry_context": "7632e76"}) /// ``` + /// /// :type output_id: str /// :type data: pyarrow.Array /// :type metadata: dict, optional @@ -154,7 +157,7 @@ impl Node { /// Merge an external event stream with dora main loop. /// This currently only work with ROS2. /// - /// :type subscription: Ros2Subscription + /// :type subscription: dora.Ros2Subscription /// :rtype: None pub fn merge_external_events( &mut self, @@ -240,34 +243,12 @@ pub fn start_runtime() -> eyre::Result<()> { dora_runtime::main().wrap_err("Dora Runtime raised an error.") } -/// Dora Status for dora python operators. -/// :rtype: DoraStatus -#[derive(Copy, Clone, Debug, PartialEq)] -#[pyclass] -pub enum DoraStatus { - CONTINUE, - STOP, - STOP_ALL, -} - -impl DoraStatus { - /// :rtype: int - pub fn value(self) -> i32 { - match self { - DoraStatus::CONTINUE => 0, - DoraStatus::STOP => 1, - DoraStatus::STOP_ALL => 2, - } - } -} - #[pymodule] fn dora(_py: Python, m: &PyModule) -> PyResult<()> { dora_ros2_bridge_python::create_dora_ros2_bridge_module(m)?; m.add_function(wrap_pyfunction!(start_runtime, m)?)?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.setattr("__version__", env!("CARGO_PKG_VERSION"))?; m.setattr("__author__", "Dora-rs Authors")?; diff --git a/apis/python/operator/src/lib.rs b/apis/python/operator/src/lib.rs index 13e5e671..a57054eb 100644 --- a/apis/python/operator/src/lib.rs +++ b/apis/python/operator/src/lib.rs @@ -14,7 +14,7 @@ pub struct PyEvent { #[pymethods] impl PyEvent { /// - /// :rtype: PyObject + /// :rtype: dora.PyObject pub fn __getitem__(&self, key: &str, py: Python<'_>) -> PyResult> { if key == "kind" { let kind = match &self.event { diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index ac77d2a4..bbfe6c82 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -38,7 +38,7 @@ pub mod typed; /// context = Ros2Context() /// ``` /// -/// :type ros_paths: List[str], optional +/// :type ros_paths: typing.List[str], optional /// #[pyclass] pub struct Ros2Context { @@ -101,8 +101,8 @@ impl Ros2Context { /// /// :type name: str /// :type namespace: str - /// :type options: Ros2NodeOptions - /// :rtype: Ros2Node + /// :type options: dora.Ros2NodeOptions + /// :rtype: dora.Ros2Node pub fn new_node( &self, name: &str, @@ -145,8 +145,8 @@ impl Ros2Node { /// /// :type name: str /// :type message_type: str - /// :type qos: Ros2QosPolicies - /// :rtype: Ros2Topic + /// :type qos: dora.Ros2QosPolicies + /// :rtype: dora.Ros2Topic pub fn create_topic( &self, name: &str, @@ -181,9 +181,9 @@ impl Ros2Node { /// pose_publisher = ros2_node.create_publisher(turtle_pose_topic) /// ``` /// - /// :type topic: Ros2Topic - /// :type qos: Ros2QosPolicies, optional - /// :rtype: Ros2Publisher + /// :type topic: dora.Ros2Topic + /// :type qos: dora.Ros2QosPolicies, optional + /// :rtype: dora.Ros2Publisher pub fn create_publisher( &mut self, topic: &Ros2Topic, @@ -204,9 +204,9 @@ impl Ros2Node { /// pose_reader = ros2_node.create_subscription(turtle_pose_topic) /// ``` /// - /// :type topic: Ros2Topic - /// :type qos: Ros2QosPolicies, optional - /// :rtype: Ros2Subscription + /// :type topic: dora.Ros2Topic + /// :type qos: dora.Ros2QosPolicies, optional + /// :rtype: dora.Ros2Subscription pub fn create_subscription( &mut self, topic: &Ros2Topic, diff --git a/libraries/extensions/ros2-bridge/python/src/qos.rs b/libraries/extensions/ros2-bridge/python/src/qos.rs index 1421dfb5..626934f5 100644 --- a/libraries/extensions/ros2-bridge/python/src/qos.rs +++ b/libraries/extensions/ros2-bridge/python/src/qos.rs @@ -3,14 +3,14 @@ use pyo3::prelude::{pyclass, pymethods}; /// ROS2 QoS Policy /// -/// :type durability: Ros2Durability, optional -/// :type liveliness: Ros2Liveliness, optional +/// :type durability: dora.Ros2Durability, optional +/// :type liveliness: dora.Ros2Liveliness, optional /// :type reliable: bool, optional /// :type keep_all: bool, optional /// :type lease_duration: float, optional /// :type max_blocking_time: float, optional /// :type keep_last: int, optional -/// :rtype: Ros2QoSPolicies +/// :rtype: dora.Ros2QoSPolicies /// #[derive(Debug, Clone)] #[pyclass] @@ -76,7 +76,7 @@ impl From for rustdds::QosPolicies { /// DDS 2.2.3.4 DURABILITY /// -/// :rtype: Ros2Durability +/// :rtype: dora.Ros2Durability #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[pyclass] pub enum Ros2Durability { @@ -86,11 +86,11 @@ pub enum Ros2Durability { Persistent, } -/// :type value: Ros2Durability -/// :rtype: Ros2Durability +/// :type value: dora.Ros2Durability +/// :rtype: dora.Ros2Durability impl From for policy::Durability { - /// :type value: Ros2Durability - /// :rtype: Ros2Durability + /// :type value: dora.Ros2Durability + /// :rtype: dora.Ros2Durability fn from(value: Ros2Durability) -> Self { match value { Ros2Durability::Volatile => policy::Durability::Volatile, @@ -102,7 +102,7 @@ impl From for policy::Durability { } /// DDS 2.2.3.11 LIVELINESS -/// :rtype: Ros2Liveliness +/// :rtype: dora.Ros2Liveliness #[derive(Copy, Clone, Debug, PartialEq)] #[pyclass] pub enum Ros2Liveliness { @@ -113,7 +113,7 @@ pub enum Ros2Liveliness { impl Ros2Liveliness { /// :type lease_duration: float - /// :rtype: Ros2Liveliness + /// :rtype: dora.Ros2Liveliness fn convert(self, lease_duration: f64) -> policy::Liveliness { let lease_duration = if lease_duration.is_infinite() { rustdds::Duration::INFINITE From 0f52773767e0c070cf9384d0c131c07de1357f82 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 30 Apr 2024 17:23:06 +0200 Subject: [PATCH 07/13] Fix minor typo --- examples/python-ros2-dataflow/random_turtle.py | 1 - test_typing.py | 17 ----------------- 2 files changed, 18 deletions(-) delete mode 100644 test_typing.py diff --git a/examples/python-ros2-dataflow/random_turtle.py b/examples/python-ros2-dataflow/random_turtle.py index 20301963..1e690d07 100755 --- a/examples/python-ros2-dataflow/random_turtle.py +++ b/examples/python-ros2-dataflow/random_turtle.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import dora as dora from dora import Node, Ros2Context, Ros2NodeOptions, Ros2QosPolicies CHECK_TICK = 50 diff --git a/test_typing.py b/test_typing.py deleted file mode 100644 index 3c14e313..00000000 --- a/test_typing.py +++ /dev/null @@ -1,17 +0,0 @@ -from dora import ( - Node, - PyEvent, - DoraStatus, - Ros2Context, - Ros2Node, - Ros2NodeOptions, - Ros2Topic, - Ros2Publisher, - Ros2Subscription, - start_runtime, - __version__, - __author__, - Ros2QosPolicies, - Ros2Durability, - Ros2Liveliness, -) From abae9c771df6c91e60b543c1f950e1785cc3f8d9 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 1 May 2024 12:00:12 +0200 Subject: [PATCH 08/13] Adding unstable warning within ROS2Context --- README.md | 16 ++++++++++++++++ .../extensions/ros2-bridge/python/src/lib.rs | 14 ++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3216125f..e14a6f63 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,22 @@ We also have [a contributing guide](CONTRIBUTING.md). | **Supported Platforms (ARM)** | macOS, Linux | | **Configuration** | YAML | +### Unstable functionality + +Some parts of the public API are marked as **unstable**. +You can recognize this functionality from the warning in the API reference, or from the warning issued when the configuration option `warn_unstable` is active. +There are a number of reasons functionality may be marked as unstable: + +- We are unsure about the exact API. The name, function signature, or implementation are likely to change in the future. +- The functionality is not tested extensively yet. Bugs may pop up when used in real-world scenarios. +- The functionality does not integrate well with the full Polars API. You may find it works in one context but not in another. + +Releasing functionality as unstable allows us to gather important feedback from users that use Polars in real-world scenarios. +This helps us fine-tune things before giving it the final stamp of approval. +Users are only interested in solid, well-tested functionality can avoid this part of the API. + +Functionality marked as unstable may change at any point without it being considered a breaking change. + ## License This project is licensed under Apache-2.0. Check out [NOTICE.md](NOTICE.md) for more information. diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index bbfe6c82..a1a61727 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -34,6 +34,10 @@ pub mod typed; /// /// You can also use `ros_paths` if you don't want to use env variable. /// +/// warning:: +/// dora Ros2 bridge functionality is considered **unstable**. It may be changed +/// at any point without it being considered a breaking change. +/// /// ```python /// context = Ros2Context() /// ``` @@ -50,7 +54,14 @@ pub struct Ros2Context { impl Ros2Context { /// Create a new context #[new] - pub fn new(ros_paths: Option>) -> eyre::Result { + pub fn new(py: Python, ros_paths: Option>) -> eyre::Result { + let warnings = py + .import("warnings") + .wrap_err("failed to import `warnings` module")?; + warnings + .call_method1("warn", ("dora-rs ROS2 Bridge is unstable and may change at any point without it being considered a breaking change",)) + .wrap_err("failed to call `warnings.warn` module")?; + let ament_prefix_path = std::env::var("AMENT_PREFIX_PATH"); let empty = String::new(); @@ -124,7 +135,6 @@ impl Ros2Context { /// ROS2 Node /// /// Warnings: -/// - There's a known issue about ROS2 nodes not being discoverable by ROS2 /// See: https://github.com/jhelovuo/ros2-client/issues/4 /// #[pyclass] From d26216aae88cf277b29996fc92f9b034cfb7333a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 1 May 2024 14:08:34 +0200 Subject: [PATCH 09/13] Use pyo3::with_gil to call python --- .../extensions/ros2-bridge/python/src/lib.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index a1a61727..73acfeaf 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -11,7 +11,7 @@ use arrow::{ pyarrow::{FromPyArrow, ToPyArrow}, }; use dora_ros2_bridge_msg_gen::types::Message; -use eyre::{eyre, Context, ContextCompat}; +use eyre::{eyre, Context, ContextCompat, Result}; use futures::{Stream, StreamExt}; use pyo3::{ prelude::{pyclass, pymethods}, @@ -54,14 +54,16 @@ pub struct Ros2Context { impl Ros2Context { /// Create a new context #[new] - pub fn new(py: Python, ros_paths: Option>) -> eyre::Result { - let warnings = py - .import("warnings") - .wrap_err("failed to import `warnings` module")?; - warnings + pub fn new(ros_paths: Option>) -> eyre::Result { + Python::with_gil(|py| -> Result<()> { + let warnings = py + .import("warnings") + .wrap_err("failed to import `warnings` module")?; + warnings .call_method1("warn", ("dora-rs ROS2 Bridge is unstable and may change at any point without it being considered a breaking change",)) .wrap_err("failed to call `warnings.warn` module")?; - + Ok(()) + })?; let ament_prefix_path = std::env::var("AMENT_PREFIX_PATH"); let empty = String::new(); From 9f80dbf0c4c647f7dda19de00b986dbb7a5a20e0 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 2 May 2024 09:42:08 +0200 Subject: [PATCH 10/13] Adding warnings in the docsctring --- apis/python/node/dora/__init__.pyi | 12 ++++++++++-- libraries/extensions/ros2-bridge/python/src/lib.rs | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi index 0d66a030..b095ba07 100644 --- a/apis/python/node/dora/__init__.pyi +++ b/apis/python/node/dora/__init__.pyi @@ -63,7 +63,7 @@ match event["id"]: case "image": ```""" - def send_output(self, /, output_id: str, data: pyarrow.Array, metadata: typing.Dict[str | str]=None) -> None: + def send_output(self, /, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None: """`send_output` send data from the node. ```python @@ -107,6 +107,10 @@ AMENT_PREFIX_PATH folder structure should be the following: You can also use `ros_paths` if you don't want to use env variable. +warning:: +dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change. + ```python context = Ros2Context() ```""" @@ -123,6 +127,10 @@ AMENT_PREFIX_PATH folder structure should be the following: You can also use `ros_paths` if you don't want to use env variable. +warning:: +dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change. + ```python context = Ros2Context() ```""" @@ -205,7 +213,7 @@ class Ros2Liveliness: class Ros2Node: """ROS2 Node -Warnings: +warnings:: - There's a known issue about ROS2 nodes not being discoverable by ROS2 See: https://github.com/jhelovuo/ros2-client/issues/4""" diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index 73acfeaf..ce830a08 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -136,7 +136,8 @@ impl Ros2Context { /// ROS2 Node /// -/// Warnings: +/// warnings:: +/// - There's a known issue about ROS2 nodes not being discoverable by ROS2 /// See: https://github.com/jhelovuo/ros2-client/issues/4 /// #[pyclass] From c72ce08bc38efe78fc6744d9f8fa9597c7443f53 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 2 May 2024 09:46:29 +0200 Subject: [PATCH 11/13] Fixing wording --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e14a6f63..4f6841a6 100644 --- a/README.md +++ b/README.md @@ -265,15 +265,14 @@ We also have [a contributing guide](CONTRIBUTING.md). ### Unstable functionality -Some parts of the public API are marked as **unstable**. -You can recognize this functionality from the warning in the API reference, or from the warning issued when the configuration option `warn_unstable` is active. +`dora-rs` Ros2 Bridge is marked as **unstable**. There are a number of reasons functionality may be marked as unstable: - We are unsure about the exact API. The name, function signature, or implementation are likely to change in the future. - The functionality is not tested extensively yet. Bugs may pop up when used in real-world scenarios. -- The functionality does not integrate well with the full Polars API. You may find it works in one context but not in another. +- The functionality does not integrate well with the full dora-rs API. You may find it works in one context but not in another. -Releasing functionality as unstable allows us to gather important feedback from users that use Polars in real-world scenarios. +Releasing functionality as unstable allows us to gather important feedback from users that use dora-rs in real-world scenarios. This helps us fine-tune things before giving it the final stamp of approval. Users are only interested in solid, well-tested functionality can avoid this part of the API. From bdf90f7219280516c79058a12ac3391908b4626b Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 2 May 2024 10:05:41 +0200 Subject: [PATCH 12/13] Fix positional argument typing --- apis/python/node/dora/__init__.pyi | 68 +++++++++++++++--------------- apis/python/node/generate_stubs.py | 5 ++- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi index b095ba07..7f3d6fb2 100644 --- a/apis/python/node/dora/__init__.pyi +++ b/apis/python/node/dora/__init__.pyi @@ -22,7 +22,7 @@ from dora import Node node = Node() ```""" - def __init__(self, /) -> None: + def __init__(self) -> None: """The custom node API lets you integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. @@ -34,16 +34,16 @@ from dora import Node node = Node() ```""" - def dataflow_descriptor(self, /) -> dict: + def dataflow_descriptor(self) -> dict: """Returns the full dataflow descriptor that this node is part of. This method returns the parsed dataflow YAML file.""" - def merge_external_events(self, /, subscription: dora.Ros2Subscription) -> None: + def merge_external_events(self, subscription: dora.Ros2Subscription) -> None: """Merge an external event stream with dora main loop. This currently only work with ROS2.""" - def next(self, /, timeout: float=None) -> dora.PyEvent: + def next(self, timeout: float=None) -> dora.PyEvent: """`.next()` gives you the next input that the node has received. It blocks until the next event becomes available. You can use timeout in seconds to return if no input is available. @@ -63,7 +63,7 @@ match event["id"]: case "image": ```""" - def send_output(self, /, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None: + def send_output(self, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None: """`send_output` send data from the node. ```python @@ -79,19 +79,19 @@ ex: node.send_output("string", b"string", {"open_telemetry_context": "7632e76"}) ```""" - def __iter__(self, /) -> typing.Any: + def __iter__(self) -> typing.Any: """Implement iter(self).""" - def __next__(self, /) -> typing.Any: + def __next__(self) -> typing.Any: """Implement next(self).""" @typing.final class PyEvent: """Dora Event""" - def inner(self, /):... + def inner(self):... - def __getitem__(self, key: typing.Any, /) -> typing.Any: + def __getitem__(self, key: typing.Any) -> typing.Any: """Return self[key].""" @typing.final @@ -115,7 +115,7 @@ at any point without it being considered a breaking change. context = Ros2Context() ```""" - def __init__(self, /, ros_paths: typing.List[str]=None) -> None: + def __init__(self, ros_paths: typing.List[str]=None) -> None: """ROS2 Context holding all messages definition for receiving and sending messages to ROS2. By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition. @@ -135,7 +135,7 @@ at any point without it being considered a breaking change. context = Ros2Context() ```""" - def new_node(self, /, name: str, namespace: str, options: dora.Ros2NodeOptions) -> dora.Ros2Node: + def new_node(self, name: str, namespace: str, options: dora.Ros2NodeOptions) -> dora.Ros2Node: """Create a new ROS2 node ```python @@ -150,28 +150,28 @@ Ros2NodeOptions(rosout=True), class Ros2Durability: """DDS 2.2.3.4 DURABILITY""" - def __eq__(self, value: typing.Any, /) -> bool: + def __eq__(self, value: typing.Any) -> bool: """Return self==value.""" - def __ge__(self, value: typing.Any, /) -> bool: + def __ge__(self, value: typing.Any) -> bool: """Return self>=value.""" - def __gt__(self, value: typing.Any, /) -> bool: + def __gt__(self, value: typing.Any) -> bool: """Return self>value.""" - def __int__(self, /) -> None: + def __int__(self) -> None: """int(self)""" - def __le__(self, value: typing.Any, /) -> bool: + def __le__(self, value: typing.Any) -> bool: """Return self<=value.""" - def __lt__(self, value: typing.Any, /) -> bool: + def __lt__(self, value: typing.Any) -> bool: """Return self bool: + def __ne__(self, value: typing.Any) -> bool: """Return self!=value.""" - def __repr__(self, /) -> str: + def __repr__(self) -> str: """Return repr(self).""" Persistent: Ros2Durability = ... Transient: Ros2Durability = ... @@ -182,28 +182,28 @@ class Ros2Durability: class Ros2Liveliness: """DDS 2.2.3.11 LIVELINESS""" - def __eq__(self, value: typing.Any, /) -> bool: + def __eq__(self, value: typing.Any) -> bool: """Return self==value.""" - def __ge__(self, value: typing.Any, /) -> bool: + def __ge__(self, value: typing.Any) -> bool: """Return self>=value.""" - def __gt__(self, value: typing.Any, /) -> bool: + def __gt__(self, value: typing.Any) -> bool: """Return self>value.""" - def __int__(self, /) -> None: + def __int__(self) -> None: """int(self)""" - def __le__(self, value: typing.Any, /) -> bool: + def __le__(self, value: typing.Any) -> bool: """Return self<=value.""" - def __lt__(self, value: typing.Any, /) -> bool: + def __lt__(self, value: typing.Any) -> bool: """Return self bool: + def __ne__(self, value: typing.Any) -> bool: """Return self!=value.""" - def __repr__(self, /) -> str: + def __repr__(self) -> str: """Return repr(self).""" Automatic: Ros2Liveliness = ... ManualByParticipant: Ros2Liveliness = ... @@ -217,21 +217,21 @@ warnings:: - There's a known issue about ROS2 nodes not being discoverable by ROS2 See: https://github.com/jhelovuo/ros2-client/issues/4""" - def create_publisher(self, /, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Publisher: + def create_publisher(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Publisher: """Create a ROS2 publisher ```python pose_publisher = ros2_node.create_publisher(turtle_pose_topic) ```""" - def create_subscription(self, /, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Subscription: + def create_subscription(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Subscription: """Create a ROS2 subscription ```python pose_reader = ros2_node.create_subscription(turtle_pose_topic) ```""" - def create_topic(self, /, name: str, message_type: str, qos: dora.Ros2QosPolicies) -> dora.Ros2Topic: + def create_topic(self, name: str, message_type: str, qos: dora.Ros2QosPolicies) -> dora.Ros2Topic: """Create a ROS2 topic to connect to. ```python @@ -244,14 +244,14 @@ turtle_twist_topic = ros2_node.create_topic( class Ros2NodeOptions: """ROS2 Node Options""" - def __init__(self, /, rosout: bool=None) -> None: + def __init__(self, rosout: bool=None) -> None: """ROS2 Node Options""" @typing.final class Ros2Publisher: """ROS2 Publisher""" - def publish(self, /, data: pyarrow.Array) -> None: + def publish(self, data: pyarrow.Array) -> None: """Publish a message into ROS2 topic. Remember that the data format should respect the structure of the ROS2 message usinng an arrow Structure. @@ -274,14 +274,14 @@ pa.array( class Ros2QosPolicies: """ROS2 QoS Policy""" - def __init__(self, /, durability: dora.Ros2Durability=None, liveliness: dora.Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> dora.Ros2QoSPolicies: + def __init__(self, durability: dora.Ros2Durability=None, liveliness: dora.Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> dora.Ros2QoSPolicies: """ROS2 QoS Policy""" @typing.final class Ros2Subscription: """ROS2 Subscription""" - def next(self, /):... + def next(self):... @typing.final class Ros2Topic: diff --git a/apis/python/node/generate_stubs.py b/apis/python/node/generate_stubs.py index 7144e0ed..db9e3f83 100644 --- a/apis/python/node/generate_stubs.py +++ b/apis/python/node/generate_stubs.py @@ -333,8 +333,9 @@ def arguments_stub( ) if param.kind == param.POSITIONAL_ONLY: - posonlyargs.append(param_ast) - defaults.append(default_ast) + args.append(param_ast) + # posonlyargs.append(param_ast) + # defaults.append(default_ast) elif param.kind == param.POSITIONAL_OR_KEYWORD: args.append(param_ast) defaults.append(default_ast) From ef3f0876824bac1767c281b7799f0f910a3792a8 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 2 May 2024 10:19:36 +0200 Subject: [PATCH 13/13] Adding additional ros2 bridge warnings in the documentation --- apis/python/node/dora/__init__.pyi | 38 ++++++++++++++++--- .../extensions/ros2-bridge/python/src/lib.rs | 24 ++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/apis/python/node/dora/__init__.pyi b/apis/python/node/dora/__init__.pyi index 7f3d6fb2..e74b2a96 100644 --- a/apis/python/node/dora/__init__.pyi +++ b/apis/python/node/dora/__init__.pyi @@ -144,7 +144,11 @@ ros2_node = ros2_context.new_node( "/ros2_demo", Ros2NodeOptions(rosout=True), ) -```""" +``` + +warning:: +dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change.""" @typing.final class Ros2Durability: @@ -214,6 +218,8 @@ class Ros2Node: """ROS2 Node warnings:: +- dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change. - There's a known issue about ROS2 nodes not being discoverable by ROS2 See: https://github.com/jhelovuo/ros2-client/issues/4""" @@ -222,14 +228,21 @@ See: https://github.com/jhelovuo/ros2-client/issues/4""" ```python pose_publisher = ros2_node.create_publisher(turtle_pose_topic) -```""" +``` +warnings: +- dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change.""" def create_subscription(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Subscription: """Create a ROS2 subscription ```python pose_reader = ros2_node.create_subscription(turtle_pose_topic) -```""" +``` + +warnings: +- dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change.""" def create_topic(self, name: str, message_type: str, qos: dora.Ros2QosPolicies) -> dora.Ros2Topic: """Create a ROS2 topic to connect to. @@ -249,7 +262,11 @@ class Ros2NodeOptions: @typing.final class Ros2Publisher: - """ROS2 Publisher""" + """ROS2 Publisher + +warnings: +- dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change.""" def publish(self, data: pyarrow.Array) -> None: """Publish a message into ROS2 topic. @@ -279,13 +296,22 @@ class Ros2QosPolicies: @typing.final class Ros2Subscription: - """ROS2 Subscription""" + """ROS2 Subscription + + +warnings: +- dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change.""" def next(self):... @typing.final class Ros2Topic: - """ROS2 Topic""" + """ROS2 Topic + +warnings: +- dora Ros2 bridge functionality is considered **unstable**. It may be changed +at any point without it being considered a breaking change.""" def start_runtime() -> None: """Start a runtime for Operators""" \ No newline at end of file diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index ce830a08..b335a4e8 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -112,6 +112,10 @@ impl Ros2Context { /// ) /// ``` /// + /// warning:: + /// dora Ros2 bridge functionality is considered **unstable**. It may be changed + /// at any point without it being considered a breaking change. + /// /// :type name: str /// :type namespace: str /// :type options: dora.Ros2NodeOptions @@ -137,6 +141,8 @@ impl Ros2Context { /// ROS2 Node /// /// warnings:: +/// - dora Ros2 bridge functionality is considered **unstable**. It may be changed +/// at any point without it being considered a breaking change. /// - There's a known issue about ROS2 nodes not being discoverable by ROS2 /// See: https://github.com/jhelovuo/ros2-client/issues/4 /// @@ -193,6 +199,9 @@ impl Ros2Node { /// ```python /// pose_publisher = ros2_node.create_publisher(turtle_pose_topic) /// ``` + /// warnings: + /// - dora Ros2 bridge functionality is considered **unstable**. It may be changed + /// at any point without it being considered a breaking change. /// /// :type topic: dora.Ros2Topic /// :type qos: dora.Ros2QosPolicies, optional @@ -217,6 +226,10 @@ impl Ros2Node { /// pose_reader = ros2_node.create_subscription(turtle_pose_topic) /// ``` /// + /// warnings: + /// - dora Ros2 bridge functionality is considered **unstable**. It may be changed + /// at any point without it being considered a breaking change. + /// /// :type topic: dora.Ros2Topic /// :type qos: dora.Ros2QosPolicies, optional /// :rtype: dora.Ros2Subscription @@ -264,6 +277,9 @@ impl From for ros2_client::NodeOptions { /// ROS2 Topic /// :type rosout: bool, optional /// +/// warnings: +/// - dora Ros2 bridge functionality is considered **unstable**. It may be changed +/// at any point without it being considered a breaking change. #[pyclass] #[non_exhaustive] pub struct Ros2Topic { @@ -272,6 +288,10 @@ pub struct Ros2Topic { } /// ROS2 Publisher +/// +/// warnings: +/// - dora Ros2 bridge functionality is considered **unstable**. It may be changed +/// at any point without it being considered a breaking change. #[pyclass] #[non_exhaustive] pub struct Ros2Publisher { @@ -338,6 +358,10 @@ impl Ros2Publisher { /// ROS2 Subscription /// +/// +/// warnings: +/// - dora Ros2 bridge functionality is considered **unstable**. It may be changed +/// at any point without it being considered a breaking change. #[pyclass] #[non_exhaustive] pub struct Ros2Subscription {