Skip to content

Client

Attributes

GRPC_CLIENT_CONN_RETRY_TIMEOUT module-attribute

GRPC_CLIENT_CONN_RETRY_TIMEOUT = 2

GRPC_CLIENT_TASK_REQUEST_TIMEOUT module-attribute

GRPC_CLIENT_TASK_REQUEST_TIMEOUT = 3600

MTLS_PEER_ID_HEADER module-attribute

MTLS_PEER_ID_HEADER = 'fbm-peer-component-id'

RESEARCHER_UNAVAILABLE_LOG_INTERVAL module-attribute

RESEARCHER_UNAVAILABLE_LOG_INTERVAL = 5 * 60

RESEARCHER_UNAVAILABLE_STATE_REFRESH module-attribute

RESEARCHER_UNAVAILABLE_STATE_REFRESH = timedelta(minutes=30)

Classes

Channels

Channels(researcher, on_connection_state=None)

Keeps gRPC server channels

Parameters:

Name Type Description Default
researcher ResearcherCredentials

An instance of ResearcherCredentials

required
on_connection_state Optional[Callable]

Called with the state of this channel whenever it changes, for a node that records it. None reports nothing.

None
Source code in fedbiomed/transport/client.py
def __init__(
    self,
    researcher: ResearcherCredentials,
    on_connection_state: Optional[Callable] = None,
):
    """Create channels and stubs

    Args:
        researcher: An instance of ResearcherCredentials
        on_connection_state: Called with the state of this channel whenever it
            changes, for a node that records it. None reports nothing.
    """
    self._researcher = researcher
    self._on_connection_state = on_connection_state

    # Id of the researcher on the other end, once the node learns it
    self.researcher_id: Optional[str] = None

    # Last time the researcher was logged as unavailable at INFO level
    self._unavailable_logged_at: Optional[float] = None

    self._channels = {}
    self._stubs = {}
    self._stub_types = [
        _StubType.LISTENER_TASK_STUB,
        _StubType.SENDER_TASK_STUB,
        _StubType.SENDER_FEEDBACK_STUB,
    ]
    for st in self._stub_types:
        self._channels[st]: grpc.aio.Channel = None
        self._stubs[st]: ResearcherServiceStub = None

    # lock for accessing channels and stubs
    self._channels_stubs_lock = asyncio.Lock()

Attributes

certificate property
certificate

Researcher server certificate the channel verifies the peer against.

endpoint property
endpoint

Researcher server endpoint as host:port.

host property
host

Researcher server host.

mtls property
mtls

Whether the node connects to the researcher with mutual authentication.

port property
port

Researcher server port.

researcher_id instance-attribute
researcher_id = None

Methods:

connect async
connect(stub_type=_StubType.ANY_STUB)

Connects gRPC server and instantiates stubs.

Parameters:

Name Type Description Default
stub_type _StubType

only (re)connect for matching stub type(s)

ANY_STUB
Source code in fedbiomed/transport/client.py
async def connect(self, stub_type: _StubType = _StubType.ANY_STUB):
    """Connects gRPC server and instantiates stubs.

    Args:
        stub_type: only (re)connect for matching stub type(s)
    """

    async with self._channels_stubs_lock:
        san_names = certificate_san_names(self._researcher.certificate)

        # A pinned certificate would be verified on its Common Name, free text the
        # issuer fills as it likes. A fetched one is trusted as served, so its name
        # protects nothing. Refused before the close loop, so it tears down no
        # channel.
        if self.mtls and not san_names:
            msg = (
                f"{ErrorNumbers.FB628.value}: The researcher certificate at "
                f"{self.endpoint} states no host: its Subject Alternative Name "
                "carries no host name and no address, so nothing in it says which "
                "server it is valid for. Request the researcher to reissue its "
                "certificate for the hosts nodes reach it at."
            )
            logger.error(msg)
            raise FedbiomedCommunicationError(msg)

        # Closes if channels are open
        for st, channel in self._channels.items():
            if channel and (stub_type == _StubType.ANY_STUB or stub_type == st):
                await channel.close()

        # Creates channels
        for st in self._channels.keys():
            if stub_type == _StubType.ANY_STUB or stub_type == st:
                self._channels[st] = self._create(san_names)
                self._stubs[st] = ResearcherServiceStub(channel=self._channels[st])
report_researcher_unavailable
report_researcher_unavailable()

Reports that the researcher server does not answer while it is retried.

The node retries every few seconds, so the recorded state is refreshed rarely and the INFO log is emitted only every few minutes; the retries themselves are logged at DEBUG by the callers.

Source code in fedbiomed/transport/client.py
def report_researcher_unavailable(self) -> None:
    """Reports that the researcher server does not answer while it is retried.

    The node retries every few seconds, so the recorded state is refreshed
    rarely and the INFO log is emitted only every few minutes; the retries
    themselves are logged at DEBUG by the callers.
    """
    endpoint = self.endpoint
    now = time.monotonic()
    if (
        self._unavailable_logged_at is None
        or now - self._unavailable_logged_at >= RESEARCHER_UNAVAILABLE_LOG_INTERVAL
    ):
        self._unavailable_logged_at = now
        logger.info(
            f"Researcher server at {endpoint} is not available, "
            "the node keeps retrying to connect."
        )

    self.report_state(
        ClientStatus.DISCONNECTED,
        operation="researcher_unavailable",
        reason=f"Researcher server at {endpoint} is not available; retrying.",
        refresh_interval=RESEARCHER_UNAVAILABLE_STATE_REFRESH,
    )
report_state
report_state(
    state,
    *,
    operation=None,
    reason=None,
    identity_verified=None,
    refresh_interval=None,
)

Reports the state of this channel to the node, when it asked to be told.

The facts of the channel itself are added here; the caller supplies what only it knows. operation names the matching security audit event. refresh_interval tells how rarely the node may refresh this state while it repeats itself; the node's default applies when None.

Source code in fedbiomed/transport/client.py
def report_state(
    self,
    state: ClientStatus,
    *,
    operation: Optional[str] = None,
    reason: Optional[str] = None,
    identity_verified: Optional[bool] = None,
    refresh_interval: Optional[timedelta] = None,
) -> None:
    """Reports the state of this channel to the node, when it asked to be told.

    The facts of the channel itself are added here; the caller supplies what only
    it knows. `operation` names the matching security audit event.
    `refresh_interval` tells how rarely the node may refresh this state while it
    repeats itself; the node's default applies when None.
    """
    if self._on_connection_state is None:
        return

    extra = (
        {} if refresh_interval is None else {"refresh_interval": refresh_interval}
    )
    self._on_connection_state(
        state=state,
        host=self.host,
        port=self.port,
        mtls=self.mtls,
        researcher_id=self.researcher_id,
        identity_verified=identity_verified,
        operation=operation,
        reason=reason,
        certificate=certificate_audit_fields(self._researcher.certificate)
        if self._researcher.certificate
        else None,
        **extra,
    )
stub async
stub(stub_type)

Get stub for a given stub type.

Parameters:

Name Type Description Default
stub_type _StubType

the stub type to get

required

Returns:

Type Description
ResearcherServiceStub

the stub if it exists or None

Source code in fedbiomed/transport/client.py
async def stub(self, stub_type: _StubType) -> ResearcherServiceStub:
    """Get stub for a given stub type.

    Args:
        stub_type: the stub type to get

    Returns:
        the stub if it exists or None
    """
    if stub_type in self._stub_types:
        async with self._channels_stubs_lock:
            return self._stubs[stub_type]
    else:
        return None

ClientStatus

Bases: Enum

Attributes

CONNECTED class-attribute instance-attribute
CONNECTED = 1
DISCONNECTED class-attribute instance-attribute
DISCONNECTED = 0
FAILED class-attribute instance-attribute
FAILED = 2

GrpcClient

GrpcClient(
    node_id,
    researcher,
    update_id_map,
    on_connection_state=None,
)

An agent of remote researcher gRPC server.

Parameters:

Name Type Description Default
node_id str

unique ID of this node (connection client)

required
researcher ResearcherCredentials

the researcher to which the node connects (connection server)

required
update_id_map Awaitable

awaitable to call when updating the researcher ID, needs proper prototype

required
on_connection_state Optional[Callable]

Called with the connection state whenever it changes, for a node that records it. None reports nothing.

None
Source code in fedbiomed/transport/client.py
def __init__(
    self,
    node_id: str,
    researcher: ResearcherCredentials,
    update_id_map: Awaitable,
    on_connection_state: Optional[Callable] = None,
) -> None:
    """Class constructor

    Args:
        node_id: unique ID of this node (connection client)
        researcher: the researcher to which the node connects (connection server)
        update_id_map: awaitable to call when updating the researcher ID, needs proper prototype
        on_connection_state: Called with the connection state whenever it changes,
            for a node that records it. None reports nothing.
    """
    self._id = None
    self._researcher = researcher
    self._channels = Channels(researcher, on_connection_state)

    self._task_listener = TaskListener(
        channels=self._channels,
        node_id=node_id,
        on_status_change=self._on_status_change,
        update_id=self._update_id,
    )

    self._sender = Sender(
        channels=self._channels, on_status_change=self._on_status_change
    )

    # TODO: use `self._status` for finer gRPC agent handling.
    # Currently, the (tentative) status is maintained but not used
    self._status = ClientStatus.DISCONNECTED
    # lock for accessing self._status
    self._status_lock = asyncio.Lock()

    self._update_id_map = update_id_map
    self._tasks = []

Attributes

tasks property
tasks

Returns running asyncio task(s) owned by this client.

Methods:

send async
send(message)

Sends messages from node to researcher server.

Parameters:

Name Type Description Default
message Message

message to send from node to server

required
Source code in fedbiomed/transport/client.py
async def send(self, message: Message) -> None:
    """Sends messages from node to researcher server.

    Args:
        message: message to send from node to server
    """

    await self._sender.send(message)
start
start(on_task)

Start researcher gRPC agent.

Starts long-lived tasks, one waiting for server requests, one waiting on the async queue for the replies from the node that are going to be sent back to researcher.

Parameters:

Name Type Description Default
on_task

Callback function to execute once a payload received from researcher.

required

Returns:

Type Description
Task

The main task object of the agent

Source code in fedbiomed/transport/client.py
def start(self, on_task) -> asyncio.Task:
    """Start researcher gRPC agent.

    Starts long-lived tasks, one waiting for server requests, one waiting on the async queue
    for the replies from the node that are going to be sent back to researcher.

    Args:
        on_task: Callback function to execute once a payload received from researcher.

    Returns:
        The main task object of the agent
    """

    async def run():
        """Connects and dispatches the tasks"""

        # First connects to channel
        await self._connect()

        # Launch listeners
        await asyncio.gather(
            self._task_listener.listen(on_task), self._sender.listen()
        )

    # Keep a stable reference so controller health checks can inspect client tasks.
    task = asyncio.create_task(run())
    self._tasks = [task]
    return task

Listener

Listener(channels)

Abstract generic listener method for a node's communications.

Parameters:

Name Type Description Default
channels Channels

Keeps channels and stubs.

required
Source code in fedbiomed/transport/client.py
def __init__(self, channels: Channels) -> None:
    """Constructs task listener channels

    Args:
        channels: Keeps channels and stubs.
    """
    self._channels = channels
    self._retry_on_error = False
    # Report the repeating failure once until the connection recovers.
    self._connection_failure_logged = False
    # Whether the last connection was retired on its maximum age.
    self._connection_recycled = False

Methods:

listen
listen(callback=None)

Listens for tasks from given channels

Parameters:

Name Type Description Default
callback Optional[Callable]

Callback function to execute once a task is processed

None

Returns:

Type Description
Awaitable[Optional[Callable]]

Asyncio task to run task listener

Source code in fedbiomed/transport/client.py
def listen(
    self, callback: Optional[Callable] = None
) -> Awaitable[Optional[Callable]]:
    """Listens for tasks from given channels

    Args:
        callback: Callback function to execute once a task is processed

    Returns:
        Asyncio task to run task listener
    """
    return asyncio.create_task(self._listen(callback))

NodeClientIdentity dataclass

NodeClientIdentity(
    private_key=None, certificate_chain=None
)

The node's own client identity, presented to the researcher.

Owned by the node, not the researcher. Only populated when mutual authentication is enabled.

Attributes

certificate_chain class-attribute instance-attribute
certificate_chain = None
private_key class-attribute instance-attribute
private_key = field(default=None, repr=False)

ResearcherCredentials dataclass

ResearcherCredentials(
    port,
    host,
    certificate=None,
    mtls=False,
    node_identity=None,
)

Connection details and pinned server certificate of a researcher.

Identifies the researcher endpoint (host/port) and pins its public server certificate. Under mutual authentication the node additionally presents its own client identity, carried separately in node_identity.

Attributes

certificate class-attribute instance-attribute
certificate = None
host instance-attribute
host
mtls class-attribute instance-attribute
mtls = False
node_identity class-attribute instance-attribute
node_identity = None
port instance-attribute
port

Sender

Sender(channels, on_status_change)

Bases: Listener

Parameters:

Name Type Description Default
channels Channels

RPC channels and stubs to be used for polling tasks from researcher

required
on_status_change Awaitable

Callback awaitable to run for changing node agent status

required
Source code in fedbiomed/transport/client.py
def __init__(
    self,
    channels: Channels,
    on_status_change: Awaitable,
) -> None:
    """Class constructor.

    Args:
        channels: RPC channels and stubs to be used for polling tasks from researcher
        on_status_change: Callback awaitable to run for changing node agent status
    """
    super().__init__(channels)

    self._queue = asyncio.Queue()
    self._on_status_change = on_status_change
    self._retry_count = 0
    self._retry_item = None
    self._stub_type = _StubType.NO_STUB
    self._retry_on_error = True

Methods:

send async
send(message)

Send a message to peer researcher.

Parameters:

Name Type Description Default
message Message

Message to send

required
Source code in fedbiomed/transport/client.py
async def send(self, message: Message) -> None:
    """Send a message to peer researcher.

    Args:
        message: Message to send
    """
    # Switch-case for message type and gRPC calls
    match message.__class__.__name__:
        case FeedbackMessage.__name__:
            # Note: FeedbackMessage is designed as proto serializable message.
            await self._queue.put(
                {"stub": _StubType.SENDER_FEEDBACK_STUB, "message": message}
            )

        case _:
            await self._queue.put(
                {"stub": _StubType.SENDER_TASK_STUB, "message": message}
            )

TaskListener

TaskListener(
    channels, node_id, on_status_change, update_id
)

Bases: Listener

Listener for the task assigned by the researcher component

Parameters:

Name Type Description Default
channels Channels

RPC channels and stubs to be used for polling tasks from researcher

required
node_id str

unique ID for this node

required
on_status_change Awaitable

Callback awaitable to run for changing node agent status

required
update_id Awaitable

Callback function to run updating peer researcher ID

required
Source code in fedbiomed/transport/client.py
def __init__(
    self,
    channels: Channels,
    node_id: str,
    on_status_change: Awaitable,
    update_id: Awaitable,
) -> None:
    """Class constructor.

    Args:
        channels: RPC channels and stubs to be used for polling tasks from researcher
        node_id: unique ID for this node
        on_status_change: Callback awaitable to run for changing node agent status
        update_id: Callback function to run updating peer researcher ID
    """
    super().__init__(channels)

    self._node_id = node_id
    self._on_status_change = on_status_change
    self._update_id = update_id
    self._retry_count = 0
    self._communication_established = False

Functions:

is_server_alive

is_server_alive(host, port)

Checks if the server is alive

Parameters:

Name Type Description Default
host str

The host/ip of researcher/server component

required
port str

Port number of researcher/server component

required
Source code in fedbiomed/transport/client.py
def is_server_alive(host: str, port: str):
    """Checks if the server is alive

    Args:
        host: The host/ip of researcher/server component
        port: Port number of researcher/server component
    """

    port = int(port)
    address_info = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
    for family, socktype, protocol, _, address in address_info:
        # Context manager so the socket is closed even when connect() raises
        with socket.socket(family, socktype, protocol) as s:
            # Need this timeout for the case where the server does not answer
            # If not present, socket timeout increases and this function takes more
            # than GRPC_CLIENT_CONN_RETRY_TIMEOUT to execute
            s.settimeout(GRPC_CLIENT_CONN_RETRY_TIMEOUT)
            try:
                s.connect(address)
            except socket.error:
                return False
            else:
                return True