Attributes¶
RESEARCHER_UNAVAILABLE_STATE_REFRESH module-attribute ¶
RESEARCHER_UNAVAILABLE_STATE_REFRESH = timedelta(minutes=30)
Classes¶
Channels ¶
Channels(researcher, on_connection_state=None)
Keeps gRPC server channels
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
researcher | ResearcherCredentials | An instance of ResearcherCredentials | required |
on_connection_state | Optional[Callable] | Called with the state of this channel whenever it changes, for a node that records it. None reports nothing. | None |
Source code in fedbiomed/transport/client.py
def __init__(
self,
researcher: ResearcherCredentials,
on_connection_state: Optional[Callable] = None,
):
"""Create channels and stubs
Args:
researcher: An instance of ResearcherCredentials
on_connection_state: Called with the state of this channel whenever it
changes, for a node that records it. None reports nothing.
"""
self._researcher = researcher
self._on_connection_state = on_connection_state
# Id of the researcher on the other end, once the node learns it
self.researcher_id: Optional[str] = None
# Last time the researcher was logged as unavailable at INFO level
self._unavailable_logged_at: Optional[float] = None
self._channels = {}
self._stubs = {}
self._stub_types = [
_StubType.LISTENER_TASK_STUB,
_StubType.SENDER_TASK_STUB,
_StubType.SENDER_FEEDBACK_STUB,
]
for st in self._stub_types:
self._channels[st]: grpc.aio.Channel = None
self._stubs[st]: ResearcherServiceStub = None
# lock for accessing channels and stubs
self._channels_stubs_lock = asyncio.Lock()
Attributes¶
certificate property ¶
certificate
Researcher server certificate the channel verifies the peer against.
Methods:¶
connect async ¶
connect(stub_type=_StubType.ANY_STUB)
Connects gRPC server and instantiates stubs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stub_type | _StubType | only (re)connect for matching stub type(s) | ANY_STUB |
Source code in fedbiomed/transport/client.py
async def connect(self, stub_type: _StubType = _StubType.ANY_STUB):
"""Connects gRPC server and instantiates stubs.
Args:
stub_type: only (re)connect for matching stub type(s)
"""
async with self._channels_stubs_lock:
san_names = certificate_san_names(self._researcher.certificate)
# A pinned certificate would be verified on its Common Name, free text the
# issuer fills as it likes. A fetched one is trusted as served, so its name
# protects nothing. Refused before the close loop, so it tears down no
# channel.
if self.mtls and not san_names:
msg = (
f"{ErrorNumbers.FB628.value}: The researcher certificate at "
f"{self.endpoint} states no host: its Subject Alternative Name "
"carries no host name and no address, so nothing in it says which "
"server it is valid for. Request the researcher to reissue its "
"certificate for the hosts nodes reach it at."
)
logger.error(msg)
raise FedbiomedCommunicationError(msg)
# Closes if channels are open
for st, channel in self._channels.items():
if channel and (stub_type == _StubType.ANY_STUB or stub_type == st):
await channel.close()
# Creates channels
for st in self._channels.keys():
if stub_type == _StubType.ANY_STUB or stub_type == st:
self._channels[st] = self._create(san_names)
self._stubs[st] = ResearcherServiceStub(channel=self._channels[st])
report_researcher_unavailable ¶
report_researcher_unavailable()
Reports that the researcher server does not answer while it is retried.
The node retries every few seconds, so the recorded state is refreshed rarely and the INFO log is emitted only every few minutes; the retries themselves are logged at DEBUG by the callers.
Source code in fedbiomed/transport/client.py
def report_researcher_unavailable(self) -> None:
"""Reports that the researcher server does not answer while it is retried.
The node retries every few seconds, so the recorded state is refreshed
rarely and the INFO log is emitted only every few minutes; the retries
themselves are logged at DEBUG by the callers.
"""
endpoint = self.endpoint
now = time.monotonic()
if (
self._unavailable_logged_at is None
or now - self._unavailable_logged_at >= RESEARCHER_UNAVAILABLE_LOG_INTERVAL
):
self._unavailable_logged_at = now
logger.info(
f"Researcher server at {endpoint} is not available, "
"the node keeps retrying to connect."
)
self.report_state(
ClientStatus.DISCONNECTED,
operation="researcher_unavailable",
reason=f"Researcher server at {endpoint} is not available; retrying.",
refresh_interval=RESEARCHER_UNAVAILABLE_STATE_REFRESH,
)
report_state ¶
report_state(
state,
*,
operation=None,
reason=None,
identity_verified=None,
refresh_interval=None,
)
Reports the state of this channel to the node, when it asked to be told.
The facts of the channel itself are added here; the caller supplies what only it knows. operation names the matching security audit event. refresh_interval tells how rarely the node may refresh this state while it repeats itself; the node's default applies when None.
Source code in fedbiomed/transport/client.py
def report_state(
self,
state: ClientStatus,
*,
operation: Optional[str] = None,
reason: Optional[str] = None,
identity_verified: Optional[bool] = None,
refresh_interval: Optional[timedelta] = None,
) -> None:
"""Reports the state of this channel to the node, when it asked to be told.
The facts of the channel itself are added here; the caller supplies what only
it knows. `operation` names the matching security audit event.
`refresh_interval` tells how rarely the node may refresh this state while it
repeats itself; the node's default applies when None.
"""
if self._on_connection_state is None:
return
extra = (
{} if refresh_interval is None else {"refresh_interval": refresh_interval}
)
self._on_connection_state(
state=state,
host=self.host,
port=self.port,
mtls=self.mtls,
researcher_id=self.researcher_id,
identity_verified=identity_verified,
operation=operation,
reason=reason,
certificate=certificate_audit_fields(self._researcher.certificate)
if self._researcher.certificate
else None,
**extra,
)
stub async ¶
stub(stub_type)
Get stub for a given stub type.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stub_type | _StubType | the stub type to get | required |
Returns:
| Type | Description |
|---|---|
ResearcherServiceStub | the stub if it exists or None |
Source code in fedbiomed/transport/client.py
async def stub(self, stub_type: _StubType) -> ResearcherServiceStub:
"""Get stub for a given stub type.
Args:
stub_type: the stub type to get
Returns:
the stub if it exists or None
"""
if stub_type in self._stub_types:
async with self._channels_stubs_lock:
return self._stubs[stub_type]
else:
return None
ClientStatus ¶
GrpcClient ¶
GrpcClient(
node_id,
researcher,
update_id_map,
on_connection_state=None,
)
An agent of remote researcher gRPC server.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
node_id | str | unique ID of this node (connection client) | required |
researcher | ResearcherCredentials | the researcher to which the node connects (connection server) | required |
update_id_map | Awaitable | awaitable to call when updating the researcher ID, needs proper prototype | required |
on_connection_state | Optional[Callable] | Called with the connection state whenever it changes, for a node that records it. None reports nothing. | None |
Source code in fedbiomed/transport/client.py
def __init__(
self,
node_id: str,
researcher: ResearcherCredentials,
update_id_map: Awaitable,
on_connection_state: Optional[Callable] = None,
) -> None:
"""Class constructor
Args:
node_id: unique ID of this node (connection client)
researcher: the researcher to which the node connects (connection server)
update_id_map: awaitable to call when updating the researcher ID, needs proper prototype
on_connection_state: Called with the connection state whenever it changes,
for a node that records it. None reports nothing.
"""
self._id = None
self._researcher = researcher
self._channels = Channels(researcher, on_connection_state)
self._task_listener = TaskListener(
channels=self._channels,
node_id=node_id,
on_status_change=self._on_status_change,
update_id=self._update_id,
)
self._sender = Sender(
channels=self._channels, on_status_change=self._on_status_change
)
# TODO: use `self._status` for finer gRPC agent handling.
# Currently, the (tentative) status is maintained but not used
self._status = ClientStatus.DISCONNECTED
# lock for accessing self._status
self._status_lock = asyncio.Lock()
self._update_id_map = update_id_map
self._tasks = []
Attributes¶
Methods:¶
send async ¶
send(message)
Sends messages from node to researcher server.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
message | Message | message to send from node to server | required |
Source code in fedbiomed/transport/client.py
async def send(self, message: Message) -> None:
"""Sends messages from node to researcher server.
Args:
message: message to send from node to server
"""
await self._sender.send(message)
start ¶
start(on_task)
Start researcher gRPC agent.
Starts long-lived tasks, one waiting for server requests, one waiting on the async queue for the replies from the node that are going to be sent back to researcher.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
on_task | Callback function to execute once a payload received from researcher. | required |
Returns:
| Type | Description |
|---|---|
Task | The main task object of the agent |
Source code in fedbiomed/transport/client.py
def start(self, on_task) -> asyncio.Task:
"""Start researcher gRPC agent.
Starts long-lived tasks, one waiting for server requests, one waiting on the async queue
for the replies from the node that are going to be sent back to researcher.
Args:
on_task: Callback function to execute once a payload received from researcher.
Returns:
The main task object of the agent
"""
async def run():
"""Connects and dispatches the tasks"""
# First connects to channel
await self._connect()
# Launch listeners
await asyncio.gather(
self._task_listener.listen(on_task), self._sender.listen()
)
# Keep a stable reference so controller health checks can inspect client tasks.
task = asyncio.create_task(run())
self._tasks = [task]
return task
Listener ¶
Listener(channels)
Abstract generic listener method for a node's communications.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
channels | Channels | Keeps channels and stubs. | required |
Source code in fedbiomed/transport/client.py
def __init__(self, channels: Channels) -> None:
"""Constructs task listener channels
Args:
channels: Keeps channels and stubs.
"""
self._channels = channels
self._retry_on_error = False
# Report the repeating failure once until the connection recovers.
self._connection_failure_logged = False
# Whether the last connection was retired on its maximum age.
self._connection_recycled = False
Methods:¶
listen ¶
listen(callback=None)
Listens for tasks from given channels
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
callback | Optional[Callable] | Callback function to execute once a task is processed | None |
Returns:
| Type | Description |
|---|---|
Awaitable[Optional[Callable]] | Asyncio task to run task listener |
Source code in fedbiomed/transport/client.py
def listen(
self, callback: Optional[Callable] = None
) -> Awaitable[Optional[Callable]]:
"""Listens for tasks from given channels
Args:
callback: Callback function to execute once a task is processed
Returns:
Asyncio task to run task listener
"""
return asyncio.create_task(self._listen(callback))
NodeClientIdentity dataclass ¶
NodeClientIdentity(
private_key=None, certificate_chain=None
)
The node's own client identity, presented to the researcher.
Owned by the node, not the researcher. Only populated when mutual authentication is enabled.
ResearcherCredentials dataclass ¶
ResearcherCredentials(
port,
host,
certificate=None,
mtls=False,
node_identity=None,
)
Connection details and pinned server certificate of a researcher.
Identifies the researcher endpoint (host/port) and pins its public server certificate. Under mutual authentication the node additionally presents its own client identity, carried separately in node_identity.
Sender ¶
Sender(channels, on_status_change)
Bases: Listener
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
channels | Channels | RPC channels and stubs to be used for polling tasks from researcher | required |
on_status_change | Awaitable | Callback awaitable to run for changing node agent status | required |
Source code in fedbiomed/transport/client.py
def __init__(
self,
channels: Channels,
on_status_change: Awaitable,
) -> None:
"""Class constructor.
Args:
channels: RPC channels and stubs to be used for polling tasks from researcher
on_status_change: Callback awaitable to run for changing node agent status
"""
super().__init__(channels)
self._queue = asyncio.Queue()
self._on_status_change = on_status_change
self._retry_count = 0
self._retry_item = None
self._stub_type = _StubType.NO_STUB
self._retry_on_error = True
Methods:¶
send async ¶
send(message)
Send a message to peer researcher.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
message | Message | Message to send | required |
Source code in fedbiomed/transport/client.py
async def send(self, message: Message) -> None:
"""Send a message to peer researcher.
Args:
message: Message to send
"""
# Switch-case for message type and gRPC calls
match message.__class__.__name__:
case FeedbackMessage.__name__:
# Note: FeedbackMessage is designed as proto serializable message.
await self._queue.put(
{"stub": _StubType.SENDER_FEEDBACK_STUB, "message": message}
)
case _:
await self._queue.put(
{"stub": _StubType.SENDER_TASK_STUB, "message": message}
)
TaskListener ¶
TaskListener(
channels, node_id, on_status_change, update_id
)
Bases: Listener
Listener for the task assigned by the researcher component
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
channels | Channels | RPC channels and stubs to be used for polling tasks from researcher | required |
node_id | str | unique ID for this node | required |
on_status_change | Awaitable | Callback awaitable to run for changing node agent status | required |
update_id | Awaitable | Callback function to run updating peer researcher ID | required |
Source code in fedbiomed/transport/client.py
def __init__(
self,
channels: Channels,
node_id: str,
on_status_change: Awaitable,
update_id: Awaitable,
) -> None:
"""Class constructor.
Args:
channels: RPC channels and stubs to be used for polling tasks from researcher
node_id: unique ID for this node
on_status_change: Callback awaitable to run for changing node agent status
update_id: Callback function to run updating peer researcher ID
"""
super().__init__(channels)
self._node_id = node_id
self._on_status_change = on_status_change
self._update_id = update_id
self._retry_count = 0
self._communication_established = False
Functions:¶
is_server_alive ¶
is_server_alive(host, port)
Checks if the server is alive
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
host | str | The host/ip of researcher/server component | required |
port | str | Port number of researcher/server component | required |
Source code in fedbiomed/transport/client.py
def is_server_alive(host: str, port: str):
"""Checks if the server is alive
Args:
host: The host/ip of researcher/server component
port: Port number of researcher/server component
"""
port = int(port)
address_info = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
for family, socktype, protocol, _, address in address_info:
# Context manager so the socket is closed even when connect() raises
with socket.socket(family, socktype, protocol) as s:
# Need this timeout for the case where the server does not answer
# If not present, socket timeout increases and this function takes more
# than GRPC_CLIENT_CONN_RETRY_TIMEOUT to execute
s.settimeout(GRPC_CLIENT_CONN_RETRY_TIMEOUT)
try:
s.connect(address)
except socket.error:
return False
else:
return True